def make_total_event_list(): """Make total event list. TODO: move this function to the datastore class and the sky box selection to the `make_counts_image` function. """ data_store = DataStore(dir=HESSFITS_MPP) observation_selection = dict(type='sky_box', frame='galactic', lon=Quantity([-120, 70], 'deg'), lat=Quantity([-5, 5], 'deg'), border=Quantity(2, 'deg') ) observation_table = data_store.make_observation_table(observation_selection) # For testing, only process a small subset of observations # observation_table = observation_table.select_linspace_subset(num=1000) event_list_files = data_store.make_table_of_files(observation_table, filetypes=['events']) ds = EventListDataset.vstack_from_files(event_list_files['filename']) print('Total number of events: {}'.format(len(ds.event_list))) print('Total number of GTIs: {}'.format(len(ds.good_time_intervals))) print('Converting EventListDataset to HDUList ...') hdu_list = ds.to_fits() print('Writing {}'.format(TOTAL_EVENTS_FILE)) hdu_list.writeto(TOTAL_EVENTS_FILE, clobber=True)
def test_image_pipe(tmpdir): tmpdir = str(tmpdir) from subprocess import call outdir = tmpdir outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2") ds.copy_obs(ds.obs_table, tmpdir) data_store = DataStore.from_dir(tmpdir) bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True) tmpdir = str(tmpdir) center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir(tmpdir) # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) # Pb with the load psftable for one of the run that is not implemented yet... data_store.hdu_table.remove_row(14) mosaic = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) mosaic.make_images(make_background_image=True, for_integral_flux=True, radius=10.) assert_allclose(mosaic.maps['counts'].data.sum(), 2334.0, atol=3) assert_allclose(mosaic.maps['bkg'].data.sum(), 1987.1513636663785, atol=3) assert_allclose(mosaic.maps['exposure'].data.sum(), 54190569251987.68, atol=3) assert_allclose(mosaic.maps['significance'].lookup(center), 33.707901541600634, atol=3) assert_allclose(mosaic.maps['excess'].data.sum(), 346.8486363336217, atol=3)
def test_image_pipe(tmpdir): tmpdir = str(tmpdir) from subprocess import call outdir = tmpdir outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) ds = DataStore.from_dir("$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2") ds.copy_obs(ds.obs_table, tmpdir) data_store = DataStore.from_dir(tmpdir) bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True) tmpdir = str(tmpdir) center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir(tmpdir) # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyMap.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = ExclusionMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) images = ImageAnalysis(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) images.make_maps(radius=10., bkg_norm=True, spectral_index=2.3, for_integral_flux=True)
def make_bg_model_two_groups(): from subprocess import call outdir = '/Users/jouvin/Desktop/these/temp/bg_model_image/' outdir2 = outdir + '/background' cmd = 'mkdir -p {}'.format(outdir2) print('Executing: {}'.format(cmd)) call(cmd, shell=True) cmd = 'cp -r $GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/ {}'.format(outdir) print('Executing: {}'.format(cmd)) call(cmd, shell=True) data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') bgmaker = OffDataBackgroundMaker(data_store, outdir=outdir2) bgmaker.select_observations(selection='all') bgmaker.group_observations() bgmaker.make_model("2D") bgmaker.save_models("2D") fn = outdir2 + '/group-def.fits' hdu_index_table = bgmaker.make_total_index_table( data_store=data_store, modeltype='2D', out_dir_background_model=outdir2, filename_obs_group_table=fn ) fn = outdir + '/hdu-index.fits.gz' hdu_index_table.write(fn, overwrite=True)
def make_image(): table = Table.read('acceptance_curve.fits') table.pprint() center = SkyCoord(83.63, 22.01, unit='deg').galactic counts_image = make_empty_image(nxpix=1000, nypix=1000, binsz=0.01, xref=center.l.deg, yref=center.b.deg, proj='TAN') bkg_image = counts_image.copy() data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') for events in data_store.load_all("events"): center = events.pointing_radec.galactic livetime = events.observation_live_time_duration solid_angle = Angle(0.01, "deg") ** 2 counts_image.data += bin_events_in_image(events, counts_image).data #interp_param = dict(bounds_error=False, fill_value=None) acc_hdu = fill_acceptance_image(bkg_image.header, center, table["offset"], table["Acceptance"]) acc = Quantity(acc_hdu.data, table["Acceptance"].unit) * solid_angle * livetime bkg_image.data += acc.decompose() print(acc.decompose().sum()) counts_image.writeto("counts_image.fits", clobber=True) bkg_image.writeto("bkg_image.fits", clobber=True)
def make_counts_array(): """Make an example counts array with energy and offset axes.""" data_store = DataStore.from_dir(gammapy_extra.dir / 'datasets/hess-crab4') event_lists = data_store.load_all('events') ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = np.linspace(0, 2.5, 100) array = EnergyOffsetArray(ebounds, offset) array.fill_events(event_lists) return array
def make_counts_array(): """Make an example counts array with energy and offset axes.""" data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') event_lists = data_store.load_all('events') ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = Angle(np.linspace(0, 2.5, 100), "deg") array = EnergyOffsetArray(ebounds, offset) array.fill_events(event_lists) return array
def test_obs(): # data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-pa') data_store.info() obs = data_store.obs(obs_id=23523) # import IPython; IPython.embed() # obs.info() # print(type(obs.events)) # print(type(obs.gti)) print(type(obs.aeff)) print(type(obs.edisp)) print(type(obs.psf))
def test_fill_cube(): filename = '$GAMMAPY_EXTRA/test_datasets/background/bg_cube_model_test1.fits' array = FOVCube.read(filename, format='table', scheme='bg_cube') array.data = Quantity(np.zeros_like(array.data.value), 'u') print(type(array.data)) dir = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2' data_store = DataStore.from_dir(dir) ev_list = data_store.load_all('events') array.fill_events(ev_list) array.write('test_background.fits', format='image', clobber=True)
def make_model(): dir = str(gammapy_extra.dir) + '/datasets/hess-crab4-hd-hap-prod2' data_store = DataStore.from_dir(dir) obs_table = data_store.obs_table ebounds = EnergyBounds.equal_log_spacing(0.1, 100, 100, 'TeV') offset = sqrt_space(start=0, stop=2.5, num=100) * u.deg excluded_sources = make_excluded_sources() multi_array = EnergyOffsetBackgroundModel(ebounds, offset) multi_array.fill_obs(obs_table, data_store, excluded_sources) #multi_array.fill_obs(obs_table, data_store) multi_array.compute_rate() bgarray = multi_array.bg_rate energy_range = Energy([1, 10], 'TeV') table = bgarray.acceptance_curve_in_energy_band(energy_range, energy_bins=10) multi_array.write('energy_offset_array.fits', overwrite=True) table.write('acceptance_curve.fits', overwrite=True)
def make_image_from_2d_bg(): center = SkyCoord(83.63, 22.01, unit='deg').galactic energy_band = Energy([1, 10], 'TeV') offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir('/Users/jouvin/Desktop/these/temp/bg_model_image/') # TODO: fix `binarize` implementation # exclusion_mask = exclusion_mask.binarize() image = SkyImage.empty(nxpix=250, nypix=250, binsz=0.02, xref=center.l.deg, yref=center.b.deg, proj='TAN', coordsys='GAL') refheader = image.to_image_hdu().header exclusion_mask = SkyMask.read('$GAMMAPY_EXTRA/datasets/exclusion_masks/tevcat_exclusion.fits') exclusion_mask = exclusion_mask.reproject(reference=refheader) mosaic_images = MosaicImage(image, energy_band=energy_band, offset_band=offset_band, data_store=data_store, obs_table=data_store.obs_table, exclusion_mask=exclusion_mask) mosaic_images.make_images(make_background_image=True, for_integral_flux=True, radius=10., make_psf = True, region_center=center) filename = 'fov_bg_maps.fits' log.info('Writing {}'.format(filename)) mosaic_images.maps.write(filename, clobber=True)
def run_test_fitsexport(directory): """Run example analysis to test a fits data production hap-data-fits-export crab has to be run in order to produce the example data """ log.info('Running test analysis of fits data') from gammapy.data import DataStore from gammapy.datasets import gammapy_extra from gammapy.utils.scripts import read_yaml from gammapy.spectrum.spectrum_pipe import run_spectrum_analysis_using_config from gammapy.spectrum.results import SpectrumResult s = DataStore.from_dir(directory) print(s.info()) configfile = gammapy_extra.filename( 'test_datasets/spectrum/spectrum_analysis_example.yaml') config = read_yaml(configfile) config['extraction']['data']['datastore'] = directory config['extraction']['data']['runlist'] = [23523, 23526, 23559, 23592] fit, analysis = run_spectrum_analysis_using_config(config) res = SpectrumResult(fit=fit.result, stats=analysis.observations.total_spectrum.spectrum_stats) print(res.to_table())
def run_benchmark(): # Set up data store and select N_OBS times the observation OBS_ID data_store = DataStore.from_dir('$GAMMAPY_EXTRA/test_datasets/cta_1dc/') obs_ids = OBS_ID * np.ones(N_OBS) obs_list = data_store.obs_list(obs_id=obs_ids) target_position = SkyCoord(0, 0, unit='deg', frame='galactic') on_radius = 0.2 * u.deg on_region = CircleSkyRegion(center=target_position, radius=on_radius) bkg_estimator = RingBackgroundEstimator( r_in=0.5 * u.deg, width=0.2 * u.deg, ) # Define reference image centered on the target xref = target_position.galactic.l.value yref = target_position.galactic.b.value ref_image = SkyImage.empty( nxpix=800, nypix=600, binsz=0.02, xref=xref, yref=yref, proj='TAN', coordsys='GAL', ) exclusion_mask = ref_image.region_mask(on_region) exclusion_mask.data = 1 - exclusion_mask.data image_estimator = IACTBasicImageEstimator( reference=ref_image, emin=100 * u.GeV, emax=100 * u.TeV, offset_max=3 * u.deg, background_estimator=bkg_estimator, exclusion_mask=exclusion_mask, ) result = image_estimator.run(obs_list)
def observations(): """Example observation list for testing.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps") return datastore.get_observations([111630])
def observations(): data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/") obs_id = [110380, 111140] return data_store.get_observations(obs_id)
def setup(self): self.data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps")
def observations_cta_dc1(): """CTA DC1 observation list.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/") obs_ids = [110380, 111140] return datastore.get_observations(obs_ids)
def observations_magic_dl3(): """MAGIC DL3 observation list.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/joint-crab/dl3/magic/") obs_ids = [5029748] return datastore.get_observations(obs_ids, required_irf=["aeff", "edisp"])
def data_prep(): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") OBS_ID = 23523 obs_ids = OBS_ID * np.ones(N_OBS) observations = data_store.get_observations(obs_ids) target_position = SkyCoord(ra=83.63, dec=22.01, unit="deg", frame="icrs") on_region_radius = Angle("0.11 deg") on_region = CircleSkyRegion(center=target_position, radius=on_region_radius) exclusion_region = CircleSkyRegion( center=SkyCoord(183.604, -8.708, unit="deg", frame="galactic"), radius=0.5 * u.deg, ) skydir = target_position.galactic exclusion_mask = Map.create(npix=(150, 150), binsz=0.05, skydir=skydir, proj="TAN", frame="galactic") mask = exclusion_mask.geom.region_mask([exclusion_region], inside=False) exclusion_mask.data = mask e_reco = MapAxis.from_bounds(0.1, 40, nbin=40, interp="log", unit="TeV", name="energy") e_true = MapAxis.from_bounds(0.05, 100, nbin=200, interp="log", unit="TeV", name="energy_true") empty = SpectrumDatasetOnOff.create( region=on_region, e_reco=e_reco, e_true=e_true, ) dataset_maker = SpectrumDatasetMaker(containment_correction=True, selection=["counts", "aeff", "edisp"]) bkg_maker = ReflectedRegionsBackgroundMaker(exclusion_mask=exclusion_mask) safe_mask_masker = SafeMaskMaker(methods=["aeff-max"], aeff_percent=10) spectral_model = PowerLawSpectralModel(index=2, amplitude=2e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.TeV) spatial_model = PointSpatialModel(lon_0=target_position.ra, lat_0=target_position.dec, frame="icrs") spatial_model.lon_0.frozen = True spatial_model.lat_0.frozen = True sky_model = SkyModel(spatial_model=spatial_model, spectral_model=spectral_model, name="") # Data preparation datasets = [] for idx, observation in enumerate(observations): dataset = empty.copy(name=f"dataset{idx}") dataset = dataset_maker.run(dataset=dataset, observation=observation) dataset_on_off = bkg_maker.run(dataset, observation) dataset_on_off = safe_mask_masker.run(dataset_on_off, observation) dataset_on_off.models = sky_model datasets.append(dataset_on_off) return Datasets(datasets)
def test_data_store2(): dir = gammapy_extra.dir / 'datasets/hess-crab4-hd-hap-prod2' ds = DataStore.from_dir(dir) ds.info()
def proceeding_plot(obs): # import IPython; IPython.embed() fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(12, 4)) obs.aeff.plot_energy_dependence(ax=axes[0]) edisp = obs.edisp.to_energy_dispersion(offset='1 deg') # edisp.plot_matrix(ax=axes[1]) psf_edep = obs.psf.to_table_psf(theta='1 deg') for energy in [1, 3] * u.TeV: psf = psf_edep.table_psf_at_energy(energy) psf.plot_psf_vs_theta() plt.xscale('linear') plt.yscale('linear') plt.xlim(0, 0.5) fig.tight_layout() filename = 'iact-dl3.pdf' print('Writing ', filename) fig.savefig(filename) if __name__ == '__main__': data_store = DataStore.from_dir('$HOME/code/HESS-DL3-DR1/release_store/') obs = data_store.obs(obs_id=23592) proceeding_plot(obs) peek_plots(obs)
def setup(self): data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps") self.observation = data_store.obs(111140)
def check_energy_binning_effects(): """Check how spectral fit results change with energy binnings. Actually this is still using the default: In [14]: print(analysis.extraction.observations[0].edisp) EnergyDispersion NDDataArray summary info e_true : size = 108, min = 0.010 TeV, max = 301.416 TeV e_reco : size = 72, min = 0.011 TeV, max = 93.804 TeV Data : size = 7776, min = 0.000, max = 1.000 But now, the fit results from SpectrumAnalysisIACT, which is just driving Fitspectrum, are different, almost the same as Sherpa. Current results Parameters: name value error unit min max frozen --------- --------- --------- --------------- --- --- ------ index 2.620e+00 1.540e-01 nan nan False amplitude 3.849e-11 5.407e-12 1 / (cm2 s TeV) nan nan False reference 1.000e+00 0.000e+00 TeV nan nan True Covariance: name/name index amplitude --------- -------- --------- index 0.0237 5.85e-13 amplitude 5.85e-13 2.92e-23 Statistic: -157.166 (cash) Fit Range: [ 1. 27.82559402] TeV ??? """ data_store = DataStore.from_dir("data/hess") obs_list = data_store.obs_list([23523]) on_region = CircleSkyRegion(conf.crab_position, conf.on_radius["hess"]) fp_binning = [1, 10, 30] * u.TeV exclusion_mask = get_exclusion_mask(conf.crab_position) model = PowerLaw( amplitude=1.23 * 1e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.Unit("TeV"), index=2.14 * u.Unit(""), ) cfg = dict( outdir=None, background=dict( on_region=on_region, exclusion_mask=exclusion_mask, # min_distance=0.1 * u.rad, ), extraction=dict(containment_correction=True), fit=dict( model=model, stat="cash", # forward_folded=True, fit_range=energy_range, ), fp_binning=fp_binning, ) analysis = SpectrumAnalysisIACT(obs_list, cfg) analysis.run() analysis.fit.est_errors() print("BEFORE", analysis.fit.result[0]) # print(analysis.extraction.observations[0].edisp) # pprint(analysis.fit.__dict__) # obs = analysis.fit.obs_list[0] # print(obs) # # import IPython; IPython.embed() # obs.peek() # import matplotlib.pyplot as plt # plt.savefig('check_energy_binning_effects.png') # print('This is check_energy_binning_effects') # Try to see if the I/O causes a change in results. analysis.extraction.observations.write("temp123") analysis.extraction.observations.write("temp123", use_sherpa=True) obs = SpectrumObservation.read("temp123/pha_obs23523.fits") model = PowerLaw( amplitude=1.23 * 1e-11 * u.Unit("cm-2 s-1 TeV-1"), reference=1 * u.Unit("TeV"), index=2.14 * u.Unit(""), ) fit = SpectrumFit(obs_list=obs, model=model, fit_range=energy_range, stat="cash") fit.run() print("AFTER", fit.result[0])
def setup(self): paths = [ f"$GAMMAPY_DATA/cta-1dc/data/baseline/gps/gps_baseline_{obs_id:06d}.fits" for obs_id in [110380, 111140, 111630, 111159] ] self.data_store = DataStore.from_events_files(paths)
def observation(): """Example observation list for testing.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") obs_id = 23523 return datastore.obs(obs_id)
import numpy as np import matplotlib.pyplot as plt import astropy.units as u from astropy.coordinates import SkyCoord from astropy.convolution import Ring2DKernel, Tophat2DKernel from astropy.visualization import simple_norm from gammapy.data import DataStore from gammapy.image import SkyImage, SkyImageList from gammapy.detect import KernelBackgroundEstimator as KBE #choose the obsevartion name="PKS 2155-304" datastore = DataStore.from_dir("$HESS_DATA") src=SkyCoord.from_name(name) sep=SkyCoord.separation(src,datastore.obs_table.pointing_radec) srcruns=(datastore.obs_table[sep<2.0*u.deg]) obsid=srcruns['OBS_ID'].data #mylist=datastore.obs_list((obsid[0],)) mylist=datastore.obs_list(obsid[0:5]) obs_table=Table() obs_table['OBS_ID']=obsid[0:5] ref_image = SkyImage.empty( nxpix=400, nypix=400, binsz=0.02, xref=src.ra.deg, yref=src.dec.deg, coordsys='CEL', proj='TAN', )
def make_reco_model(): """Make a reco bg cube model.""" METHOD = 'default' data_dir = 'test_dataset' overwrite = OVERWRITE test = TEST group_id = GROUP_ID # create output folder outdir = os.path.join(OUTDIR, 'reco') _create_dir(outdir, overwrite) # 0. create dummy observation grouping obs_groups = create_dummy_observation_grouping() # save outfile = os.path.join(outdir, 'bg_observation_groups.ecsv') print('Writing {}'.format(outfile, overwrite=overwrite)) obs_groups.write(outfile) # 1. create dummy dataset # use enough stats so that rebinning (and resmoothing?) doesn't take place n_obs = 100 if test: # run fast n_obs = 2 az_range = AZ_RANGE alt_range = ALT_RANGE random_state = np.random.RandomState(seed=0) sigma = SIGMA spectral_index = INDEX make_test_dataset(outdir=data_dir, overwrite=overwrite, observatory_name='HESS', n_obs=n_obs, az_range=az_range, alt_range=alt_range, date_range=(Time('2010-01-01T00:00:00', format='isot', scale='utc'), Time('2015-01-01T00:00:00', format='isot', scale='utc')), n_tels_range=(3, 4), sigma=sigma, spectral_index=spectral_index, random_state=random_state) # 2. get observation table data_store = DataStore.from_dir(dir=data_dir) observation_table = data_store.make_observation_table() outfile = os.path.join(outdir, 'bg_observation_table_group{}.fits.gz'.format(group_id)) print("Writing {}".format(outfile)) observation_table.write(outfile, overwrite=overwrite) # 3. build bg model method = METHOD bg_cube_model = make_bg_cube_model(observation_table=observation_table, data_dir=data_dir, method=method, do_not_force_mev_units=True) # save outfile = os.path.join(outdir, 'bg_cube_model_group{}'.format(group_id)) print("Writing {}".format('{}_table.fits.gz'.format(outfile))) print("Writing {}".format('{}_image.fits.gz'.format(outfile))) bg_cube_model.write('{}_table.fits.gz'.format(outfile), format='table', clobber=overwrite) bg_cube_model.write('{}_image.fits.gz'.format(outfile), format='image', clobber=overwrite)
from pathlib import Path from gammapy.utils.nddata import sqrt_space from gammapy.data import DataStore from gammapy.irf import Background2D # ## Select off data # # We start by selecting the observations used to estimate the background model. # # In this case, we just take all "off runs" as defined in the observation table. # In[ ]: data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1") # Select just the off data runs obs_table = data_store.obs_table obs_table = obs_table[obs_table["TARGET_NAME"] == "Off data"] observations = data_store.get_observations(obs_table["OBS_ID"]) print("Number of observations:", len(observations)) # ## Background model # # The background model we will estimate is a differential background rate model in unit `s-1 MeV-1 sr-1` as a function of reconstructed energy and field of fiew offset. # # We estimate it by histogramming off data events and then smoothing a bit (not using a good method) to get a less noisy estimate. To get the differential rate, we divide by observation time and also take bin sizes into account to get the rate per energy and solid angle. So overall we fill two arrays called `counts` and `exposure` with `exposure` filled so that `background_rate = counts / exposure` will give the final background rate we're interested in. # # The processing can be done either one observation at a time, or first for counts and then for exposure. Either way is fine. Here we do one observation at a time, starting with empty histograms and then accumulating counts and exposure. Since this is a multi-step algorithm, we put the code to do this computation in a `BackgroundModelEstimator` class. #
# ## Define inputs # # We start by defining the inputs to use for the analysis: # # * which data and instrument response functions to use # * sky image geometry # * energy band # * maximum field of view offset cut # # The data (events, background models, effective area for exposure computation) consist of a few H.E.S.S. Crab observation runs as example. The background models there were produced as explained in the [background_model.ipynb](background_model.ipynb) tutorial. # In[3]: # What data to analyse data_store = DataStore.from_dir('$GAMMAPY_EXTRA/test_datasets/cube/data') # Define runlist obs_table = Table() obs_table['OBS_ID'] = [23523, 23526, 23592] # There's a problem with the PSF for run 23559, so we don't use that run for now. # In[4]: # Define sky image ref_image = SkyImage.empty( nxpix=300, nypix=300, binsz=0.02, xref=83.63, yref=22.01, proj='TAN', coordsys='CEL', )
def make_datasets_example(): # Define which data to use and print some information energy_axis = MapAxis.from_edges( np.logspace(-1.0, 1.0, 4), unit="TeV", name="energy", interp="log" ) geom0 = WcsGeom.create( skydir=(0, 0), binsz=0.1, width=(1, 1), frame="galactic", proj="CAR", axes=[energy_axis], ) geom1 = WcsGeom.create( skydir=(1, 0), binsz=0.1, width=(1, 1), frame="galactic", proj="CAR", axes=[energy_axis], ) geoms = [geom0, geom1] sources_coords = [(0, 0), (0.9, 0.1)] names = ["gc", "g09"] models = [] for idx, (lon, lat) in enumerate(sources_coords): spatial_model = PointSpatialModel( lon_0=lon * u.deg, lat_0=lat * u.deg, frame="galactic" ) spectral_model = ExpCutoffPowerLawSpectralModel( index=2 * u.Unit(""), amplitude=3e-12 * u.Unit("cm-2 s-1 TeV-1"), reference=1.0 * u.TeV, lambda_=0.1 / u.TeV, ) model_ecpl = SkyModel( spatial_model=spatial_model, spectral_model=spectral_model, name=names[idx] ) models.append(model_ecpl) # test to link a spectral parameter params0 = models[0].spectral_model.parameters params1 = models[1].spectral_model.parameters params0.link("reference", params1["reference"]) # update the sky model models[0].parameters.link("reference", params1["reference"]) obs_ids = [110380, 111140, 111159] data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps/") diffuse_model = SkyDiffuseCube.read( "$GAMMAPY_DATA/fermi_3fhl/gll_iem_v06_cutout.fits" ) datasets_list = [] for idx, geom in enumerate(geoms): observations = data_store.get_observations(obs_ids) stacked = MapDataset.create(geom=geom) stacked.background_model.name = "background_irf_" + names[idx] maker = MapDatasetMaker(offset_max=4.0 * u.deg) for obs in observations: dataset = maker.run(stacked, obs) stacked.stack(dataset) stacked.psf = stacked.psf.get_psf_kernel( position=geom.center_skydir, geom=geom, max_radius="0.3 deg" ) stacked.name = names[idx] stacked.models = models[idx] + diffuse_model datasets_list.append(stacked) datasets = Datasets(datasets_list) dataset0 = datasets[0] print("dataset0") print("counts sum : ", dataset0.counts.data.sum()) print("expo sum : ", dataset0.exposure.data.sum()) print("bkg0 sum : ", dataset0.background_model.evaluate().data.sum()) datasets.write("$GAMMAPY_DATA/tests/models", prefix="gc_example_", overwrite=True)
results["status"] = "failed" results["n_failed_bins"] = fail_count self.results["containment"] = results def check_all_table_psf(data_store): """Check all `gammapy.irf.PSF3D` for a given `gammapy.data.DataStore`. """ config = { "d_norm": 0.01, "containment_fraction": 0.68, "d_rel_containment": 0.7 } obs_ids = data_store.obs_table["OBS_ID"].data for obs_id in obs_ids[:10]: obs = data_store.obs(obs_id=obs_id) psf = obs.load(hdu_class="psf_table") checker = PSF3DChecker(psf=psf, **config) checker.check_all() print(checker.results) if __name__ == "__main__": import sys from gammapy.data import DataStore data_store = DataStore.from_dir(sys.argv[1]) check_all_table_psf(data_store)
source_name, center, obsdir) add_bkgmodel_to_indextable(bkg_model_directory, source_name, obsdir) # Make the images and psf model for different energy bands #Energy binning energy_reco_bins = EnergyBounds.equal_log_spacing( input_param["energy binning"]["Emin"], input_param["energy binning"]["Emax"], input_param["energy binning"]["nbin"], 'TeV') outdir = make_outdir(source_name, name_bkg, len(energy_reco_bins), config_name, image_size, for_integral_flux, use_cube, use_etrue) offset_band = Angle([0, 2.49], 'deg') data_store = DataStore.from_dir(obsdir) exclusion_mask = SkyMask.read(input_param["general"]["exclusion_mask"]) obs_table_subset = data_store.obs_table[0:nobs] # Cette observetion n est pas utilise par gammapy car cette observation est centre sur le Crab donc ne peut pas trouve de reflected region... if ((source_name == "Crab") | (source_name == "Crab_Test")): i_remove = np.where(obs_table_subset["OBS_ID"] == 18373) if len(i_remove[0]) != 0: obs_table_subset.remove_row(i_remove[0][0]) if use_cube: energy_reco = [ Energy(input_param["energy binning"]["Emin"], "TeV"), Energy(input_param["energy binning"]["Emax"], "TeV"), input_param["energy binning"]["nbin"] ] if use_etrue:
def make_reco_model(): """Make a reco bg cube model.""" METHOD = 'default' data_dir = 'test_dataset' overwrite = OVERWRITE test = TEST group_id = GROUP_ID # create output folder outdir = os.path.join(OUTDIR, 'reco') _create_dir(outdir, overwrite) # 0. create dummy observation grouping obs_groups = create_dummy_observation_grouping() # save outfile = os.path.join(outdir, 'bg_observation_groups.ecsv') print('Writing {}'.format(outfile, overwrite=overwrite)) obs_groups.write(outfile) # 1. create dummy dataset # use enough stats so that rebinning (and resmoothing?) doesn't take place n_obs = 100 if test: # run fast n_obs = 2 az_range = AZ_RANGE alt_range = ALT_RANGE random_state = np.random.RandomState(seed=0) sigma = SIGMA spectral_index = INDEX make_test_dataset(outdir=data_dir, overwrite=overwrite, observatory_name='HESS', n_obs=n_obs, az_range=az_range, alt_range=alt_range, date_range=(Time('2010-01-01T00:00:00', format='isot', scale='utc'), Time('2015-01-01T00:00:00', format='isot', scale='utc')), n_tels_range=(3, 4), sigma=sigma, spectral_index=spectral_index, random_state=random_state) # 2. get observation table data_store = DataStore.from_dir(dir=data_dir) observation_table = data_store.make_observation_table() outfile = os.path.join( outdir, 'bg_observation_table_group{}.fits.gz'.format(group_id)) print("Writing {}".format(outfile)) observation_table.write(outfile, overwrite=overwrite) # 3. build bg model method = METHOD bg_cube_model = make_bg_cube_model(observation_table=observation_table, data_dir=data_dir, method=method, do_not_force_mev_units=True) # save outfile = os.path.join(outdir, 'bg_cube_model_group{}'.format(group_id)) print("Writing {}".format('{}_table.fits.gz'.format(outfile))) print("Writing {}".format('{}_image.fits.gz'.format(outfile))) bg_cube_model.write('{}_table.fits.gz'.format(outfile), format='table', clobber=overwrite) bg_cube_model.write('{}_image.fits.gz'.format(outfile), format='image', clobber=overwrite)
from gammapy.cube.models import SkyModel, BackgroundModel from gammapy.spectrum.models import PowerLaw from gammapy.spectrum import create_crab_spectral_model from gammapy.image.models import SkyPointSource from gammapy.detect import compute_lima_on_off_image from gammapy.scripts import SpectrumAnalysisIACT from gammapy.utils.fitting import Fit from gammapy.background import RingBackgroundEstimator # ## Data access # # To access the data, we use the `DataStore`, and we use the ``obs_table`` to select the Crab runs. # In[ ]: data_store = DataStore.from_file( "$GAMMAPY_DATA/hess-dl3-dr1/hess-dl3-dr3-with-background.fits.gz") mask = data_store.obs_table["TARGET_NAME"] == "Crab" obs_table = data_store.obs_table[mask] observations = data_store.get_observations(obs_table["OBS_ID"]) # In[ ]: # pos_crab = SkyCoord.from_name('Crab') pos_crab = SkyCoord(83.633, 22.014, unit="deg") # ## Maps # # Let's make some 3D cubes, as well as 2D images. # # For the energy, we make 5 bins from 1 TeV to 10 TeV.
def data_store(): return DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/")
print("\n") print("CHECK PACKAGE VERSION ########") print('gammapy:', gammapy.__version__) print('numpy:', np.__version__) print('astropy:', astropy.__version__) print('regions:', regions.__version__) print('sherpa:', sherpa.__version__) print('uncertainties:', uncertainties.__version__) print('photutils:', photutils.__version__) print("##############################\n") ###metti all al posto di gps o cambia all in extragalactic survey data_store = DataStore.from_dir('/Users/lucatosti/Desktop/GEMINGA/1dc/index/gps') #### oggetto generale tipo database per coordinate #data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') data_store.info() show_it=True # in generale selezione osservazioni---->>> funziona anche direttamente SkyCoord.from_name('crab') --->>>metti geminga # from astropy.coordinates import SkyCoord # table = data_store.obs_table # pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg') # pos_target = SkyCoord(0, 0, frame='galactic', unit='deg') # offset = pos_target.separation(pos_obs).deg # mask = (1 < offset) & (offset < 2) # table = table[mask] # table.show_in_browser(jsviewer=True)
def observations_hess_dl3(): """HESS DL3 observation list.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") obs_ids = [23523, 23526] return datastore.get_observations(obs_ids)
# Configure the logger, so that the spectral analysis # isn't so chatty about what it's doing. import logging logging.basicConfig() log = logging.getLogger('gammapy.spectrum') log.setLevel(logging.ERROR) # ## Select observations # # Like explained in [cta_1dc_introduction.ipynb](cta_1dc_introduction.ipynb), a Gammapy analysis usually starts by creating a `DataStore` and selecting observations. # # This is shown in detail in the other notebook, here we just pick three observations near the galactic center. # In[5]: data_store = DataStore.from_dir('$CTADATA/index/gps') # In[6]: # Just as a reminder: this is how to select observations # from astropy.coordinates import SkyCoord # table = data_store.obs_table # pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg') # pos_target = SkyCoord(0, 0, frame='galactic', unit='deg') # offset = pos_target.separation(pos_obs).deg # mask = (1 < offset) & (offset < 2) # table = table[mask] # table.show_in_browser(jsviewer=True) # In[7]:
# # For 1DC there is one set of index files per simulated dataset, as well as a set of index files listing all available data in the all directory. # In[ ]: get_ipython().system('(cd $GAMMAPY_DATA/cta-1dc && tree index)') # ### Gammapy DataStore # # If you want to access data and IRFs from the CTA 1DC GPS dataset, just create a `DataStore` by pointing at a folder where the index files are located. # In[ ]: from gammapy.data import DataStore data_store = DataStore.from_dir("$GAMMAPY_DATA/cta-1dc/index/gps") # If you want to access all CTA DC-1 data, # set the CTADATA env var and use this: # data_store = DataStore.from_dir("$CTADATA/index/gps") # Or point at the directly with the index files directly # In[ ]: # Print out some basic information about the available data: data_store.info() # In[ ]: # The observation index is loaded as a table print("Number of observations: ", len(data_store.obs_table))
"""Add phase for two obs on Vela pulsar from CTA DC-1""" import numpy as np import astropy.units as u from astropy.time import Time from astropy.coordinates import SkyCoord from gammapy.data import DataStore ########################################################################### ####################### Extracting Vela files ########################### ########################################################################### # Load the data store (contains the information about all the DC1 data) # Assumes you have an environment variable CTADATA set pointing to the DC1 folder data_store = DataStore.from_dir('$CTADATA/index/gps') table = data_store.obs_table # Defining the offset as the angular separation between the observation position and the target position pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg') pos_target = SkyCoord.from_name('vela') offset = pos_target.separation(pos_obs) # Defining a mask to select all runs targeting Vela with an offset < 2 deg mask = (offset < 2 * u.deg) # Applying the mask table = table[mask] # Getting the list of observation indices of Vela obs_ids = table['OBS_ID']
"""Example how to compute and plot reflected regions.""" import numpy as np import astropy.units as u from astropy.coordinates import SkyCoord from regions import RectangleSkyRegion import matplotlib.pyplot as plt from gammapy.data import DataStore from gammapy.datasets import SpectrumDataset from gammapy.makers import ReflectedRegionsBackgroundMaker, SpectrumDatasetMaker from gammapy.maps import Map, MapAxis from gammapy.visualization import plot_spectrum_datasets_off_regions data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") mask = data_store.obs_table["TARGET_NAME"] == "Crab" obs_ids = data_store.obs_table["OBS_ID"][mask].data observations = data_store.get_observations(obs_ids) crab_position = SkyCoord(83.63, 22.01, unit="deg", frame="icrs") # The ON region center is defined in the icrs frame. The angle is defined w.r.t. to its axis. rectangle = RectangleSkyRegion(center=crab_position, width=0.5 * u.deg, height=0.4 * u.deg, angle=0 * u.deg) bkg_maker = ReflectedRegionsBackgroundMaker(min_distance=0.1 * u.rad) dataset_maker = SpectrumDatasetMaker(selection=["counts"]) e_reco = MapAxis.from_energy_bounds(0.1, 100, 30, unit="TeV") dataset_empty = SpectrumDataset.create(e_reco=e_reco, region=rectangle)
def observations(): data_store = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1/") run_list = [23523, 23526] return Observations([data_store.obs(_) for _ in run_list])
log.basicConfig(level=log.INFO) import numpy as np from astropy.wcs import WCS from astropy.units import Quantity from astropy.io import fits from astropy.coordinates import SkyCoord from gammapy.data import DataStore from gammapy.cube import exposure_cube, SpectralCube from gammapy.utils.energy import EnergyBounds dirname = '$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2' log.info('Reading data from {}'.format(dirname)) data_store = DataStore.from_dir(dirname) events = data_store.load(obs_id=23523, filetype='events') log.info('Number of events in event list: {}'.format(len(events))) log.info('Max. event energy: {}'.format(events['ENERGY'].max())) log.info('Min. event energy: {}'.format(events['ENERGY'].min())) aeff = data_store.load(obs_id=23523, filetype='aeff') counts = SpectralCube.empty(emin=0.5, emax=80, enbins=8, eunit='TeV', nxpix=200, nypix=200, xref=events.meta['RA_OBJ'], yref=events.meta['DEC_OBJ'], dtype='int', coordsys='CEL') log.info('Bin events into cube.') counts.fill(events) log.info('Counts cube shape: {}'.format(counts.data.shape))
from gammapy.image import SkyImage from gammapy.spectrum import models from regions import CircleSkyRegion from astropy.coordinates import SkyCoord import astropy.units as u import logging logging.basicConfig() log = logging.getLogger('gammapy.spectrum') log.setLevel(logging.ERROR) print("\nlog created") obs_cols = ['OBS_ID', 'GLON_PNT', 'GLAT_PNT'] #store_dir = '/Users/lucatosti/gammapy-extra/datasets/hess-crab4-hd-hap-prod2' store_dir = '/Users/lucatosti/Desktop/GEMINGA/1dc/index/all' data_store = DataStore.from_dir(store_dir) data_store.info() table = data_store.obs_table pos_obs = SkyCoord(table['GLON_PNT'], table['GLAT_PNT'], frame='galactic', unit='deg') crab_pos = SkyCoord.from_name('crab', frame='galactic') #crab_pos = SkyCoord( 0 , 0 ,unit='deg', frame='galactic')#184.5575 , -05.7844 offset = crab_pos.separation(pos_obs).deg #print(offset) #if(offset.any()<10): # print("qualcuno") mask = (offset < 0.5) #(1 < offset) & (offset < 2)
""" from astropy.coordinates import SkyCoord, Angle from gammapy.datasets import gammapy_extra from gammapy.image import ExclusionMask from gammapy.data import DataStore from gammapy.region import SkyCircleRegion from gammapy.spectrum import SpectrumAnalysis from gammapy.utils.energy import EnergyBounds center = SkyCoord(83.63, 22.01, unit='deg', frame='icrs') radius = Angle('0.3 deg') on_region = SkyCircleRegion(pos=center, radius=radius) bkg_method = dict(type='reflected', n_min=3) exclusion_file = gammapy_extra.filename("datasets/exclusion_masks/" "tevcat_exclusion.fits") excl = ExclusionMask.from_fits(exclusion_file) bounds = EnergyBounds.equal_log_spacing(1, 10, 40, unit='TeV') store = gammapy_extra.filename("datasets/hess-crab4") ds = DataStore.from_dir(store) obs = [23523, 23559] ana = SpectrumAnalysis(datastore=ds, obs=obs, on_region=on_region, bkg_method=bkg_method, exclusion=excl, ebounds=bounds) ana.write_ogip_data(outdir='ogip_data')
import matplotlib.pyplot as plt import numpy as np import astropy import regions import sherpa import astropy.units as u from astropy.coordinates import SkyCoord, Angle from astropy.table import Table from regions import CircleSkyRegion from gammapy.data import DataStore from gammapy.image import SkyImage, IACTBasicImageEstimator, SkyImageList from gammapy.background import FoVBackgroundEstimator from importlib import reload datastore = DataStore.from_dir("/Users/asinha/HESS_newbkg") name = "LMC" src = SkyCoord.from_name(name) myid = np.loadtxt("LMC_id.txt") mylist = datastore.obs_list(myid) ref_image = SkyImage.empty( nxpix=400, nypix=400, binsz=0.05, xref=src.galactic.l.value, yref=src.galactic.b.value, coordsys='GAL', proj='TAN', ) src_names = ["LMC N132D", "30 Dor C", "LHA 120-N 157B", "LMC P3"]
"""Example how to load and take a peek at an IACT observation. """ import matplotlib.pyplot as plt from gammapy.data import DataStore from gammapy.utils.mpl_style import gammapy_mpl_style # TODO: Update once this issue is resolved: # https://github.com/astropy/astropy/issues/4140 # plt.style.use(gammapy_mpl_style) plt.rcParams.update(gammapy_mpl_style) fig, axes = plt.subplots(2, 4, figsize=(20, 8)) axes = axes.flat data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') obs = data_store.obs(obs_id=23523) obs.events.peek() obs.aeff.plot_energy_dependence(ax=next(axes)) obs.aeff.plot_offset_dependence(ax=next(axes)) aeff = obs.aeff.to_effective_area_table(offset='1 deg') # import IPython; IPython.embed() aeff.plot(ax=next(axes)) obs.edisp.plot_bias(ax=next(axes)) obs.edisp.plot_migration(ax=next(axes)) edisp = obs.edisp.to_energy_dispersion(offset='1 deg') edisp.plot_matrix(ax=next(axes))
def observations(): """Example observation list for testing.""" datastore = DataStore.from_dir("$GAMMAPY_DATA/hess-dl3-dr1") obs_ids = [23523, 23526] return datastore.get_observations(obs_ids)
# In[3]: # We only need to import the `DataStore`, # all other data objects can be loaded via the data store. from gammapy.data import DataStore import astropy.units as u # ## Data store # # First, we need to select some observations for our spectral analysis. To this end we use the [data management](http://docs.gammapy.org/dev/data/dm.html) functionality in gammapy. The following example uses a simulated crab dataset in [gammapy-extra](https://github.com/gammapy/gammapy-extra). Ideally, we'd use crabs runs from the H.E.S.S. public data release, so if you have the released files just change the ``DATA_DIR`` variable to the corresponding folder. # In[4]: # data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') data_store = DataStore.from_dir('$GAMMAPY_EXTRA/test_datasets/cta_1dc') # In[5]: data_store.info() # In[6]: print(data_store.hdu_table.colnames) data_store.hdu_table[:7] # ## Observation selection # # Select observations using the observation table # In[7]:
import matplotlib.pyplot as plt from gammapy.data import DataStore ds = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2') events = ds.obs(obs_id=23523).events events.plot_time_map() plt.show()