def test_datasets_to_io(tmp_path): path = "$GAMMAPY_DATA/tests/models" filedata = "gc_example_datasets.yaml" filemodel = "gc_example_models.yaml" datasets = Datasets.read(path, filedata, filemodel) assert len(datasets) == 2 print(list(datasets.models)) assert len(datasets.models) == 5 dataset0 = datasets[0] assert dataset0.name == "gc" assert dataset0.counts.data.sum() == 22258 assert_allclose(dataset0.exposure.data.sum(), 8.057342e+12, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 15726.8, atol=0.1) assert dataset0.background_model.name == "gc-bkg" dataset1 = datasets[1] assert dataset1.name == "g09" assert dataset1.background_model.name == "g09-bkg" assert (dataset0.models["gll_iem_v06_cutout"] == dataset1.models["gll_iem_v06_cutout"]) assert isinstance(dataset0.models, Models) assert len(dataset0.models) == 4 assert dataset0.models[0].name == "gc" assert dataset0.models[1].name == "gll_iem_v06_cutout" assert dataset0.models[2].name == "gc-bkg" assert (dataset0.models["gc"].parameters["reference"] is dataset1.models["g09"].parameters["reference"]) assert_allclose(dataset1.models["g09"].parameters["lon_0"].value, 0.9, atol=0.1) datasets.write(tmp_path, prefix="written") datasets_read = Datasets.read(tmp_path, "written_datasets.yaml", "written_models.yaml") assert len(datasets.parameters) == 22 assert len(datasets_read) == 2 dataset0 = datasets_read[0] assert dataset0.counts.data.sum() == 22258 assert_allclose(dataset0.exposure.data.sum(), 8.057342e+12, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 15726.8, atol=0.1) dataset_copy = dataset0.copy(name="dataset0-copy") assert dataset_copy.background_model.datasets_names == ["dataset0-copy"]
def test_datasets_to_io(tmp_path): filedata = "$GAMMAPY_DATA/tests/models/gc_example_datasets.yaml" filemodel = "$GAMMAPY_DATA/tests/models/gc_example_models.yaml" datasets = Datasets.read(filedata, filemodel) assert len(datasets) == 2 dataset0 = datasets[0] assert dataset0.name == "gc" assert dataset0.counts.data.sum() == 6824 assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1) assert dataset0.background_model.name == "background_irf_gc" dataset1 = datasets[1] assert dataset1.name == "g09" assert dataset1.background_model.name == "background_irf_g09" assert (dataset0.models["gll_iem_v06_cutout"] == dataset1.models["gll_iem_v06_cutout"]) assert isinstance(dataset0.models, Models) assert len(dataset0.models) == 5 assert dataset0.models[0].name == "gc" assert dataset0.models[1].name == "gll_iem_v06_cutout" assert (dataset0.models["gc"].parameters["reference"] is dataset1.models["g09"].parameters["reference"]) assert_allclose(dataset1.models["g09"].parameters["lon_0"].value, 0.9, atol=0.1) datasets.write(tmp_path, prefix="written") datasets_read = Datasets.read(tmp_path / "written_datasets.yaml", tmp_path / "written_models.yaml") assert len(datasets.parameters) == 22 assert len(datasets_read) == 2 dataset0 = datasets_read[0] assert dataset0.counts.data.sum() == 6824 assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1)
def fermi_datasets(): fermi_datasets = Datasets.read( "$GAMMAPY_DATA/fermi-3fhl-crab", "Fermi-LAT-3FHL_datasets.yaml", "Fermi-LAT-3FHL_models.yaml", ) return fermi_datasets
def read_regions(self): for kr in self.ROIs_sel: filedata = self.resdir / f"3FHL_ROI_num{kr}_datasets.yaml" filemodel = self.resdir / f"3FHL_ROI_num{kr}_models.yaml" try: dataset = list(Datasets.read(filedata, filemodel, lazy=False))[0] except (FileNotFoundError, IOError): continue infos = np.load(self.resdir / f"3FHL_ROI_num{kr}_fit_infos.npz") self.diags["message"].append(infos["message"]) self.diags["stat"].append(infos["stat"]) if self.savefig: self.plot_maps(dataset) for model in dataset.models: if (isinstance(model, FoVBackgroundModel) is False and self.FHL3[model.name].data["ROI_num"] == kr and self.FHL3[model.name].data["Signif_Avg"] >= self.sig_cut): res_spec = model.spectral_model cat_spec = self.FHL3[model.name].spectral_model() res_fp = FluxPoints.read(self.resdir / f"{model.name}_flux_points.fits", reference_model=cat_spec) cat_fp = self.FHL3[model.name].flux_points self.update_spec_diags(dataset, model, cat_spec, res_spec, cat_fp, res_fp) if self.savefig: self.plot_spec(kr, model, cat_spec, res_spec, cat_fp, res_fp)
def test_flux_point_dataset_serialization(tmp_path): path = "$GAMMAPY_DATA/tests/spectrum/flux_points/diff_flux_points.fits" data = FluxPoints.read(path) data.table["e_ref"] = data.energy_ref.to("TeV") spectral_model = PowerLawSpectralModel( index=2.3, amplitude="2e-13 cm-2 s-1 TeV-1", reference="1 TeV" ) model = SkyModel(spectral_model=spectral_model, name="test_model") dataset = FluxPointsDataset(model, data, name="test_dataset") Datasets([dataset]).write( filename=tmp_path / "tmp_datasets.yaml", filename_models=tmp_path / "tmp_models.yaml", ) datasets = Datasets.read( filename=tmp_path / "tmp_datasets.yaml", filename_models=tmp_path / "tmp_models.yaml", ) new_dataset = datasets[0] assert_allclose(new_dataset.data.dnde, dataset.data.dnde, 1e-4) if dataset.mask_fit is None: assert np.all(new_dataset.mask_fit == dataset.mask_safe) assert np.all(new_dataset.mask_safe == dataset.mask_safe) assert new_dataset.name == "test_dataset"
def test_flux_point_dataset_serialization(tmp_path): path = "$GAMMAPY_DATA/tests/spectrum/flux_points/diff_flux_points.fits" table = Table.read(make_path(path)) table["e_ref"] = table["e_ref"].quantity.to("TeV") data = FluxPoints.from_table(table, format="gadf-sed") spectral_model = PowerLawSpectralModel(index=2.3, amplitude="2e-13 cm-2 s-1 TeV-1", reference="1 TeV") model = SkyModel(spectral_model=spectral_model, name="test_model") dataset = FluxPointsDataset(model, data, name="test_dataset") dataset2 = FluxPointsDataset.read(path, name="test_dataset2") assert_allclose(dataset.data.dnde.data, dataset2.data.dnde.data) assert dataset.mask_safe.data == dataset2.mask_safe.data assert dataset2.name == "test_dataset2" Datasets([dataset]).write( filename=tmp_path / "tmp_datasets.yaml", filename_models=tmp_path / "tmp_models.yaml", ) datasets = Datasets.read( filename=tmp_path / "tmp_datasets.yaml", filename_models=tmp_path / "tmp_models.yaml", ) new_dataset = datasets[0] assert_allclose(new_dataset.data.dnde, dataset.data.dnde, 1e-4) if dataset.mask_fit is None: assert np.all(new_dataset.mask_fit == dataset.mask_safe) assert np.all(new_dataset.mask_safe == dataset.mask_safe) assert new_dataset.name == "test_dataset"
def input_dataset(): datasets = Datasets.read( filedata="$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_datasets.yaml", filemodel="$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_models.yaml", ) dataset = datasets[0] dataset.psf = None return dataset
def test_spectrum_dataset_on_off_to_yaml(tmpdir): spectrum_datasets_on_off = make_observation_list() datasets = Datasets(spectrum_datasets_on_off) datasets.write(path=tmpdir) datasets_read = Datasets.read(tmpdir / "_datasets.yaml", tmpdir / "_models.yaml") assert len(datasets_read) == len(datasets) assert datasets_read[0].name == datasets[0].name assert datasets_read[1].name == datasets[1].name assert datasets_read[1].counts.data.sum() == datasets[1].counts.data.sum()
def input_dataset(): filename = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_datasets.yaml" filename_models = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_models.yaml" datasets = Datasets.read(filename=filename, filename_models=filename_models) dataset = datasets[0] dataset.psf = None return dataset
def read_datasets_and_set_model(instrument, model): # Read from disk datasets = Datasets.read(f"reduced_{instrument}", f"_datasets.yaml", "_models.yaml") e_min = u.Quantity(instrument_opts[instrument]["emin"]) e_max = u.Quantity(instrument_opts[instrument]["emax"]) # Set model and fit range for ds in datasets: ds.models = model ds.mask_fit = ds.counts.geom.energy_mask(e_min, e_max) return datasets
def read_datasets(self): """Read datasets from YAML file. File names are taken from the configuration file. """ filename = self.config.general.datasets_file filename_models = self.config.general.models_file if filename is not None: self.datasets = Datasets.read(filename) log.info(f"Datasets loaded from {filename}.") if filename_models is not None: self.read_models(filename_models, extend=False) else: raise RuntimeError("Missing datasets_file in config.general")
def dataset(): dataset = Datasets.read( "$GAMMAPY_DATA/fermi-3fhl-crab/", "Fermi-LAT-3FHL_datasets.yaml", "Fermi-LAT-3FHL_models.yaml", ) # Define the free parameters and min, max values parameters = dataset.models.parameters parameters["lon_0"].frozen = False parameters["lat_0"].frozen = False parameters["norm"].frozen = True parameters["alpha"].frozen = True parameters["beta"].frozen = True parameters["lat_0"].min = -90 parameters["lat_0"].max = 90 parameters["lon_0"].min = 0 parameters["lon_0"].max = 360 parameters["amplitude"].min = 0.01 * parameters["amplitude"].value parameters["amplitude"].max = 100 * parameters["amplitude"].value return dataset
def dataset(): filename_models = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_models.yaml" models = Models.read(filename_models) # Define the free parameters and min, max values parameters = models.parameters parameters["lon_0"].frozen = False parameters["lat_0"].frozen = False parameters["norm"].frozen = True parameters["alpha"].frozen = True parameters["beta"].frozen = True parameters["lat_0"].min = -90 parameters["lat_0"].max = 90 parameters["lon_0"].min = 0 parameters["lon_0"].max = 360 parameters["amplitude"].min = 0.01 * parameters["amplitude"].value parameters["amplitude"].max = 100 * parameters["amplitude"].value filename = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_datasets.yaml" datasets = Datasets.read(filename=filename) datasets.models = models return datasets
def fermi_datasets(): filename = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_datasets.yaml" filename_models = "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_models.yaml" return Datasets.read(filename=filename, filename_models=filename_models)
def read(filename): return Datasets.read(os.getcwd(), f"{filename}_datasets.yaml", f"{filename}_models.yaml")
def test_models_management(tmp_path): path = "$GAMMAPY_DATA/tests/models" filedata = "gc_example_datasets.yaml" filemodel = "gc_example_models.yaml" datasets = Datasets.read(path, filedata, filemodel) model1 = datasets.models[0].copy(name="model1", datasets_names=None) model2 = datasets.models[0].copy(name="model2", datasets_names=[datasets[1].name]) model3 = datasets.models[0].copy(name="model3", datasets_names=[datasets[0].name]) model1b = datasets.models[0].copy(name="model1", datasets_names=None) model1b.spectral_model.amplitude.value *= 2 names0 = datasets[0].models.names names1 = datasets[1].models.names datasets[0].models.append(model1) _ = datasets[0].models + model2 assert datasets[0].models.names == names0 + ["model1", "model2"] assert datasets[0].models["model1"].datasets_names == None assert datasets[0].models["model2"].datasets_names == [ datasets[1].name, datasets[0].name, ] assert datasets[1].models.names == names1 + ["model1", "model2"] # TODO consistency check at datasets level ? # or force same Models for each dataset._models on datasets init ? # here we have the right behavior: model1 and model2 are also added to dataset1 # because serialization create a global model object shared by all datasets # if that was not the case we could have inconsistancies # such as model1.datasets_names == None added only to dataset1 # user can still create such inconsistancies if they define datasets # with diferent Models objects for each dataset. del datasets[0].models["model1"] assert datasets[0].models.names == names0 + ["model2"] datasets[0].models.remove(model2) assert datasets[0].models.names == names0 datasets.models.append(model2) assert model2 in datasets.models assert model2 in datasets[1].models assert datasets[0].models.names == names0 + ["model2"] datasets[0].models.extend([model1, model3]) assert datasets[0].models.names == names0 + ["model2", "model1", "model3"] for m in [model1, model2, model3]: datasets.models.remove(m) assert datasets[0].models.names == names0 assert datasets[1].models.names == names1 datasets.models.extend([model1, model2, model3]) assert datasets[0].models.names == names0 + ["model1", "model2", "model3"] assert datasets[1].models.names == names1 + ["model1", "model2"] for m in [model1, model2, model3]: datasets.models.remove(m) _ = datasets.models + [model1, model2] assert datasets[0].models.names == names0 + ["model1", "model2"] assert datasets[1].models.names == names1 + ["model1", "model2"] datasets[0].models["model2"] = model3 assert datasets[0].models.names == names0 + ["model1", "model3"] assert datasets[1].models.names == names1 + ["model1"] datasets.models.remove(model1) datasets[0].models = model1 _ = datasets.models # auto-update models assert datasets[0].models.names == ["model1", "gll_iem_v06_cutout"] # the consistency check added diffuse model contained in the global model npred1 = datasets[0].npred().data.sum() datasets.models.remove(model1) npred0 = datasets[0].npred().data.sum() datasets.models.append(model1b) npred1b = datasets[0].npred().data.sum() assert npred1b != npred1 assert npred1b != npred0 assert_allclose(npred1b, 2147.407023024028) datasets.models.remove(model1b) _ = datasets.models # auto-update models newmodels = [datasets.models[0].copy() for k in range(48)] datasets.models.extend(newmodels) datasets[0].use_cache = False nocache = datasets[0].npred().data.sum() datasets[0].use_cache = True assert_allclose(datasets[0].npred().data.sum(), nocache)
def read(filename): path = Path.cwd() return Datasets.read(path / f"{filename}_datasets.yaml", filename_models=path / f"{filename}_models.yaml")
os.makedirs(outdir) # loop through the sources: for src in config.keys(): if src == 'global': continue if args.select_source is not None: if not src == args.select_source: continue logging.info(f" ====== {src} ======= ") logging.info("Loading IACT datasets...") # read 3d data set dataset_3d = Datasets.read(config['global']['iact_dataset_3d'].replace( "*", src)) geom = dataset_3d[0].geoms['geom'] on_region = CircleSkyRegion(center=geom.center_skydir, radius=on_radius) # Load the 1D data set dataset_1d_file = config['global']['iact_dataset_1d'].replace("*", src) if os.path.exists(dataset_1d_file.replace(".yaml", "_stacked.yaml")): logging.info("Loading stacked dataset...") dataset_stack = Datasets.read( dataset_1d_file.replace(".yaml", "_stacked.yaml")) else: dataset_1d = Datasets.read( config['global']['iact_dataset_1d'].replace("*", src)) # stack reduce the data set logging.info("stacking datasets...")
def crab_datasets_fermi(): return Datasets.read( "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_datasets.yaml", "$GAMMAPY_DATA/fermi-3fhl-crab/Fermi-LAT-3FHL_models.yaml")
def test_datasets_to_io(tmp_path): path = "$GAMMAPY_DATA/tests/models" filedata = "gc_example_datasets.yaml" filemodel = "gc_example_models.yaml" datasets = Datasets.read(path, filedata, filemodel) assert len(datasets) == 2 print(list(datasets.models)) assert len(datasets.models) == 5 dataset0 = datasets[0] assert dataset0.name == "gc" assert dataset0.counts.data.sum() == 6824 assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1) assert dataset0.background_model.name == "background_irf_gc" dataset1 = datasets[1] assert dataset1.name == "g09" assert dataset1.background_model.name == "background_irf_g09" assert (dataset0.models["gll_iem_v06_cutout"] == dataset1.models["gll_iem_v06_cutout"]) assert isinstance(dataset0.models, Models) assert len(dataset0.models) == 3 assert dataset0.models[0].name == "gc" assert dataset0.models[1].name == "gll_iem_v06_cutout" assert dataset0.models[2].name == "background_irf_gc" assert (dataset0.models["background_irf_gc"].parameters["norm"] is dataset1.models["background_irf_g09"].parameters["norm"]) assert (dataset0.models["gc"].parameters["reference"] is dataset1.models["g09"].parameters["reference"]) assert_allclose(dataset1.models["g09"].parameters["lon_0"].value, 0.9, atol=0.1) datasets.write(tmp_path, prefix="written") datasets_read = Datasets.read(tmp_path, "written_datasets.yaml", "written_models.yaml") assert len(datasets.parameters) == 21 assert len(datasets_read) == 2 dataset0 = datasets_read[0] assert dataset0.counts.data.sum() == 6824 assert_allclose(dataset0.exposure.data.sum(), 2072125400000.0, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.background_model.evaluate().data.sum(), 4094.2, atol=0.1) Fit(datasets).run() assert_allclose( datasets.models["background_irf_g09"].covariance, datasets.models["background_irf_gc"].covariance, ) dataset_copy = dataset0.copy(name="dataset0-copy") assert dataset_copy.background_model.datasets_names == ["dataset0-copy"]
def test_datasets_to_io(tmp_path): filedata = "$GAMMAPY_DATA/tests/models/gc_example_datasets.yaml" filemodel = "$GAMMAPY_DATA/tests/models/gc_example_models.yaml" datasets = Datasets.read( filename=filedata, filename_models=filemodel, ) assert len(datasets) == 2 assert len(datasets.models) == 5 dataset0 = datasets[0] assert dataset0.name == "gc" assert dataset0.counts.data.sum() == 22258 assert_allclose(dataset0.exposure.data.sum(), 8.057342e12, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.npred_background().data.sum(), 15726.8, atol=0.1) assert dataset0.background_model.name == "gc-bkg" dataset1 = datasets[1] assert dataset1.name == "g09" assert dataset1.background_model.name == "g09-bkg" assert (dataset0.models["gll_iem_v06_cutout"] == dataset1.models["gll_iem_v06_cutout"]) assert isinstance(dataset0.models, DatasetModels) assert len(dataset0.models) == 4 assert dataset0.models[0].name == "gc" assert dataset0.models[1].name == "gll_iem_v06_cutout" assert dataset0.models[2].name == "gc-bkg" assert (dataset0.models["gc"].parameters["reference"] is dataset1.models["g09"].parameters["reference"]) assert_allclose(dataset1.models["g09"].parameters["lon_0"].value, 0.9, atol=0.1) datasets.write( filename=tmp_path / "written_datasets.yaml", filename_models=tmp_path / "written_models.yaml", ) datasets_read = Datasets.read( filename=tmp_path / "written_datasets.yaml", filename_models=tmp_path / "written_models.yaml", ) assert len(datasets.parameters) == 22 assert len(datasets_read) == 2 dataset0 = datasets_read[0] assert dataset0.counts.data.sum() == 22258 assert_allclose(dataset0.exposure.data.sum(), 8.057342e12, atol=0.1) assert dataset0.psf is not None assert dataset0.edisp is not None assert_allclose(dataset0.npred_background().data.sum(), 15726.8, atol=0.1) dataset_copy = dataset0.copy(name="dataset0-copy") assert dataset_copy.models is None