def test_OGIP_response_arf_rsp_accessors():
# Then load rsp and arf in XSpec
rsp_file = get_path_of_data_file("ogip_test_xmm_pn.rmf")
arf_file = get_path_of_data_file("ogip_test_xmm_pn.arf")
rsp = OGIPResponse(rsp_file, arf_file=arf_file)
assert rsp.arf_filename == arf_file
assert rsp.rsp_filename == rsp_file
def test_response_set_constructor():
[rsp_aw,
rsp_bw], exposure_getter, counts_getter = get_matrix_set_elements()
with pytest.raises(RuntimeError):
# This should raise because there is no time information for the matrices
_ = InstrumentResponseSet([rsp_aw, rsp_bw], exposure_getter,
counts_getter)
# Add the time information
(
[rsp_a, rsp_b],
exposure_getter,
counts_getter,
) = get_matrix_set_elements_with_coverage()
# This should work now
rsp_set = InstrumentResponseSet([rsp_a, rsp_b], exposure_getter,
counts_getter)
assert rsp_set[0] == rsp_a
assert rsp_set[1] == rsp_b
# Check that the constructor order the matrices by time when needed
# This should work now
rsp_set = InstrumentResponseSet([rsp_b, rsp_a], exposure_getter,
counts_getter)
assert rsp_set[0] == rsp_a
assert rsp_set[1] == rsp_b
# Now test construction from the .from_rsp2 method
rsp2_file = get_path_of_data_file("ogip_test_gbm_b0.rsp2")
with warnings.catch_warnings():
warnings.simplefilter("error", np.VisibleDeprecationWarning)
rsp_set = InstrumentResponseSet.from_rsp2_file(rsp2_file,
exposure_getter,
counts_getter)
assert len(rsp_set) == 3
# Now test that we cannot initialize a response set with matrices which have non-contiguous coverage intervals
matrix, mc_energies, ebounds = get_matrix_elements()
rsp_c = InstrumentResponse(matrix, ebounds, mc_energies,
TimeInterval(0.0, 10.0))
rsp_d = InstrumentResponse(matrix, ebounds, mc_energies,
TimeInterval(20.0, 30.0))
with pytest.raises(RuntimeError):
_ = InstrumentResponseSet([rsp_c, rsp_d], exposure_getter,
counts_getter)
def _load_styles():
# Discover defined styles
styles = _discover_styles()
# Load them
defined_styles = {}
for style_file in styles:
this_style = PlotStyle.from_style_file(style_file)
# The name of the style is just the file name without the .yml extension
style_name = os.path.splitext(os.path.basename(style_file))[0]
defined_styles[style_name] = this_style
# Now load the default style
default_style_filename = get_path_of_data_file("default_style.yml")
defined_styles["default"] = PlotStyle.from_style_file(
default_style_filename)
return defined_styles
def test_OGIP_response_first_channel():
# Get path of response file
rsp_file = get_path_of_data_file("ogip_test_gbm_n6.rsp")
rsp = OGIPResponse(rsp_file)
assert rsp.first_channel == 1
def test_response_write_to_fits3():
# Now do the same for a file with a ARF
rsp_file = get_path_of_data_file("ogip_test_xmm_pn.rmf")
arf_file = get_path_of_data_file("ogip_test_xmm_pn.arf")
rsp = OGIPResponse(rsp_file, arf_file=arf_file)
temp_file = "__test.rsp"
rsp.to_fits(temp_file, "TEST", "TEST", overwrite=True)
rsp_reloaded = OGIPResponse(temp_file)
assert np.allclose(rsp_reloaded.matrix, rsp.matrix)
assert np.allclose(rsp_reloaded.ebounds, rsp.ebounds)
assert np.allclose(rsp_reloaded.monte_carlo_energies,
rsp.monte_carlo_energies)
os.remove(temp_file)
def get_basic_config(evfile, scfile, ra, dec, emin=100.0, emax=100000.0, zmax=100.0, evclass=128, evtype=3,
filter='DATA_QUAL>0 && LAT_CONFIG==1'):
from fermipy.config import ConfigManager
# Get default config from fermipy
basic_config = ConfigManager.load(get_path_of_data_file("fermipy_basic_config.yml")) # type: dict
evfile = sanitize_filename(evfile)
scfile = sanitize_filename(scfile)
assert os.path.exists(evfile), "The provided evfile %s does not exist" % evfile
assert os.path.exists(scfile), "The provided scfile %s does not exist" % scfile
basic_config['data']['evfile'] = evfile
basic_config['data']['scfile'] = scfile
ra = float(ra)
dec = float(dec)
assert 0 <= ra <= 360, "The provided R.A. (%s) is not valid. Should be 0 <= ra <= 360.0" % ra
assert -90 <= dec <= 90, "The provided Dec (%s) is not valid. Should be -90 <= dec <= 90.0" % dec
basic_config['selection']['ra'] = ra
basic_config['selection']['dec'] = dec
emin = float(emin)
emax = float(emax)
basic_config['selection']['emin'] = emin
basic_config['selection']['emax'] = emax
zmax = float(zmax)
assert 0.0 <= zmax <= 180.0, "The provided Zenith angle cut (zmax = %s) is not valid. " \
"Should be 0 <= zmax <= 180.0" % zmax
basic_config['selection']['zmax'] = zmax
evclass = int(evclass)
assert is_power_of_2(evclass), "The provided evclass is not a power of 2."
basic_config['selection']['evclass'] = evclass
evtype = int(evtype)
basic_config['selection']['evtype'] = evtype
basic_config['selection']['filter'] = filter
return DictWithPrettyPrint(basic_config)
def test_response_write_to_fits2():
# Now do the same for a response read from a file
rsp_file = get_path_of_data_file("ogip_test_gbm_n6.rsp")
rsp = OGIPResponse(rsp_file)
temp_file = "__test.rsp"
rsp.to_fits(temp_file, "TEST", "TEST", overwrite=True)
rsp_reloaded = OGIPResponse(temp_file)
assert np.allclose(rsp_reloaded.matrix, rsp.matrix)
assert np.allclose(rsp_reloaded.ebounds, rsp.ebounds)
assert np.allclose(rsp_reloaded.monte_carlo_energies,
rsp.monte_carlo_energies)
os.remove(temp_file)
def test_dispersionspectrumlike_fit():
response = OGIPResponse(
get_path_of_data_file("datasets/ogip_powerlaw.rsp"))
sim_K = 1e-1
sim_kT = 20.0
# get a blackbody source function
source_function = Blackbody(K=sim_K, kT=sim_kT)
# power law background function
background_function = Powerlaw(K=1, index=-1.5, piv=100.0)
spectrum_generator = DispersionSpectrumLike.from_function(
"test",
source_function=source_function,
response=response,
background_function=background_function,
)
bb = Blackbody()
pts = PointSource("mysource", 0, 0, spectral_shape=bb)
model = Model(pts)
# MLE fitting
jl = JointLikelihood(model, DataList(spectrum_generator))
result = jl.fit()
K_variates = jl.results.get_variates("mysource.spectrum.main.Blackbody.K")
kT_variates = jl.results.get_variates(
"mysource.spectrum.main.Blackbody.kT")
assert np.all(
np.isclose([K_variates.average, kT_variates.average], [sim_K, sim_kT],
atol=1))
def test_all():
response = OGIPResponse(
get_path_of_data_file("datasets/ogip_powerlaw.rsp"))
np.random.seed(1234)
# rescale the functions for the response
source_function = Blackbody(K=1e-7, kT=500.0)
background_function = Powerlaw(K=1, index=-1.5, piv=1.0e3)
spectrum_generator = DispersionSpectrumLike.from_function(
"fake",
source_function=source_function,
background_function=background_function,
response=response)
source_function.K.prior = Log_normal(mu=np.log(1e-7), sigma=1)
source_function.kT.prior = Log_normal(mu=np.log(300), sigma=2)
ps = PointSource("demo", 0, 0, spectral_shape=source_function)
model = Model(ps)
ba = BayesianAnalysis(model, DataList(spectrum_generator))
ba.set_sampler()
ba.sample(quiet=True)
ppc = compute_ppc(ba,
ba.results,
n_sims=500,
file_name="my_ppc.h5",
overwrite=True,
return_ppc=True)
ppc.fake.plot(bkg_subtract=True)
ppc.fake.plot(bkg_subtract=False)
def test_dispersionspectrumlike_fit():
response = OGIPResponse(get_path_of_data_file('datasets/ogip_powerlaw.rsp'))
sim_K = 1E-1
sim_kT = 20.
# get a blackbody source function
source_function = Blackbody(K=sim_K, kT=sim_kT)
# power law background function
background_function = Powerlaw(K=1, index=-1.5, piv=100.)
spectrum_generator = DispersionSpectrumLike.from_function('test', source_function=source_function,
response=response,
background_function=background_function)
bb = Blackbody()
pts = PointSource('mysource', 0, 0, spectral_shape=bb)
model = Model(pts)
# MLE fitting
jl = JointLikelihood(model, DataList(spectrum_generator))
result = jl.fit()
K_variates = jl.results.get_variates('mysource.spectrum.main.Blackbody.K')
kT_variates = jl.results.get_variates('mysource.spectrum.main.Blackbody.kT')
assert np.all(np.isclose([K_variates.mean(), kT_variates.mean()], [sim_K, sim_kT], atol=1))
def _load_styles():
# Discover defined styles
styles = _discover_styles()
# Load them
defined_styles = {}
for style_file in styles:
this_style = PlotStyle.from_style_file(style_file)
# The name of the style is just the file name without the .yml extension
style_name = os.path.splitext(os.path.basename(style_file))[0]
defined_styles[style_name] = this_style
# Now load the default style
default_style_filename = get_path_of_data_file("default_style.yml")
defined_styles['default'] = PlotStyle.from_style_file(default_style_filename)
return defined_styles
def get_basic_config(
evfile,
scfile,
ra,
dec,
emin=100.0,
emax=100000.0,
zmax=100.0,
evclass=128,
evtype=3,
filter="DATA_QUAL>0 && LAT_CONFIG==1",
fermipy_verbosity=2,
fermitools_chatter=2,
):
from fermipy.config import ConfigManager
# Get default config from fermipy
basic_config = ConfigManager.load(
get_path_of_data_file("fermipy_basic_config.yml")) # type: dict
evfile = str(sanitize_filename(evfile))
scfile = str(sanitize_filename(scfile))
if not os.path.exists(evfile):
log.critical("The provided evfile %s does not exist" % evfile)
if not os.path.exists(scfile):
log.critical("The provided scfile %s does not exist" % scfile)
basic_config["data"]["evfile"] = evfile
basic_config["data"]["scfile"] = scfile
ra = float(ra)
dec = float(dec)
if not ((0 <= ra) and (ra <= 360)):
log.critical(
"The provided R.A. (%s) is not valid. Should be 0 <= ra <= 360.0"
% ra)
if not ((-90 <= dec) and (dec <= 90)):
log.critical(
"The provided Dec (%s) is not valid. Should be -90 <= dec <= 90.0"
% dec)
basic_config["selection"]["ra"] = ra
basic_config["selection"]["dec"] = dec
emin = float(emin)
emax = float(emax)
basic_config["selection"]["emin"] = emin
basic_config["selection"]["emax"] = emax
zmax = float(zmax)
if not ((0.0 <= zmax) and (zmax <= 180.0)):
log.critical(
"The provided Zenith angle cut (zmax = %s) is not valid. "
"Should be 0 <= zmax <= 180.0" % zmax)
basic_config["selection"]["zmax"] = zmax
with fits.open(scfile) as ft2_:
tmin = float(ft2_[0].header["TSTART"])
tmax = float(ft2_[0].header["TSTOP"])
basic_config["selection"]["tmin"] = tmin
basic_config["selection"]["tmax"] = tmax
evclass = int(evclass)
if not is_power_of_2(evclass):
log.critical("The provided evclass is not a power of 2.")
basic_config["selection"]["evclass"] = evclass
evtype = int(evtype)
basic_config["selection"]["evtype"] = evtype
basic_config["selection"]["filter"] = filter
basic_config["logging"]["verbosity"] = fermipy_verbosity
#(In fermipy convention, 0 = critical only, 1 also errors, 2 also warnings, 3 also info, 4 also debug)
basic_config["logging"][
"chatter"] = fermitools_chatter #0 = no screen output. 2 = some output, 4 = lot of output.
return DictWithPrettyPrint(basic_config)
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats as stats
from astromodels import clone_model
from threeML.classicMLE.joint_likelihood import JointLikelihood
from threeML.classicMLE.joint_likelihood_set import JointLikelihoodSet
from threeML.data_list import DataList
from threeML.io.logging import setup_logger
from threeML.io.package_data import get_path_of_data_file
from threeML.plugins.OGIPLike import OGIPLike
from threeML.utils.OGIP.pha import PHAWrite
plt.style.use(str(get_path_of_data_file("threeml.mplstyle")))
log = setup_logger(__name__)
class LikelihoodRatioTest(object):
def __init__(self, joint_likelihood_instance0: JointLikelihood,
joint_likelihood_instance1: JointLikelihood) -> None:
self._joint_likelihood_instance0: JointLikelihood = (
joint_likelihood_instance0) # type: JointLikelihood
self._joint_likelihood_instance1: JointLikelihood = (
joint_likelihood_instance1) # type: JointLikelihood
# Restore best fit and store the reference value for the likelihood
self._joint_likelihood_instance0.restore_best_fit()
def get_threeML_style() -> str:
return str(get_path_of_data_file(_submenu.mplstyle))
def __init__(self):
# Read first the default configuration file
default_configuration_path = get_path_of_data_file(_config_file_name)
assert os.path.exists(default_configuration_path), \
"Default configuration %s does not exist. Re-install 3ML" % default_configuration_path
with open(default_configuration_path) as f:
try:
configuration = yaml.load(f, Loader=yaml.SafeLoader)
except:
raise ConfigurationFileCorrupt("Default configuration file %s cannot be parsed!" %
(default_configuration_path))
# This needs to be here for the _check_configuration to work
self._default_configuration_raw = configuration
# Test the default configuration
try:
self._check_configuration(configuration, default_configuration_path)
except:
raise
else:
self._default_path = default_configuration_path
# Check if the user has a user-supplied config file under .threeML
user_config_path = os.path.join(get_path_of_user_dir(), _config_file_name)
if os.path.exists(user_config_path):
with open(user_config_path) as f:
configuration = yaml.load(f, Loader=yaml.SafeLoader)
# Test if the local/configuration is ok
try:
self._configuration = self._check_configuration(configuration, user_config_path)
except ConfigurationFileCorrupt:
# Probably an old configuration file
custom_warnings.warn("The user configuration file at %s does not appear to be valid. We will "
"substitute it with the default configuration. You will find a copy of the "
"old configuration at %s so you can transfer any customization you might "
"have from there to the new configuration file. We will use the default "
"configuration for this session."
%(user_config_path, "%s.bak" % user_config_path))
# Move the config file to a backup file
shutil.copy2(user_config_path, "%s.bak" % user_config_path)
# Remove old file
os.remove(user_config_path)
# Copy the default configuration
shutil.copy2(self._default_path, user_config_path)
self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
self._filename = self._default_path
else:
self._filename = user_config_path
print("Configuration read from %s" % (user_config_path))
else:
custom_warnings.warn("Using default configuration from %s. "
"You might want to copy it to %s to customize it and avoid this warning."
% (self._default_path, user_config_path))
self._configuration = self._check_configuration(self._default_configuration_raw, self._default_path)
self._filename = self._default_path
def set_threeML_style() -> None:
plt.style.use(str(get_path_of_data_file(_submenu.mplstyle)))
def test_OGIP_response_against_xspec():
# Test for various photon indexes
for index in [-0.5, 0.0, 0.5, 1.5, 2.0, 3.0, 4.0]:
print("Processing index %s" % index)
# First reset xspec
xspec.AllData.clear()
# Create a model in XSpec
mo = xspec.Model("po")
# Change the default value for the photon index
# (remember that in XSpec the definition of the powerlaw is norm * E^(-PhoIndex),
# so PhoIndex is positive normally. This is the opposite of astromodels.
mo.powerlaw.PhoIndex = index
mo.powerlaw.norm = 12.2
# Now repeat the same in 3ML
# Generate the astromodels function and set it to the same values as the XSpec power law
# (the pivot in XSpec is set to 1). Remember also that the definition in xspec has the
# sign of the photon index opposite
powerlaw = Powerlaw()
powerlaw.piv = 1.0
powerlaw.index = -mo.powerlaw.PhoIndex.values[0]
powerlaw.K = mo.powerlaw.norm.values[0]
# Exploit the fact that the power law integral is analytic
powerlaw_integral = Powerlaw()
# Remove transformation
powerlaw_integral.K._transformation = None
powerlaw_integral.K.bounds = (None, None)
powerlaw_integral.index = powerlaw.index.value + 1
powerlaw_integral.K = old_div(powerlaw.K.value, (powerlaw.index.value + 1))
powerlaw_integral.display()
integral_function = lambda e1, e2: powerlaw_integral(e2) - powerlaw_integral(e1)
# Now check that the two convoluted model give the same number of counts in each channel
# Fake a spectrum so we can actually compute the convoluted model
# Get path of response file
rsp_file = str(get_path_of_data_file("ogip_test_gbm_n6.rsp"))
fs1 = xspec.FakeitSettings(
rsp_file, exposure=1.0, fileName="_fake_spectrum.pha"
)
xspec.AllData.fakeit(noWrite=True, applyStats=False, settings=fs1)
# Get the expected counts
xspec_counts = mo.folded(1)
# Now get the convolution from 3ML
rsp = OGIPResponse(rsp_file)
rsp.set_function(integral_function)
threeML_counts = rsp.convolve()
# Compare them
assert np.allclose(xspec_counts, threeML_counts)
# Now do the same with a matrix with a ARF
# First reset xspec
xspec.AllData.clear()
# Then load rsp and arf in XSpec
rsp_file = str(get_path_of_data_file("ogip_test_xmm_pn.rmf"))
arf_file = str(get_path_of_data_file("ogip_test_xmm_pn.arf"))
fs1 = xspec.FakeitSettings(
rsp_file, arf_file, exposure=1.0, fileName="_fake_spectrum.pha"
)
xspec.AllData.fakeit(noWrite=True, applyStats=False, settings=fs1)
# Get the expected counts
xspec_counts = mo.folded(1)
# Now get the convolution from 3ML
rsp = OGIPResponse(rsp_file, arf_file=arf_file)
rsp.set_function(integral_function)
threeML_counts = rsp.convolve()
# Compare them
assert np.allclose(xspec_counts, threeML_counts)
def test_OGIP_response_against_xspec():
# Test for various photon indexes
for index in [-0.5, 0.0, 0.5, 1.5, 2.0, 3.0, 4.0]:
print("Processing index %s" % index)
# First reset xspec
xspec.AllData.clear()
# Create a model in XSpec
mo = xspec.Model("po")
# Change the default value for the photon index
# (remember that in XSpec the definition of the powerlaw is norm * E^(-PhoIndex),
# so PhoIndex is positive normally. This is the opposite of astromodels.
mo.powerlaw.PhoIndex = index
mo.powerlaw.norm = 12.2
# Now repeat the same in 3ML
# Generate the astromodels function and set it to the same values as the XSpec power law
# (the pivot in XSpec is set to 1). Remember also that the definition in xspec has the
# sign of the photon index opposite
powerlaw = Powerlaw()
powerlaw.piv = 1.0
powerlaw.index = -mo.powerlaw.PhoIndex.values[0]
powerlaw.K = mo.powerlaw.norm.values[0]
# Exploit the fact that the power law integral is analytic
powerlaw_integral = Powerlaw()
# Remove transformation
powerlaw_integral.K._transformation = None
powerlaw_integral.K.bounds = (None, None)
powerlaw_integral.index = powerlaw.index.value + 1
powerlaw_integral.K = powerlaw.K.value / (powerlaw.index.value + 1)
powerlaw_integral.display()
integral_function = lambda e1, e2: powerlaw_integral(e2) - powerlaw_integral(e1)
# Now check that the two convoluted model give the same number of counts in each channel
# Fake a spectrum so we can actually compute the convoluted model
# Get path of response file
rsp_file = get_path_of_data_file("ogip_test_gbm_n6.rsp")
fs1 = xspec.FakeitSettings(rsp_file, exposure=1.0, fileName="_fake_spectrum.pha")
xspec.AllData.fakeit(noWrite=True, applyStats=False, settings=fs1)
# Get the expected counts
xspec_counts = mo.folded(1)
# Now get the convolution from 3ML
rsp = OGIPResponse(rsp_file)
rsp.set_function(integral_function)
threeML_counts = rsp.convolve()
# Compare them
assert np.allclose(xspec_counts, threeML_counts)
# Now do the same with a matrix with a ARF
# First reset xspec
xspec.AllData.clear()
# Then load rsp and arf in XSpec
rsp_file = get_path_of_data_file("ogip_test_xmm_pn.rmf")
arf_file = get_path_of_data_file("ogip_test_xmm_pn.arf")
fs1 = xspec.FakeitSettings(rsp_file, arf_file, exposure=1.0, fileName="_fake_spectrum.pha")
xspec.AllData.fakeit(noWrite=True, applyStats=False, settings=fs1)
# Get the expected counts
xspec_counts = mo.folded(1)
# Now get the convolution from 3ML
rsp = OGIPResponse(rsp_file, arf_file=arf_file)
rsp.set_function(integral_function)
threeML_counts = rsp.convolve()
# Compare them
assert np.allclose(xspec_counts, threeML_counts)
def test_get_package_data():
# Try and get the config file
config_file = get_path_of_data_file("threeML_config.yml")
assert os.path.exists(config_file)
