def __init__(self,
maptree,
response,
model,
verbose=False,
process="annihilation",
energy_bins=True):
self.maptree = maptree
self.response = response
self.model = model
self.verbose = verbose
self.DM_process = process
# - construct the log likelihood
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
# these are the bins from the Energy Estimator, Spectrum Fitting Memo
if energy_bins:
self.hawc.set_active_measurements(bin_list=[
"1c", "1d", "1e", "2c", "2d", "2e", "2f", "3c", "3d", "3e",
"3f", "4c", "4d", "4e", "4f", "4g", "5d", "5e", "5f", "5g",
"5h", "6e", "6f", "6g", "6h", "7f", "7g", "7h", "7i", "8f",
"8g", "8h", "8i", "8j", "9g", "9h", "9i", "9j", "9k", "9l"
])
else:
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
self.jl = JointLikelihood(self.model, self.datalist, verbose=True)
def deepcopy_hal(extended=False):
src_ra, src_dec = 82.628, 22.640
src_name = 'test_source'
roi = HealpixConeROI(data_radius=5.,
model_radius=8.,
ra=src_ra,
dec=src_dec)
hawc = HAL('HAWC', maptree(), response(), roi)
hawc.set_active_measurements(1, 9)
data = DataList(hawc)
# Define model
spectrum = Log_parabola()
if not extended:
source = PointSource(src_name,
ra=src_ra,
dec=src_dec,
spectral_shape=spectrum)
else:
shape = astromodels.Gaussian_on_sphere()
source = ExtendedSource(src_name,
spatial_shape=shape,
spectral_shape=spectrum)
model = Model(source)
jl = JointLikelihood(model, data, verbose=False)
hawc_copy = copy.deepcopy(hawc)
def __init__(self, maptree, response, models, verbose=False):
self.maptree = maptree
self.response = response
self.verbose = verbose
self.models = models
self.fluxUnit = (1. / (u.TeV * u.cm**2 * u.s))
# - construct HAL
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
# - construct joint likelihood
self.jl_M87 = JointLikelihood(self.models[0],
self.datalist,
verbose=True)
self.jl_M49 = JointLikelihood(self.models[1],
self.datalist,
verbose=True)
self.jl_linked = JointLikelihood(self.models[2],
self.datalist,
verbose=False)
self.jl_notlinked = JointLikelihood(self.models[3],
self.datalist,
verbose=True)
# - set the minimizer
self.jl_M87.set_minimizer("minuit")
self.jl_M49.set_minimizer("minuit")
self.jl_linked.set_minimizer("minuit")
self.jl_notlinked.set_minimizer("minuit")
def fit_point_source(roi, maptree, response, point_source_model, liff=False):
data_radius = roi.data_radius.to("deg").value
if not liff:
# This is a 3ML plugin
hawc = HAL("HAWC", maptree, response, roi)
else:
from threeML import HAWCLike
hawc = HAWCLike("HAWC", maptree, response, fullsky=True)
ra_roi, dec_roi = roi.ra_dec_center
hawc.set_ROI(ra_roi, dec_roi, data_radius)
hawc.set_active_measurements(1, 9)
if not liff: hawc.display()
data = DataList(hawc)
jl = JointLikelihood(point_source_model, data, verbose=False)
point_source_model.display(complete=True)
beg = time.time()
# jl.set_minimizer("ROOT")
param_df, like_df = jl.fit()
# _ = jl.get_errors()
print("Fit time: %s" % (time.time() - beg))
return param_df, like_df
def test_fit(roi, maptree, response):
pts_model = point_source_model()
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Get the likelihood value for the saturated model
hawc.get_saturated_model_likelihood()
data = DataList(hawc)
jl = JointLikelihood(pts_model, data, verbose=True)
param_df, like_df = jl.fit()
return jl, hawc, pts_model, param_df, like_df, data
def test_complete_analysis(roi, maptree, response, point_source_model):
# Define the ROI
# Instance the plugin
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Get the likelihood value for the saturated model
hawc.get_saturated_model_likelihood()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("hal_src_stacked_image.png")
data = DataList(hawc)
jl = JointLikelihood(point_source_model, data, verbose=False)
param_df, like_df = jl.fit()
# Generate a simulated dataset and write it to disk
sim = hawc.get_simulated_dataset("HAWCsim")
sim.write("sim_resp.hd5", "sim_maptree.hd5")
# See the model in counts space and the residuals
fig = hawc.display_spectrum()
# Save it to file
fig.savefig("hal_src_residuals.png")
# Look at the different energy planes (the columns are model, data, residuals)
fig = hawc.display_fit(smoothing_kernel_sigma=0.3)
fig.savefig("hal_src_fit_planes.png")
# Compute TS
src_name = point_source_model.pts.name
jl.compute_TS(src_name, like_df)
# Compute goodness of fit with Monte Carlo
gf = GoodnessOfFit(jl)
gof, param, likes = gf.by_mc(10)
print(
"Prob. of obtaining -log(like) >= observed by chance if null hypothesis is true: %.2f"
% gof['HAWC'])
# it is a good idea to inspect the results of the simulations with some plots
# Histogram of likelihood values
fig, sub = plt.subplots()
likes.hist(ax=sub)
# Overplot a vertical dashed line on the observed value
plt.axvline(jl.results.get_statistic_frame().loc['HAWC',
'-log(likelihood)'],
color='black',
linestyle='--')
fig.savefig("hal_sim_all_likes.png")
# Plot the value of beta for all simulations (for example)
fig, sub = plt.subplots()
param.loc[(slice(None), ['pts.spectrum.main.Cutoff_powerlaw.index']),
'value'].plot()
fig.savefig("hal_sim_all_index.png")
# Free the position of the source
point_source_model.pts.position.ra.free = True
point_source_model.pts.position.dec.free = True
# Set boundaries (no need to go further than this)
ra = point_source_model.pts.position.ra.value
dec = point_source_model.pts.position.dec.value
point_source_model.pts.position.ra.bounds = (ra - 0.5, ra + 0.5)
point_source_model.pts.position.dec.bounds = (dec - 0.5, dec + 0.5)
# Fit with position free
param_df, like_df = jl.fit()
# Make localization contour
# pts.position.ra(8.362 + / - 0.00028) x
# 10
# deg
# pts.position.dec(2.214 + / - 0.00025) x
# 10
a, b, cc, fig = jl.get_contours(point_source_model.pts.position.dec, 22.13,
22.1525, 10,
point_source_model.pts.position.ra, 83.615,
83.635, 10)
plt.plot([ra], [dec], 'x')
fig.savefig("hal_src_localization.png")
# Of course we can also do a Bayesian analysis the usual way
# NOTE: here the position is still free, so we are going to obtain marginals about that
# as well
# For this quick example, let's use a uniform prior for all parameters
for parameter in point_source_model.parameters.values():
if parameter.fix:
continue
if parameter.is_normalization:
parameter.set_uninformative_prior(Log_uniform_prior)
else:
parameter.set_uninformative_prior(Uniform_prior)
# Let's execute our bayes analysis
bs = BayesianAnalysis(point_source_model, data)
samples = bs.sample(30, 20, 20)
fig = bs.corner_plot()
fig.savefig("hal_corner_plot.png")
def go(args):
spectrum = Powerlaw()
shape = Continuous_injection_diffusion_legacy()
source = ExtendedSource("Geminga",
spatial_shape=shape,
spectral_shape=spectrum)
fluxUnit = 1. / (u.TeV * u.cm**2 * u.s)
# Set spectral parameters
spectrum.K = 1e-14 * fluxUnit
spectrum.K.bounds = (1e-16 * fluxUnit, 1e-12 * fluxUnit)
spectrum.piv = 20 * u.TeV
spectrum.piv.fix = True
spectrum.index = -2.4
spectrum.index.bounds = (-4., -1.)
# Set spatial parameters
shape.lon0 = args.RA * u.degree
shape.lon0.fix = True
shape.lat0 = args.Dec * u.degree
shape.lat0.fix = True
shape.rdiff0 = 6.0 * u.degree
shape.rdiff0.fix = False
shape.rdiff0.max_value = 12.
shape.delta.min_value = 0.1
shape.delta = args.delta
shape.delta.fix = True
shape.uratio = args.uratio
shape.uratio.fix = True
shape.piv = 2e10
shape.piv.fix = True
shape.piv2 = 1e9
shape.piv2.fix = True
spectrum2 = Powerlaw()
shape2 = Continuous_injection_diffusion_legacy()
source2 = ExtendedSource("B0656",
spatial_shape=shape2,
spectral_shape=spectrum2)
fluxUnit = 1. / (u.TeV * u.cm**2 * u.s)
# Set spectral parameters for the 2nd source
spectrum2.K = 1e-14 * fluxUnit
spectrum2.K.bounds = (1e-16 * fluxUnit, 1e-12 * fluxUnit)
spectrum2.piv = 20 * u.TeV
spectrum2.piv.fix = True
spectrum2.index = -2.2
# spectrum2.index.fix = True
spectrum2.index.bounds = (-4., -1.)
# Set spatial parameters for the 2nd source
shape2.lon0 = 104.95 * u.degree
shape2.lon0.fix = True
shape2.lat0 = 14.24 * u.degree
shape2.lat0.fix = True
shape2.rdiff0 = 6.0 * u.degree
shape2.rdiff0.fix = False
shape2.rdiff0.max_value = 12.
shape2.delta.min_value = 0.2
shape2.delta = args.delta
shape2.delta.fix = True
shape2.uratio = args.uratio
shape2.uratio.fix = True
shape2.piv = 2e10
shape2.piv.fix = True
shape2.piv2 = 1e9
shape2.piv2.fix = True
# Set up a likelihood model using the source
if args.ROI == 0:
lm = Model(source, source2)
elif args.ROI == 1 or args.ROI == 2:
lm = Model(source)
elif args.ROI == 3 or args.ROI == 4:
lm = Model(source2)
ra_c, dec_c, rad = (None, None, None)
if args.ROI == 0:
ra_c, dec_c, rad = 101.7, 16, 9.
# llh.set_ROI(101.7, 16, 9., True)
elif args.ROI == 1:
ra_c, dec_c, rad = 98.5, 17.76, 4.5
# llh.set_ROI(98.5, 17.76, 4.5, True)
elif args.ROI == 2:
ra_c, dec_c, rad = 97, 18.5, 6
# llh.set_ROI(97, 18.5, 6, True)
elif args.ROI == 3:
ra_c, dec_c, rad = 104.95, 14.24, 3.
# llh.set_ROI(104.95, 14.24, 3., True)
elif args.ROI == 4:
ra_c, dec_c, rad = 107, 13, 5.4
# llh.set_ROI(107, 13, 5.4, True)
if args.plugin == 'old':
llh = HAWCLike("Geminga", args.mtfile, args.rsfile, fullsky=True)
llh.set_active_measurements(args.startBin, args.stopBin)
llh.set_ROI(ra_c, dec_c, rad, True)
else:
roi = HealpixConeROI(data_radius=rad,
model_radius=rad + 10.0,
ra=ra_c,
dec=dec_c)
llh = HAL("HAWC", args.mtfile, args.rsfile, roi)
llh.set_active_measurements(args.startBin, args.stopBin)
print(lm)
# we fit a common diffusion coefficient so parameters are linked
if "B0656" in lm and "Geminga" in lm:
law = Line(a=250. / 288., b=0.)
lm.link(lm.B0656.spatial_shape.rdiff0, lm.Geminga.spatial_shape.rdiff0,
law)
lm.B0656.spatial_shape.rdiff0.Line.a.fix = True
lm.B0656.spatial_shape.rdiff0.Line.b.fix = True
# Double check the free parameters
print("Likelihood model:\n")
print(lm)
# Set up the likelihood and run the fit
TS = 0.
try:
lm.Geminga.spatial_shape.rdiff0 = 5.5
lm.Geminga.spatial_shape.rdiff0.fix = False
lm.Geminga.spectrum.main.Powerlaw.K = 1.36e-23
lm.Geminga.spectrum.main.Powerlaw.index = -2.34
except:
pass
try:
lm.B0656.spectrum.main.Powerlaw.K = 5.7e-24
lm.B0656.spectrum.main.Powerlaw.index = -2.14
except:
pass
print("Performing likelihood fit...\n")
datalist = DataList(llh)
jl = JointLikelihood(lm, datalist, verbose=True)
try:
jl.set_minimizer("minuit")
param, like_df = jl.fit(compute_covariance=False)
except AttributeError:
jl.set_minimizer("minuit")
param, like_df = jl.fit(compute_covariance=False)
# Print the TS, significance, and fit parameters, and then plot stuff
print("\nTest statistic:")
if args.plugin == 'old':
TS = llh.calc_TS()
else:
TS_df = jl.compute_TS("Geminga", like_df)
TS = TS_df.loc['HAWC', 'TS']
print("Test statistic: %g" % TS)
freepars = []
fixedpars = []
for p in lm.parameters:
par = lm.parameters[p]
if par.free:
freepars.append("%-45s %35.6g %s" % (p, par.value, par._unit))
else:
fixedpars.append("%-45s %35.6g %s" % (p, par.value, par._unit))
# Output TS, significance, and parameter values to a file
with open(args.output, "w") as f:
f.write("Test statistic: %g\n" % TS)
f.write("\nFree parameters:\n")
for l in freepars:
f.write("%s\n" % l)
f.write("\nFixed parameters:\n")
for l in fixedpars:
f.write("%s\n" % l)
return param, like_df, TS
def test_plugin_from_hd5(maptree, response, roi):
roi = HealpixConeROI(ra=82.628, dec=22.640, data_radius=5, model_radius=10)
hal = HAL("HAL", maptree, response, roi)
def test_plugin_from_root(geminga_maptree, geminga_response, geminga_roi):
hal = HAL("HAL", geminga_maptree, geminga_response, geminga_roi)
def main():
ra, dec = 83.630, 22.020 #2HWC catalog position of Crab Nebula
maptree = '../data/HAWC_9bin_507days_crab_data.hd5'
response = '../data/HAWC_9bin_507days_crab_response.hd5'
veritasdata = '../data/threemlVEGAS20hr2p45_run54809_run57993.root'
latdirectory = '../data/lat_crab_data' # will put downloaded Fermi data there
data_radius = 3.0
model_radius = 8.0
#set up HAWC dataset
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra,
dec=dec)
hawc = HAL("HAWC", maptree, response, roi)
hawc.set_active_measurements(
1, 9) # Perform the fist only within the last nine bins
hawc_data = {
"name": "HAWC",
"data": [hawc],
"Emin": 1e3 * u.GeV,
"Emax": 37e3 * u.GeV,
"E0": 7 * u.TeV
}
#VERTIAS plugin
with np.errstate(divide='ignore', invalid='ignore'):
# This VERITASLike spits a lot of numpy errors. Silent them, I hope that's OK...
# Udara told me that's normal.
veritas = VERITASLike('veritas', veritasdata)
veritas_data = {
"name": "VERITAS",
"data": [veritas],
"Emin": 160 * u.GeV,
"Emax": 30e3 * u.GeV,
"E0": 1.0 * u.TeV
}
# Fermi via Fermipy
tstart = '2017-01-01 00:00:00'
tstop = '2017-03-01 00:00:00'
evfile, scfile = threeML.download_LAT_data(
ra,
dec,
10.0,
tstart,
tstop,
time_type='Gregorian',
destination_directory=latdirectory)
config = threeML.FermipyLike.get_basic_config(evfile=evfile,
scfile=scfile,
ra=ra,
dec=dec)
config['selection']['emax'] = 300000.0 #MeV = 300 GeV
config['selection']['emin'] = 100.0 #MeV = 0.1 GeV
config['gtlike'] = {'edisp': False}
fermi_lat = threeML.FermipyLike("LAT", config)
lat_data = {
"name": "Fermi_LAT",
"data": [fermi_lat],
"Emin": 0.1 * u.GeV,
"Emax": 300 * u.GeV,
"E0": 10 * u.GeV
}
# Made up "Fermi-LAT" flux points
# Not used for now, these are just an example for how to set up XYLike data
# XYLike points are amsumed in base units of 3ML: keV, and keV s-1 cm-2 (bug: even if you provide something else...).
x = [1.38e6, 2.57e6, 4.46e6, 7.76e6, 18.19e6, 58.88e6] # keV
y = [5.92e-14, 1.81e-14, 6.39e-15, 1.62e-15, 2.41e-16,
1.87e-17] # keV s-1 cm-2
yerr = [1.77e-15, 5.45e-16, 8.93e-17, 4.86e-17, 5.24e-18,
7.28e-19] # keV s-1 cm-2
# Just save a copy for later use (plot points). Will redefine similar objects with other "source_name"
xy_test = threeML.XYLike("xy_test",
x,
y,
yerr,
poisson_data=False,
quiet=False,
source_name='XY_Test')
joint_data = {
"name": "Fermi_VERITAS_HAWC",
"data": [fermi_lat, veritas, hawc],
"Emin": 0.1 * u.GeV,
"Emax": 37e3 * u.GeV,
"E0": 1 * u.TeV
}
datasets = [veritas_data, hawc_data, lat_data, joint_data]
fig, ax = plt.subplots()
#Loop through datasets and do the fit.
for dataset in datasets:
data = threeML.DataList(*dataset["data"])
spectrum = threeML.Log_parabola()
source = threeML.PointSource(dataset["name"],
ra=ra,
dec=dec,
spectral_shape=spectrum)
model = threeML.Model(source)
spectrum.alpha.bounds = (-4.0, -1.0)
spectrum.value = -2.653
spectrum.piv.value = dataset["E0"]
spectrum.K.value = 3.15e-22 #if not giving units, will be interpreted as (keV cm^2 s)^-1
spectrum.K.bounds = (1e-50, 1e10)
spectrum.beta.value = 0.15
spectrum.beta.bounds = (-1.0, 1.0)
model.display()
jl = threeML.JointLikelihood(model, data, verbose=True)
jl.set_minimizer("ROOT")
with np.errstate(divide='ignore', invalid='ignore'):
# This VERITASLike spits a lot of numpy errors. Silent them, I hope that's OK...
# Udara told me that's normal.
best_fit_parameters, likelihood_values = jl.fit()
err = jl.get_errors()
jl.results.write_to("likelihoodresults_{0}.fits".format(
dataset["name"]),
overwrite=True)
#plot spectra
color = next(ax._get_lines.prop_cycler)['color']
try:
# Using a fixed version of model_plot.py
threeML.plot_spectra(
jl.results,
ene_min=dataset["Emin"],
ene_max=dataset["Emax"],
energy_unit='GeV',
flux_unit='erg/(s cm2)',
subplot=ax,
fit_colors=color,
contour_colors=color,
)
except:
# Using a bugged version of model_plot.py
print(
'Warning: fallback without colors... Use a fixed version of model_plot.py! (3ML PR #304)'
)
threeML.plot_point_source_spectra(
jl.results,
ene_min=dataset["Emin"],
ene_max=dataset["Emax"],
energy_unit='GeV',
flux_unit='erg/(s cm2)',
subplot=ax,
)
#workaround to get rit of gammapy temporary file
find_and_delete("ccube.fits", ".")
plt.xlim(5e-2, 50e3)
plt.xlabel("Energy [GeV]")
plt.ylabel(r"$E^2$ dN/dE [erg/$cm^2$/s]")
plt.savefig("joint_spectrum.png")
class LimitCalculator:
def __init__(self, maptree, response, models, verbose=False):
self.maptree = maptree
self.response = response
self.verbose = verbose
self.models = models
self.fluxUnit = (1. / (u.TeV * u.cm**2 * u.s))
# - construct HAL
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
# - construct joint likelihood
self.jl_M87 = JointLikelihood(self.models[0],
self.datalist,
verbose=True)
self.jl_M49 = JointLikelihood(self.models[1],
self.datalist,
verbose=True)
self.jl_linked = JointLikelihood(self.models[2],
self.datalist,
verbose=False)
self.jl_notlinked = JointLikelihood(self.models[3],
self.datalist,
verbose=True)
# - set the minimizer
self.jl_M87.set_minimizer("minuit")
self.jl_M49.set_minimizer("minuit")
self.jl_linked.set_minimizer("minuit")
self.jl_notlinked.set_minimizer("minuit")
def set_range(self, minimum, maximum):
self.min = minimum
self.max = maximum
def get_M87_likelihood(self):
return self.jl_M87
def get_M49_likelihood(self):
return self.jl_M49
def get_linked_likelihood(self):
return self.jl_linked
def calculate_limit(self, ll):
num_steps = 100
bar = progressbar.ProgressBar(maxval=num_steps,
widgets=[
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
])
TS = []
result = ll.fit()
bar.start()
bar.update(0)
best_fit = ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value
vals = np.logspace(
np.log10(
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value) -
.25,
np.log10(
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value) +
3.0, num_steps)
for ival, val in enumerate(vals):
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value = val
curr_LL = ll.data_list.values()[0].inner_fit()
#print(val, curr_LL)
TS.append(curr_LL)
bar.update(ival)
TS = np.array(TS)
TS = -TS
TS -= np.min(TS)
plt.semilogx(vals, TS)
plt.savefig("example_step1.pdf")
plt.show()
interpolator = interp1d(TS, vals)
return (ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K,
interpolator(1.35))
def test_complete_analysis(maptree=os.path.join(test_data_path,
"maptree_1024.root"),
response=os.path.join(test_data_path,
"response.root")):
# Define the ROI
ra_mkn421, dec_mkn421 = 166.113808, 38.208833
data_radius = 3.0
model_radius = 8.0
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra_mkn421,
dec=dec_mkn421)
# Instance the plugin
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("hal_mkn421_stacked_image.png")
# Define model as usual
spectrum = Log_parabola()
source = PointSource("mkn421",
ra=ra_mkn421,
dec=dec_mkn421,
spectral_shape=spectrum)
spectrum.piv = 1 * u.TeV
spectrum.piv.fix = True
spectrum.K = 1e-14 / (u.TeV * u.cm**2 * u.s) # norm (in 1/(keV cm2 s))
spectrum.K.bounds = (1e-25, 1e-19) # without units energies are in keV
spectrum.beta = 0 # log parabolic beta
spectrum.beta.bounds = (-4., 2.)
spectrum.alpha = -2.5 # log parabolic alpha (index)
spectrum.alpha.bounds = (-4., 2.)
model = Model(source)
data = DataList(hawc)
jl = JointLikelihood(model, data, verbose=False)
jl.set_minimizer("ROOT")
param_df, like_df = jl.fit()
# See the model in counts space and the residuals
fig = hawc.display_spectrum()
# Save it to file
fig.savefig("hal_mkn421_residuals.png")
# Look at the different energy planes (the columns are model, data, residuals)
fig = hawc.display_fit(smoothing_kernel_sigma=0.3)
fig.savefig("hal_mkn421_fit_planes.png")
# Compute TS
jl.compute_TS("mkn421", like_df)
# Compute goodness of fit with Monte Carlo
gf = GoodnessOfFit(jl)
gof, param, likes = gf.by_mc(100)
print(
"Prob. of obtaining -log(like) >= observed by chance if null hypothesis is true: %.2f"
% gof['HAWC'])
# it is a good idea to inspect the results of the simulations with some plots
# Histogram of likelihood values
fig, sub = plt.subplots()
likes.hist(ax=sub)
# Overplot a vertical dashed line on the observed value
plt.axvline(jl.results.get_statistic_frame().loc['HAWC',
'-log(likelihood)'],
color='black',
linestyle='--')
fig.savefig("hal_sim_all_likes.png")
# Plot the value of beta for all simulations (for example)
fig, sub = plt.subplots()
param.loc[(slice(None), ['mkn421.spectrum.main.Log_parabola.beta']),
'value'].plot()
fig.savefig("hal_sim_all_beta.png")
# Free the position of the source
source.position.ra.free = True
source.position.dec.free = True
# Set boundaries (no need to go further than this)
source.position.ra.bounds = (ra_mkn421 - 0.5, ra_mkn421 + 0.5)
source.position.dec.bounds = (dec_mkn421 - 0.5, dec_mkn421 + 0.5)
# Fit with position free
param_df, like_df = jl.fit()
# Make localization contour
a, b, cc, fig = jl.get_contours(
model.mkn421.position.dec,
38.15,
38.22,
10,
model.mkn421.position.ra,
166.08,
166.18,
10,
)
plt.plot([ra_mkn421], [dec_mkn421], 'x')
fig.savefig("hal_mkn421_localization.png")
# Of course we can also do a Bayesian analysis the usual way
# NOTE: here the position is still free, so we are going to obtain marginals about that
# as well
# For this quick example, let's use a uniform prior for all parameters
for parameter in model.parameters.values():
if parameter.fix:
continue
if parameter.is_normalization:
parameter.set_uninformative_prior(Log_uniform_prior)
else:
parameter.set_uninformative_prior(Uniform_prior)
# Let's execute our bayes analysis
bs = BayesianAnalysis(model, data)
samples = bs.sample(30, 100, 100)
fig = bs.corner_plot()
fig.savefig("hal_corner_plot.png")
def test_healpixRoi(geminga_maptree, geminga_response):
#test to make sure writing a model with HealpixMapROI works fine
ra, dec = 101.7, 16.
data_radius = 9.
model_radius = 24.
m = np.zeros(hp.nside2npix(NSIDE))
vec = Sky2Vec(ra, dec)
m[hp.query_disc(NSIDE,
vec, (data_radius * u.degree).to(u.radian).value,
inclusive=False)] = 1
#hp.fitsfunc.write_map("roitemp.fits" , m, nest=False, coord="C", partial=False, overwrite=True )
map_roi = HealpixMapROI(data_radius=data_radius,
ra=ra,
dec=dec,
model_radius=model_radius,
roimap=m)
#fits_roi = HealpixMapROI(data_radius=data_radius, ra=ra, dec=dec, model_radius=model_radius, roifile="roitemp.fits")
hawc = HAL("HAWC", geminga_maptree, geminga_response, map_roi)
hawc.set_active_measurements(1, 9)
'''
Define model: Two sources, 1 point, 1 extended
Same declination, but offset in RA
Different spectral idnex, but both power laws
'''
pt_shift = 3.0
ext_shift = 2.0
# First soource
spectrum1 = Powerlaw()
source1 = PointSource("point",
ra=ra + pt_shift,
dec=dec,
spectral_shape=spectrum1)
spectrum1.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum1.piv = 1 * u.TeV
spectrum1.index = -2.3
spectrum1.piv.fix = True
spectrum1.K.fix = True
spectrum1.index.fix = True
# Second source
shape = Gaussian_on_sphere(lon0=ra - ext_shift, lat0=dec, sigma=0.3)
spectrum2 = Powerlaw()
source2 = ExtendedSource("extended",
spatial_shape=shape,
spectral_shape=spectrum2)
spectrum2.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum2.piv = 1 * u.TeV
spectrum2.index = -2.0
spectrum2.piv.fix = True
spectrum2.K.fix = True
spectrum2.index.fix = True
shape.lon0.fix = True
shape.lat0.fix = True
shape.sigma.fix = True
model = Model(source1, source2)
hawc.set_model(model)
# Write the model map
model_map_tree = hawc.write_model_map("test.hd5", test_return_map=True)
# Read the model back
hawc_model = map_tree_factory('test.hd5', map_roi)
# Check written model and read model are the same
check_map_trees(hawc_model, model_map_tree)
os.remove("test.hd5")
class LimitCalculator:
def __init__(self,
maptree,
response,
model,
verbose=False,
process="annihilation",
energy_bins=True):
self.maptree = maptree
self.response = response
self.model = model
self.verbose = verbose
self.DM_process = process
# - construct the log likelihood
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
# these are the bins from the Energy Estimator, Spectrum Fitting Memo
if energy_bins:
self.hawc.set_active_measurements(bin_list=[
"1c", "1d", "1e", "2c", "2d", "2e", "2f", "3c", "3d", "3e",
"3f", "4c", "4d", "4e", "4f", "4g", "5d", "5e", "5f", "5g",
"5h", "6e", "6f", "6g", "6h", "7f", "7g", "7h", "7i", "8f",
"8g", "8h", "8i", "8j", "9g", "9h", "9i", "9j", "9k", "9l"
])
else:
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
self.jl = JointLikelihood(self.model, self.datalist, verbose=True)
"""
def set_ROI(self, ra, dec, radius):
self.llh.set_ROI(ra, dec, radius, True)
if self.verbose:
print("ROI is set to ({:.2f}, {:.2f}) with r={:.2f}".format(ra, dec, radius))
def set_masked_ROI(self, maskFilename):
self.llh.set_template_ROI(maskFilename, 0.5, True)
if self.verbose:
print("ROI is adjusted according to {filename}".format(filename=maskFilename))
def set_range(self, minimum, maximum):
self.min = minimum
self.max = maximum
def calculate_limit(self, rel_err=1.e-3, do_flip=True):
best_fit, TS_max = self.find_max_TS_3ml_style()
#best_fit, TS_max = self.find_max_TS()
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.bounds = (-1e-24, 1e-15)
if best_fit < 0 and do_flip:
print("the best fit is negative. taking care of it now")
best_fit = 0
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = best_fit
TS_max = self.llh.calc_TS()
lo = best_fit
lo_TS = TS_max
del_lo_TS = 2.71 - (TS_max-lo_TS)
hi = lo*20.
if hi == 0:
hi = 1e-15
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = hi
hi_TS = self.llh.calc_TS()
del_hi_TS = 2.71 - (TS_max-hi_TS)
while True:
mid = (lo+hi)/2.
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = mid
mid_TS = self.llh.calc_TS()
del_mid_TS = 2.71 - (TS_max-mid_TS)
if np.fabs(del_mid_TS) < rel_err:
norm_95cl = mid
TS_95cl = mid_TS
print("difference: {}".format(TS_95cl))
print("limit: {}".format(norm_95cl))
break
else:
if del_mid_TS*del_hi_TS > 0:
hi = mid
else:
lo = mid
print("current value: {}".format(mid))
print("current TS: {}".format(mid_TS))
print("current diff: {}".format(del_mid_TS))
"""
def calculate_limit(self, ll):
num_steps = 100
bar = progressbar.ProgressBar(maxval=num_steps,
widgets=[
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
])
TS = []
result = ll.fit()
bar.start()
bar.update(0)
if self.DM_process == "annihilation":
best_fit = ll.likelihood_model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value
else:
best_fit = ll.likelihood_model.M49.spectrum.main.DMDecayFlux.tau.value
vals = np.logspace(
np.log10(best_fit) - 1.0,
np.log10(best_fit) + 1.0, num_steps)
for ival, val in enumerate(vals):
if self.DM_process == "annihilation":
ll.likelihood_model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = val
else:
ll.likelihood_model.M49.spectrum.main.DMDecayFlux.tau.value = val
curr_LL = ll.data_list.values()[0].inner_fit()
TS.append(curr_LL)
bar.update(ival)
TS = np.array(TS)
TS = -TS
TS -= np.min(TS)
plt.semilogx(vals, TS)
plt.savefig("example_step1.pdf")
plt.show()
if self.DM_process == "annihilation":
selected_indices = vals > best_fit
else:
selected_indices = vals < best_fit
interpolator = interp1d(TS[selected_indices], vals[selected_indices])
return (best_fit, interpolator(1.35))
def test_model_residual_maps(geminga_maptree, geminga_response, geminga_roi):
#data_radius = 5.0
#model_radius = 7.0
output = dirname(geminga_maptree)
ra_src, dec_src = 101.7, 16.0
maptree, response, roi = geminga_maptree, geminga_response, geminga_roi
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
'''
Define model: Two sources, 1 point, 1 extended
Same declination, but offset in RA
Different spectral index, but both power laws
'''
pt_shift = 3.0
ext_shift = 2.0
# First source
spectrum1 = Powerlaw()
source1 = PointSource("point",
ra=ra_src + pt_shift,
dec=dec_src,
spectral_shape=spectrum1)
spectrum1.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum1.piv = 1 * u.TeV
spectrum1.index = -2.3
spectrum1.piv.fix = True
spectrum1.K.fix = True
spectrum1.index.fix = True
# Second source
shape = Gaussian_on_sphere(lon0=ra_src - ext_shift,
lat0=dec_src,
sigma=0.3)
spectrum2 = Powerlaw()
source2 = ExtendedSource("extended",
spatial_shape=shape,
spectral_shape=spectrum2)
spectrum2.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum2.piv = 1 * u.TeV
spectrum2.index = -2.0
shape.lon0.fix = True
shape.lat0.fix = True
shape.sigma.fix = True
spectrum2.piv.fix = True
spectrum2.K.fix = True
spectrum2.index.fix = True
# Define model with both sources
model = Model(source1, source2)
# Define the data we are using
data = DataList(hawc)
# Define the JointLikelihood object (glue the data to the model)
jl = JointLikelihood(model, data, verbose=False)
# This has the effect of loading the model cache
fig = hawc.display_spectrum()
# the test file names
model_file_name = "{0}/test_model.hdf5".format(output)
residual_file_name = "{0}/test_residual.hdf5".format(output)
# Write the map trees for testing
model_map_tree = hawc.write_model_map(model_file_name,
poisson_fluctuate=True,
test_return_map=True)
residual_map_tree = hawc.write_residual_map(residual_file_name,
test_return_map=True)
# Read the maps back in
hawc_model = map_tree_factory(model_file_name, roi)
hawc_residual = map_tree_factory(residual_file_name, roi)
check_map_trees(hawc_model, model_map_tree)
check_map_trees(hawc_residual, residual_map_tree)
def test_on_point_source(ra=ra_crab,
dec=dec_crab,
liff=False,
maptree=os.path.join(test_data_path,
"maptree_1024.root"),
response=os.path.join(test_data_path,
"response.root"),
data_radius=5.0,
model_radius=10.0):
if not liff:
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra,
dec=dec)
# This is a 3ML plugin
hawc = HAL("HAWC", maptree, response, roi)
else:
from threeML import HAWCLike
hawc = HAWCLike("HAWC",
os.path.join(test_data_path, "maptree_1024.root"),
os.path.join(test_data_path, "response.root"),
fullsky=True)
hawc.set_ROI(ra, dec, data_radius)
hawc.set_active_measurements(1, 9)
if not liff: hawc.display()
spectrum = Log_parabola()
source = PointSource("pts", ra=ra, dec=dec, spectral_shape=spectrum)
# NOTE: if you use units, you have to set up the values for the parameters
# AFTER you create the source, because during creation the function Log_parabola
# gets its units
source.position.ra.bounds = (ra - 0.5, ra + 0.5)
source.position.dec.bounds = (dec - 0.5, dec + 0.5)
if ra == ra_crab:
spectrum.piv = 10 * u.TeV # Pivot energy as in the paper of the Crab
else:
spectrum.piv = 1 * u.TeV # Pivot energy
spectrum.piv.fix = True
spectrum.K = 1e-14 / (u.TeV * u.cm**2 * u.s) # norm (in 1/(keV cm2 s))
spectrum.K.bounds = (1e-25, 1e-19) # without units energies are in keV
spectrum.beta = 0 # log parabolic beta
spectrum.beta.bounds = (-4., 2.)
spectrum.alpha = -2.5 # log parabolic alpha (index)
spectrum.alpha.bounds = (-4., 2.)
model = Model(source)
data = DataList(hawc)
jl = JointLikelihood(model, data, verbose=False)
beg = time.time()
param_df, like_df = jl.fit()
print("Fit time: %s" % (time.time() - beg))
return param_df, like_df
shape2.radius = 0.63 * u.degree
shape2.radius.fix = True
shape2.radius.max_value = 0.63 * u.degree
lm = Model(source, source2, source3)
bin_list = "1c 1d 1e 1f 2c 2d 2e 2f 3c 3d 3e 3f 4c 4d 4e 4f 4g 5e 5f 5g 5h 6e 6f 6g 6h 7f 7g 7h 7i 8g 8h 8i 8j 9g 9h 9i 9j 9k 9l".split()
ra, dec = 307.17, 41.17
#data_radius = 6.0
model_radius = 8.0
fits_roi = HealpixMapROI(ra = ra, dec = dec, model_radius=model_radius, roifile="roi.fits")
hawc = HAL("HAWC", maptree, response,fits_roi)
hawc.set_active_measurements(bin_list=bin_list)
#hawc.display()
# Double check the free parameters
print("Likelihood model:\n")
print(lm)
# Set up the likelihood and run the fit
print("Performing likelihood fit...\n")
datalist = DataList(hawc)
jl = JointLikelihood(lm, datalist, verbose=True)
jl.set_minimizer("ROOT")
param_df, like_df = jl.fit()
# Define the ROI
ra_crab, dec_crab = 83.63, 22.02
data_radius = 3.0
model_radius = 8.0
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra_crab,
dec=dec_crab)
# Instance the plugin
maptree = "../data/HAWC_9bin_507days_crab_data.hd5"
response = "../data/HAWC_9bin_507days_crab_response.hd5"
hawc = HAL("HAWC", maptree, response, roi, flat_sky_pixels_size=0.1)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("public_crab_logParabola_stacked_image.png")
# Define model
spectrum = Log_parabola()