def fit_point_source(roi, maptree, response, point_source_model, liff=False):
data_radius = roi.data_radius.to("deg").value
if not liff:
# This is a 3ML plugin
hawc = HAL("HAWC", maptree, response, roi)
else:
from threeML import HAWCLike
hawc = HAWCLike("HAWC", maptree, response, fullsky=True)
ra_roi, dec_roi = roi.ra_dec_center
hawc.set_ROI(ra_roi, dec_roi, data_radius)
hawc.set_active_measurements(1, 9)
if not liff: hawc.display()
data = DataList(hawc)
jl = JointLikelihood(point_source_model, data, verbose=True)
point_source_model.display(complete=True)
beg = time.time()
jl.set_minimizer("ROOT")
param_df, like_df = jl.fit()
# _ = jl.get_errors()
print("Fit time: %s" % (time.time() - beg))
if not liff:
hawc.display_fit(display_colorbar=True).savefig("display_fit.png")
return param_df, like_df
def test_fit(roi, maptree, response):
pts_model = point_source_model()
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Get the likelihood value for the saturated model
hawc.get_saturated_model_likelihood()
data = DataList(hawc)
jl = JointLikelihood(pts_model, data, verbose=True)
param_df, like_df = jl.fit()
return jl, hawc, pts_model, param_df, like_df, data
def test_complete_analysis(roi, maptree, response, point_source_model):
# Define the ROI
# Instance the plugin
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Get the likelihood value for the saturated model
hawc.get_saturated_model_likelihood()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("hal_src_stacked_image.png")
data = DataList(hawc)
jl = JointLikelihood(point_source_model, data, verbose=False)
param_df, like_df = jl.fit()
# Generate a simulated dataset and write it to disk
sim = hawc.get_simulated_dataset("HAWCsim")
sim.write("sim_resp.hd5", "sim_maptree.hd5")
# See the model in counts space and the residuals
fig = hawc.display_spectrum()
# Save it to file
fig.savefig("hal_src_residuals.png")
# Look at the different energy planes (the columns are model, data, residuals)
fig = hawc.display_fit(smoothing_kernel_sigma=0.3)
fig.savefig("hal_src_fit_planes.png")
# Compute TS
src_name = point_source_model.pts.name
jl.compute_TS(src_name, like_df)
# Compute goodness of fit with Monte Carlo
gf = GoodnessOfFit(jl)
gof, param, likes = gf.by_mc(10)
print(
"Prob. of obtaining -log(like) >= observed by chance if null hypothesis is true: %.2f"
% gof['HAWC'])
# it is a good idea to inspect the results of the simulations with some plots
# Histogram of likelihood values
fig, sub = plt.subplots()
likes.hist(ax=sub)
# Overplot a vertical dashed line on the observed value
plt.axvline(jl.results.get_statistic_frame().loc['HAWC',
'-log(likelihood)'],
color='black',
linestyle='--')
fig.savefig("hal_sim_all_likes.png")
# Plot the value of beta for all simulations (for example)
fig, sub = plt.subplots()
param.loc[(slice(None), ['pts.spectrum.main.Cutoff_powerlaw.index']),
'value'].plot()
fig.savefig("hal_sim_all_index.png")
# Free the position of the source
point_source_model.pts.position.ra.free = True
point_source_model.pts.position.dec.free = True
# Set boundaries (no need to go further than this)
ra = point_source_model.pts.position.ra.value
dec = point_source_model.pts.position.dec.value
point_source_model.pts.position.ra.bounds = (ra - 0.5, ra + 0.5)
point_source_model.pts.position.dec.bounds = (dec - 0.5, dec + 0.5)
# Fit with position free
param_df, like_df = jl.fit()
# Make localization contour
# pts.position.ra(8.362 + / - 0.00028) x
# 10
# deg
# pts.position.dec(2.214 + / - 0.00025) x
# 10
a, b, cc, fig = jl.get_contours(point_source_model.pts.position.dec, 22.13,
22.1525, 10,
point_source_model.pts.position.ra, 83.615,
83.635, 10)
plt.plot([ra], [dec], 'x')
fig.savefig("hal_src_localization.png")
# Of course we can also do a Bayesian analysis the usual way
# NOTE: here the position is still free, so we are going to obtain marginals about that
# as well
# For this quick example, let's use a uniform prior for all parameters
for parameter in point_source_model.parameters.values():
if parameter.fix:
continue
if parameter.is_normalization:
parameter.set_uninformative_prior(Log_uniform_prior)
else:
parameter.set_uninformative_prior(Uniform_prior)
# Let's execute our bayes analysis
bs = BayesianAnalysis(point_source_model, data)
samples = bs.sample(30, 20, 20)
fig = bs.corner_plot()
fig.savefig("hal_corner_plot.png")
def test_on_point_source(ra=ra_crab,
dec=dec_crab,
liff=False,
maptree=os.path.join(test_data_path,
"maptree_1024.root"),
response=os.path.join(test_data_path,
"response.root"),
data_radius=5.0,
model_radius=10.0):
if not liff:
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra,
dec=dec)
# This is a 3ML plugin
hawc = HAL("HAWC", maptree, response, roi)
else:
from threeML import HAWCLike
hawc = HAWCLike("HAWC",
os.path.join(test_data_path, "maptree_1024.root"),
os.path.join(test_data_path, "response.root"),
fullsky=True)
hawc.set_ROI(ra, dec, data_radius)
hawc.set_active_measurements(1, 9)
if not liff: hawc.display()
spectrum = Log_parabola()
source = PointSource("pts", ra=ra, dec=dec, spectral_shape=spectrum)
# NOTE: if you use units, you have to set up the values for the parameters
# AFTER you create the source, because during creation the function Log_parabola
# gets its units
source.position.ra.bounds = (ra - 0.5, ra + 0.5)
source.position.dec.bounds = (dec - 0.5, dec + 0.5)
if ra == ra_crab:
spectrum.piv = 10 * u.TeV # Pivot energy as in the paper of the Crab
else:
spectrum.piv = 1 * u.TeV # Pivot energy
spectrum.piv.fix = True
spectrum.K = 1e-14 / (u.TeV * u.cm**2 * u.s) # norm (in 1/(keV cm2 s))
spectrum.K.bounds = (1e-25, 1e-19) # without units energies are in keV
spectrum.beta = 0 # log parabolic beta
spectrum.beta.bounds = (-4., 2.)
spectrum.alpha = -2.5 # log parabolic alpha (index)
spectrum.alpha.bounds = (-4., 2.)
model = Model(source)
data = DataList(hawc)
jl = JointLikelihood(model, data, verbose=False)
beg = time.time()
param_df, like_df = jl.fit()
print("Fit time: %s" % (time.time() - beg))
return param_df, like_df
class LimitCalculator:
def __init__(self, maptree, response, models, verbose=False):
self.maptree = maptree
self.response = response
self.verbose = verbose
self.models = models
self.fluxUnit = (1. / (u.TeV * u.cm**2 * u.s))
# - construct HAL
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
# - construct joint likelihood
self.jl_M87 = JointLikelihood(self.models[0],
self.datalist,
verbose=True)
self.jl_M49 = JointLikelihood(self.models[1],
self.datalist,
verbose=True)
self.jl_linked = JointLikelihood(self.models[2],
self.datalist,
verbose=False)
self.jl_notlinked = JointLikelihood(self.models[3],
self.datalist,
verbose=True)
# - set the minimizer
self.jl_M87.set_minimizer("minuit")
self.jl_M49.set_minimizer("minuit")
self.jl_linked.set_minimizer("minuit")
self.jl_notlinked.set_minimizer("minuit")
def set_range(self, minimum, maximum):
self.min = minimum
self.max = maximum
def get_M87_likelihood(self):
return self.jl_M87
def get_M49_likelihood(self):
return self.jl_M49
def get_linked_likelihood(self):
return self.jl_linked
def calculate_limit(self, ll):
num_steps = 100
bar = progressbar.ProgressBar(maxval=num_steps,
widgets=[
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
])
TS = []
result = ll.fit()
bar.start()
bar.update(0)
best_fit = ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value
vals = np.logspace(
np.log10(
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value) -
.25,
np.log10(
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value) +
3.0, num_steps)
for ival, val in enumerate(vals):
ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K.value = val
curr_LL = ll.data_list.values()[0].inner_fit()
#print(val, curr_LL)
TS.append(curr_LL)
bar.update(ival)
TS = np.array(TS)
TS = -TS
TS -= np.min(TS)
plt.semilogx(vals, TS)
plt.savefig("example_step1.pdf")
plt.show()
interpolator = interp1d(TS, vals)
return (ll.likelihood_model.M87.spectrum.main.RangedPowerlaw.K,
interpolator(1.35))
def test_complete_analysis(maptree=os.path.join(test_data_path,
"maptree_1024.root"),
response=os.path.join(test_data_path,
"response.root")):
# Define the ROI
ra_mkn421, dec_mkn421 = 166.113808, 38.208833
data_radius = 3.0
model_radius = 8.0
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra_mkn421,
dec=dec_mkn421)
# Instance the plugin
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("hal_mkn421_stacked_image.png")
# Define model as usual
spectrum = Log_parabola()
source = PointSource("mkn421",
ra=ra_mkn421,
dec=dec_mkn421,
spectral_shape=spectrum)
spectrum.piv = 1 * u.TeV
spectrum.piv.fix = True
spectrum.K = 1e-14 / (u.TeV * u.cm**2 * u.s) # norm (in 1/(keV cm2 s))
spectrum.K.bounds = (1e-25, 1e-19) # without units energies are in keV
spectrum.beta = 0 # log parabolic beta
spectrum.beta.bounds = (-4., 2.)
spectrum.alpha = -2.5 # log parabolic alpha (index)
spectrum.alpha.bounds = (-4., 2.)
model = Model(source)
data = DataList(hawc)
jl = JointLikelihood(model, data, verbose=False)
jl.set_minimizer("ROOT")
param_df, like_df = jl.fit()
# See the model in counts space and the residuals
fig = hawc.display_spectrum()
# Save it to file
fig.savefig("hal_mkn421_residuals.png")
# Look at the different energy planes (the columns are model, data, residuals)
fig = hawc.display_fit(smoothing_kernel_sigma=0.3)
fig.savefig("hal_mkn421_fit_planes.png")
# Compute TS
jl.compute_TS("mkn421", like_df)
# Compute goodness of fit with Monte Carlo
gf = GoodnessOfFit(jl)
gof, param, likes = gf.by_mc(100)
print(
"Prob. of obtaining -log(like) >= observed by chance if null hypothesis is true: %.2f"
% gof['HAWC'])
# it is a good idea to inspect the results of the simulations with some plots
# Histogram of likelihood values
fig, sub = plt.subplots()
likes.hist(ax=sub)
# Overplot a vertical dashed line on the observed value
plt.axvline(jl.results.get_statistic_frame().loc['HAWC',
'-log(likelihood)'],
color='black',
linestyle='--')
fig.savefig("hal_sim_all_likes.png")
# Plot the value of beta for all simulations (for example)
fig, sub = plt.subplots()
param.loc[(slice(None), ['mkn421.spectrum.main.Log_parabola.beta']),
'value'].plot()
fig.savefig("hal_sim_all_beta.png")
# Free the position of the source
source.position.ra.free = True
source.position.dec.free = True
# Set boundaries (no need to go further than this)
source.position.ra.bounds = (ra_mkn421 - 0.5, ra_mkn421 + 0.5)
source.position.dec.bounds = (dec_mkn421 - 0.5, dec_mkn421 + 0.5)
# Fit with position free
param_df, like_df = jl.fit()
# Make localization contour
a, b, cc, fig = jl.get_contours(
model.mkn421.position.dec,
38.15,
38.22,
10,
model.mkn421.position.ra,
166.08,
166.18,
10,
)
plt.plot([ra_mkn421], [dec_mkn421], 'x')
fig.savefig("hal_mkn421_localization.png")
# Of course we can also do a Bayesian analysis the usual way
# NOTE: here the position is still free, so we are going to obtain marginals about that
# as well
# For this quick example, let's use a uniform prior for all parameters
for parameter in model.parameters.values():
if parameter.fix:
continue
if parameter.is_normalization:
parameter.set_uninformative_prior(Log_uniform_prior)
else:
parameter.set_uninformative_prior(Uniform_prior)
# Let's execute our bayes analysis
bs = BayesianAnalysis(model, data)
samples = bs.sample(30, 100, 100)
fig = bs.corner_plot()
fig.savefig("hal_corner_plot.png")
class LimitCalculator:
def __init__(self,
maptree,
response,
model,
verbose=False,
process="annihilation",
energy_bins=True):
self.maptree = maptree
self.response = response
self.model = model
self.verbose = verbose
self.DM_process = process
# - construct the log likelihood
self.roi = HealpixConeROI(data_radius=13.,
model_radius=25.,
ra=187.5745,
dec=10.1974)
self.hawc = HAL("VirgoCluster", maptree, response, self.roi)
# these are the bins from the Energy Estimator, Spectrum Fitting Memo
if energy_bins:
self.hawc.set_active_measurements(bin_list=[
"1c", "1d", "1e", "2c", "2d", "2e", "2f", "3c", "3d", "3e",
"3f", "4c", "4d", "4e", "4f", "4g", "5d", "5e", "5f", "5g",
"5h", "6e", "6f", "6g", "6h", "7f", "7g", "7h", "7i", "8f",
"8g", "8h", "8i", "8j", "9g", "9h", "9i", "9j", "9k", "9l"
])
else:
self.hawc.set_active_measurements(1, 9)
self.hawc.display()
self.datalist = DataList(self.hawc)
self.jl = JointLikelihood(self.model, self.datalist, verbose=True)
"""
def set_ROI(self, ra, dec, radius):
self.llh.set_ROI(ra, dec, radius, True)
if self.verbose:
print("ROI is set to ({:.2f}, {:.2f}) with r={:.2f}".format(ra, dec, radius))
def set_masked_ROI(self, maskFilename):
self.llh.set_template_ROI(maskFilename, 0.5, True)
if self.verbose:
print("ROI is adjusted according to {filename}".format(filename=maskFilename))
def set_range(self, minimum, maximum):
self.min = minimum
self.max = maximum
def calculate_limit(self, rel_err=1.e-3, do_flip=True):
best_fit, TS_max = self.find_max_TS_3ml_style()
#best_fit, TS_max = self.find_max_TS()
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.bounds = (-1e-24, 1e-15)
if best_fit < 0 and do_flip:
print("the best fit is negative. taking care of it now")
best_fit = 0
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = best_fit
TS_max = self.llh.calc_TS()
lo = best_fit
lo_TS = TS_max
del_lo_TS = 2.71 - (TS_max-lo_TS)
hi = lo*20.
if hi == 0:
hi = 1e-15
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = hi
hi_TS = self.llh.calc_TS()
del_hi_TS = 2.71 - (TS_max-hi_TS)
while True:
mid = (lo+hi)/2.
self.model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = mid
mid_TS = self.llh.calc_TS()
del_mid_TS = 2.71 - (TS_max-mid_TS)
if np.fabs(del_mid_TS) < rel_err:
norm_95cl = mid
TS_95cl = mid_TS
print("difference: {}".format(TS_95cl))
print("limit: {}".format(norm_95cl))
break
else:
if del_mid_TS*del_hi_TS > 0:
hi = mid
else:
lo = mid
print("current value: {}".format(mid))
print("current TS: {}".format(mid_TS))
print("current diff: {}".format(del_mid_TS))
"""
def calculate_limit(self, ll):
num_steps = 100
bar = progressbar.ProgressBar(maxval=num_steps,
widgets=[
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
])
TS = []
result = ll.fit()
bar.start()
bar.update(0)
if self.DM_process == "annihilation":
best_fit = ll.likelihood_model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value
else:
best_fit = ll.likelihood_model.M49.spectrum.main.DMDecayFlux.tau.value
vals = np.logspace(
np.log10(best_fit) - 1.0,
np.log10(best_fit) + 1.0, num_steps)
for ival, val in enumerate(vals):
if self.DM_process == "annihilation":
ll.likelihood_model.M49.spectrum.main.DMAnnihilationFlux.sigmav.value = val
else:
ll.likelihood_model.M49.spectrum.main.DMDecayFlux.tau.value = val
curr_LL = ll.data_list.values()[0].inner_fit()
TS.append(curr_LL)
bar.update(ival)
TS = np.array(TS)
TS = -TS
TS -= np.min(TS)
plt.semilogx(vals, TS)
plt.savefig("example_step1.pdf")
plt.show()
if self.DM_process == "annihilation":
selected_indices = vals > best_fit
else:
selected_indices = vals < best_fit
interpolator = interp1d(TS[selected_indices], vals[selected_indices])
return (best_fit, interpolator(1.35))
def test_model_residual_maps(geminga_maptree, geminga_response, geminga_roi):
#data_radius = 5.0
#model_radius = 7.0
output = dirname(geminga_maptree)
ra_src, dec_src = 101.7, 16.0
maptree, response, roi = geminga_maptree, geminga_response, geminga_roi
hawc = HAL("HAWC", maptree, response, roi)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
'''
Define model: Two sources, 1 point, 1 extended
Same declination, but offset in RA
Different spectral index, but both power laws
'''
pt_shift = 3.0
ext_shift = 2.0
# First source
spectrum1 = Powerlaw()
source1 = PointSource("point",
ra=ra_src + pt_shift,
dec=dec_src,
spectral_shape=spectrum1)
spectrum1.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum1.piv = 1 * u.TeV
spectrum1.index = -2.3
spectrum1.piv.fix = True
spectrum1.K.fix = True
spectrum1.index.fix = True
# Second source
shape = Gaussian_on_sphere(lon0=ra_src - ext_shift,
lat0=dec_src,
sigma=0.3)
spectrum2 = Powerlaw()
source2 = ExtendedSource("extended",
spatial_shape=shape,
spectral_shape=spectrum2)
spectrum2.K = 1e-12 / (u.TeV * u.cm**2 * u.s)
spectrum2.piv = 1 * u.TeV
spectrum2.index = -2.0
shape.lon0.fix = True
shape.lat0.fix = True
shape.sigma.fix = True
spectrum2.piv.fix = True
spectrum2.K.fix = True
spectrum2.index.fix = True
# Define model with both sources
model = Model(source1, source2)
# Define the data we are using
data = DataList(hawc)
# Define the JointLikelihood object (glue the data to the model)
jl = JointLikelihood(model, data, verbose=False)
# This has the effect of loading the model cache
fig = hawc.display_spectrum()
# the test file names
model_file_name = "{0}/test_model.hdf5".format(output)
residual_file_name = "{0}/test_residual.hdf5".format(output)
# Write the map trees for testing
model_map_tree = hawc.write_model_map(model_file_name,
poisson_fluctuate=True,
test_return_map=True)
residual_map_tree = hawc.write_residual_map(residual_file_name,
test_return_map=True)
# Read the maps back in
hawc_model = map_tree_factory(model_file_name, roi)
hawc_residual = map_tree_factory(residual_file_name, roi)
check_map_trees(hawc_model, model_map_tree)
check_map_trees(hawc_residual, residual_map_tree)
roi = HealpixConeROI(data_radius=data_radius,
model_radius=model_radius,
ra=ra_crab,
dec=dec_crab)
# Instance the plugin
maptree = "../data/HAWC_9bin_507days_crab_data.hd5"
response = "../data/HAWC_9bin_507days_crab_response.hd5"
hawc = HAL("HAWC", maptree, response, roi, flat_sky_pixels_size=0.1)
# Use from bin 1 to bin 9
hawc.set_active_measurements(1, 9)
# Display information about the data loaded and the ROI
hawc.display()
# Look at the data
fig = hawc.display_stacked_image(smoothing_kernel_sigma=0.17)
# Save to file
fig.savefig("public_crab_logParabola_stacked_image.png")
# Define model
spectrum = Log_parabola()
source = PointSource("crab", ra=ra_crab, dec=dec_crab, spectral_shape=spectrum)
spectrum.piv = 7 * u.TeV
spectrum.piv.fix = True
spectrum.K = 1e-14 / (u.TeV * u.cm**2 * u.s) # norm (in 1/(keV cm2 s))