def loadUnbinnedObs(self, f, verbosity=0):
if verbosity:
print 'Loading unbinned observation:',f['ft1']
obs = UA.UnbinnedObs(eventFile=f['ft1'], scFile=f['ft2'],
expMap=f['emap'],expCube=f['ecube'],
irfs=f['irfs'])
like = UA.UnbinnedAnalysis(obs, srcModel=self.model,
optimizer=self.optimizer)
return [ obs, like ]
def makeObs(self):
"""Creates either a binned or unbinned observation object for
use in the likelihood analysis. This function checks for all
of the needed files first. If you do not have a needed file,
see the quickAnalysis module for creation. This function
should be run before any of the init or fit functions."""
if (self.commonConf['binned']):
try:
qU.checkForFiles(self.logger, [
self.commonConf['base'] + '_srcMaps.fits',
self.commonConf['base'] + '_ltcube.fits',
self.commonConf['base'] + '_BinnedExpMap.fits'
])
self.obs = BAn.BinnedObs(
srcMaps=self.commonConf['base'] + '_srcMaps.fits',
expCube=self.commonConf['base'] + '_ltcube.fits',
binnedExpMap=self.commonConf['base'] +
'_BinnedExpMap.fits',
irfs=self.commonConf['irfs'])
except (qU.FileNotFound):
self.logger.critical("One or more needed files do not exist")
sys.exit()
else:
try:
qU.checkForFiles(self.logger, [
self.commonConf['base'] + '_filtered_gti.fits',
self.commonConf['base'] + '_SC.fits',
self.commonConf['base'] + '_expMap.fits',
self.commonConf['base'] + '_ltcube.fits'
])
self.obs = UbAn.UnbinnedObs(
self.commonConf['base'] + '_filtered_gti.fits',
self.commonConf['base'] + '_SC.fits',
expMap=self.commonConf['base'] + '_expMap.fits',
expCube=self.commonConf['base'] + '_ltcube.fits',
irfs=self.commonConf['irfs'])
except (qU.FileNotFound):
self.logger.critical("One or more needed files do not exist")
sys.exit()
self.logger.info(self.ret.subn(', ', str(self.obs))[0])
poi_values=poi_values,
poi_probs=poi_probs,
poi_dlogL=poi_dlogL,
flux_emin=emin,
flux_emax=emax)
return ul_flux, results
if __name__ == "__main__":
import sys
srcName = "EMS0001"
obs = UnbinnedAnalysis.UnbinnedObs('ft1_roi.fits',
scFile='ft2.fits',
expMap='expMap.fits',
expCube='expCube.fits',
irfs='P6_V9_DIFFUSE')
#min_opt = 'InteractiveMinuit,MIN 0 $TOL,HESSE,.q'
#pro_opt = 'InteractiveMinuit,SET STR 0,MIN 0 $TOL,.q'
min_opt = 'MINUIT'
pro_opt = None
like = UnbinnedAnalysis.UnbinnedAnalysis(obs, 'model.xml', min_opt)
src_spectrum = like[srcName].funcs['Spectrum']
par = src_spectrum.getParam("Index")
if par:
par.setFree(False)
par.setValue(-2.0)
def __init__(
self,
name,
eventFile,
ft2File,
livetimeCube,
kind,
exposureMap=None,
sourceMaps=None,
binnedExpoMap=None,
source_name=None,
):
# Initially the nuisance parameters dict is empty, as we don't know yet
# the likelihood model. They will be updated in set_model
super(FermiLATLike, self).__init__(name, {})
# Read the ROI cut
cc = pyLike.RoiCuts()
cc.readCuts(eventFile, "EVENTS")
self.ra, self.dec, self.rad = cc.roiCone()
# Read the IRF selection
c = pyLike.Cuts(eventFile, "EVENTS")
self.irf = c.CALDB_implied_irfs()
self.ft2File = ft2File
self.livetimeCube = livetimeCube
# These are the boundaries and the number of energies for the computation
# of the model (in keV)
self.emin = 1e4
self.emax = 5e8
self.Nenergies = 200
# This is the limit on the effective area correction factor,
# which is a multiplicative factor in front of the whole model
# to account for inter-calibration issues. By default it can vary
# by 10%. This can be changed by issuing:
# FermiLATUnbinnedLikeInstance.effCorrLimit = [new limit]
# where for example a [new limit] of 0.2 allow for an effective
# area correction up to +/- 20 %
self.effCorrLimit = 0.1
if kind.upper() != "UNBINNED" and kind.upper() != "BINNED":
raise RuntimeError("Accepted values for the kind parameter are: " +
"binned, unbinned. You specified: %s" % (kind))
else:
self.kind = kind.upper()
if kind.upper() == "UNBINNED":
assert exposureMap is not None, "You have to provide an exposure map"
self.eventFile = eventFile
self.exposureMap = exposureMap
# Read the files and generate the pyLikelihood object
self.obs = UnbinnedAnalysis.UnbinnedObs(
self.eventFile,
self.ft2File,
expMap=self.exposureMap,
expCube=self.livetimeCube,
irfs=self.irf,
)
elif kind.upper() == "BINNED":
assert sourceMaps is not None, "You have to provide a source map"
assert (
binnedExpoMap
is not None), "You have to provided a (binned) exposure map"
self.sourceMaps = sourceMaps
self.binnedExpoMap = binnedExpoMap
self.obs = BinnedAnalysis.BinnedObs(
srcMaps=self.sourceMaps,
expCube=self.livetimeCube,
binnedExpMap=self.binnedExpoMap,
irfs=self.irf,
)
pass
# Activate inner minimization by default
self.setInnerMinimization(True)
self._source_name = source_name
def runLikelihood(subdir, tpl_file):
'''This runction runs the likelihood code on a set of pixels in a
subdirectory. It takes as input the subdirectory to work on and a
template counts map. It reads it's configuration from a pickle
file (par.pck) that should be located in the subdirectory and the
pixel locations from another pickle file (pixel.pck). It then
creats an overall likelihood object, does a quick global fit and
then loops over the pixels. At each pixel, it creats a test
source, fits that source, calculates the TS of the source and
writes the results to an output file in the subdirectory called
ts_results.dat.'''
parfile = open("par.pck", "r")
pars = pickle.load(parfile)
pixelfile = open("pixel.pck", "r")
pixels = pickle.load(pixelfile)
pixel_coords = PixelCoords(tpl_file)
if pars['statistic'] == 'UNBINNED':
import UnbinnedAnalysis as UBAn
obs = UBAn.UnbinnedObs(resolve_fits_files(pars['evfile']),
resolve_fits_files(pars['scfile']),
expMap='../' + pars['expmap'],
expCube='../' + pars['expcube'],
irfs=pars['irfs'])
like = UBAn.UnbinnedAnalysis(obs, '../' + pars['srcmdl'],
pars['optimizer'])
elif pars['statistic'] == 'BINNED':
import BinnedAnalysis as BAn
obs = BAn.BinnedObs(srcMaps='../' + pars['srcmaps'],
expCube='../' + pars['expcube'],
binnedExpMap='../' + pars['bexpmap'],
irfs=pars['irfs'])
like = BAn.BinnedAnalysis(obs, '../' + pars['srcmdl'],
pars['optimizer'])
like.setFitTolType(pars['toltype'])
like.optimize(0)
loglike0 = like()
test_src = getPointSource(like)
target_name = 'testSource'
test_src.setName(target_name)
outfile = 'ts_results.dat'
finished_pixels = []
if os.path.isfile(outfile):
input = open(outfile, 'r')
for line in input:
tokens = line.strip().split()
ij = int(tokens[0]), int(tokens[1])
finished_pixels.append(ij)
input.close()
output = open(outfile, 'a')
for indx, i, j in pixels:
if (i, j) in finished_pixels:
continue
ra, dec = pixel_coords(i, j)
test_src.setDir(ra, dec, True, False)
like.addSource(test_src)
like.optimize(0)
ts = -2 * (like() - loglike0)
output.write("%3i %3i %.3f %.3f %.5f\n" % (i, j, ra, dec, ts))
output.flush()
like.deleteSource(target_name)
output.close()
def bayesian_ul(**kwargs):
# Instance the unbinned analysis
print("Instancing pyLikelihood...")
unbinned_observation = UnbinnedAnalysis.UnbinnedObs(
kwargs['ft1'], kwargs['ft2'], kwargs['expomap'], kwargs['ltcube'],
'CALDB')
pylike_instance = UnbinnedAnalysis.UnbinnedAnalysis(
unbinned_observation, kwargs['xml'], kwargs['engine'])
print("done")
# Let's start by computing the semi-Bayesian UL from the Science Tools
print("Semi-bayesian upper limit computation with ST...")
# Sync and fit
pylike_instance.syncSrcParams()
pylike_instance.fit()
# Compute ST upper limit
ul = UpperLimits.UpperLimit(pylike_instance, kwargs['src'])
try:
st_bayes_ul, parameter_value = ul.bayesianUL(0.95,
emin=kwargs['emin'],
emax=kwargs['emax'])
except:
# This fails sometimes with RuntimeError: Attempt to set parameter value outside bounds.
print("\n\nWARNING: upper limit computation with ST has failed! \n\n")
st_bayes_ul = -1
st_bayes_ul_ene = -1
# Get back to a good state
pylike_instance = UnbinnedAnalysis.UnbinnedAnalysis(
unbinned_observation, kwargs['xml'], kwargs['engine'])
pylike_instance.fit()
else:
# Convert to energy flux
best_fit_photon_index = pylike_instance[
kwargs['src']].src.spectrum().parameter('Index').getValue()
st_bayes_ul_ene = st_bayes_ul * get_conversion_factor(
best_fit_photon_index, kwargs)
print("done")
# Now find out our free parameters, and define a prior for them
# Prepare the dictionary of parameters. Note that by default they get a uniform prior
# between the current min and max values
free_parameters = collections.OrderedDict()
for p in pylike_instance.model.params:
if p.isFree():
source_name = p.srcName
parameter_name = p.parameter.getName()
p.parameter.setScale(1.0)
free_parameters[(source_name, parameter_name)] = MyParameter(p)
# Now set the priors and the boundaries
# Update boundaries (they will be propagated to the prior as well)
# Isotropic template
if (kwargs['iso'], 'Normalization') in free_parameters:
try:
free_parameters[(kwargs['iso'], 'Normalization')].bounds = (0, 100)
except:
# This happens if the best fit value is outside those boundaries
free_parameters[(kwargs['iso'], 'Normalization')].value = 1.0
free_parameters[(kwargs['iso'], 'Normalization')].bounds = (0, 100)
else:
print("WARNING: Isotropic template is not free to vary (or absent)")
# Galactic template (Truncated Gaussian with systematic error)
if (kwargs['gal'], 'Value') in free_parameters:
try:
free_parameters[(kwargs['gal'], 'Value')].bounds = (0.1, 10.0)
free_parameters[(kwargs['gal'],
'Value')].prior = TruncatedGaussianPrior(
1.0, kwargs['gal_sys_err'])
except:
# This happens if the best fit value is outside those boundaries
free_parameters[(kwargs['gal'], 'Value')].value = 1.0
free_parameters[(kwargs['gal'], 'Value')].bounds = (0.1, 10.0)
else:
print("WARNING: Galactic template is not free to vary (or absent)")
# Photon flux (uniform prior)
if (kwargs['src'], 'Integral') in free_parameters:
try:
free_parameters[(kwargs['src'], 'Integral')].bounds = (0, 10)
except:
free_parameters[(kwargs['src'], 'Integral')].value = 1e-7
free_parameters[(kwargs['src'], 'Integral')].bounds = (0, 10)
else:
raise RuntimeError(
"The Integral parameter must be a free parameter of source %s" %
kwargs['src'])
# Photon index
if (kwargs['src'], 'Index') in free_parameters:
try:
free_parameters[(kwargs['src'],
'Index')].bounds = (kwargs['min_index'],
kwargs['max_index'])
except:
raise RuntimeError(
"It looks like the best fit photon index is outside the boundaries "
"provided in the command line")
else:
raise RuntimeError(
"The Index parameter must be a free parameter of source %s" %
kwargs['src'])
# Execute a fit to get to a good state with the new boundaries
pylike_instance.fit()
# Print the configuration
print("\nFree parameters:")
print("----------------\n")
for k, v in free_parameters.iteritems():
print("* %s of %s (%s)" % (k[1], k[0], v.prior.name))
print("")
# Generate the randomized starting points for the Emcee sampler
ndim, nwalkers = len(free_parameters), kwargs['n_walkers']
p0 = [
map(lambda p: p.get_random_init(0.1), free_parameters.values())
for i in range(nwalkers)
]
# Instance the sampler
posterior = Posterior(free_parameters.values(), pylike_instance)
# Now check that the starting points we have are good (otherwise the sampler will go awry)
for pp in p0:
this_ln = posterior.lnprob(pp)
if not np.isfinite(this_ln):
raise RuntimeError(
"Infinite for values %s while setting up walkers" % pp)
sampler = emcee.EnsembleSampler(nwalkers, ndim, posterior.lnprob)
print("Burn in...")
pos, prob, state = sampler.run_mcmc(p0, kwargs['burn_in'])
print("done")
sampler.reset()
print("Sampling...")
samples = sampler.run_mcmc(pos, kwargs['n_samples'])
print("done")
print("Mean acceptance fraction: {0:.3f}".format(
np.mean(sampler.acceptance_fraction)))
# Make the corner plot
samples = sampler.flatchain
labels = map(lambda x: "%s" % (x[1]), free_parameters.keys())
print("Producing corner plot...")
fig = corner.corner(samples,
show_titles=True,
quantiles=[0.5, 0.50, 0.95],
title_fmt=u'.2g',
labels=labels,
plot_contours=True,
plot_density=False)
fig.tight_layout()
fig.savefig(kwargs['corner_plot'])
print("done")
# Now compute the upper limits
# Find index of normalization
norm_index = free_parameters.keys().index((kwargs['src'], 'Integral'))
# Find index of photon index
ph_index_index = free_parameters.keys().index((kwargs['src'], 'Index'))
photon_fluxes = np.zeros(samples.shape[0])
energy_fluxes = np.zeros(samples.shape[0])
conversion_factors = np.zeros(samples.shape[0])
for i, current_sample in enumerate(samples):
# Set the Integral parameter to the current value
free_parameters[(kwargs['src'],
'Integral')].scaled_value = current_sample[norm_index]
# Set the photon index to the current value
current_photon_index = current_sample[ph_index_index]
free_parameters[(kwargs['src'],
'Index')].scaled_value = current_photon_index
pylike_instance.syncSrcParams()
# Get photon flux for this sample
photon_flux = pylike_instance[kwargs['src']].flux(
kwargs['emin'], kwargs['emax'])
# Get energy flux for this value
conv = get_conversion_factor(current_photon_index, kwargs)
energy_flux = photon_flux * conv
# Save the results
photon_fluxes[i] = photon_flux
energy_fluxes[i] = energy_flux
conversion_factors[i] = conv
# Now compute the 95 percentile
photon_flux_p95 = np.percentile(photon_fluxes, 95)
energy_flux_p95 = np.percentile(energy_fluxes, 95)
# Save the samples
np.savez(kwargs['output_file'] + "_samples", samples=samples)
np.savez(kwargs['output_file'],
photon_fluxes=photon_fluxes,
energy_fluxes=energy_fluxes,
photon_flux_p95=photon_flux_p95,
energy_flux_p95=energy_flux_p95,
st_bayes_ul=st_bayes_ul,
st_bayes_ul_ene=st_bayes_ul_ene)
# Now summarize the results
print("\nUpper limit computation results:")
print("----------------------------------\n")
print("Photon flux:\n")
print(" * Semi-bayes from ST : %g" % (st_bayes_ul))
print(" * Bayesian : %g" % photon_flux_p95)
print("\nEnergy flux:\n")
print(" * Semi-bayes from ST : %g" % st_bayes_ul_ene)
print(" * Bayesian : %g" % energy_flux_p95)
# Find expomap
expmaps = glob.glob("%s_expomap.fit*" % root_name)
assert len(expmaps) == 1, "Couldn't find exopmap"
expmap = expmaps[0]
# Find XML model output of gtdolike
xmls = glob.glob("%s_likeRes.xml" % root_name)
assert len(xmls) == 1, "Couldn't find XML"
xml_res = xmls[0]
obs = UnbinnedAnalysis.UnbinnedObs(filteredeventfile, dataset['ft2file'], expMap=expmap, expCube=ltcube)
like = UnbinnedAnalysis.UnbinnedAnalysis(obs, xml_res, 'MINUIT')
ftm = FastTSMap(like)
(bestra, bestdec), maxTS = ftm.search_for_maximum(args.ra, args.dec, float(half_size), int(n_side), verbose=False)
#Now append the results for this interval
grb = filter(lambda x:x.name.find("GRB")>=0,sources)[0]
if args.tsmap_spec is not None:
if maxTS > grb.TS:
print("\n\n=========================================")
print(" Fast TS Map has found a better position")
