def runLikelihood(subdir, tpl_file):
'''This runction runs the likelihood code on a set of pixels in a
subdirectory. It takes as input the subdirectory to work on and a
template counts map. It reads it's configuration from a pickle
file (par.pck) that should be located in the subdirectory and the
pixel locations from another pickle file (pixel.pck). It then
creats an overall likelihood object, does a quick global fit and
then loops over the pixels. At each pixel, it creats a test
source, fits that source, calculates the TS of the source and
writes the results to an output file in the subdirectory called
ts_results.dat.'''
parfile = open("par.pck", "r")
pars = pickle.load(parfile)
pixelfile = open("pixel.pck", "r")
pixels = pickle.load(pixelfile)
pixel_coords = PixelCoords(tpl_file)
obs = UnbinnedObs(resolve_fits_files(pars['evfile']),
resolve_fits_files(pars['scfile']),
expMap='../'+pars['expmap'],
expCube='../'+pars['expcube'],
irfs=pars['irfs'])
like = UnbinnedAnalysis(obs, '../'+pars['srcmdl'], pars['optimizer'])
like.setFitTolType(pars['toltype'])
like.optimize(0)
loglike0 = like()
test_src = getPointSource(like)
target_name = 'testSource'
test_src.setName(target_name)
outfile = 'ts_results.dat'
finished_pixels = []
if os.path.isfile(outfile):
input = open(outfile, 'r')
for line in input:
tokens = line.strip().split()
ij = int(tokens[0]), int(tokens[1])
finished_pixels.append(ij)
input.close()
output = open(outfile, 'a')
for indx, i, j in pixels:
if (i, j) in finished_pixels:
continue
ra, dec = pixel_coords(i, j)
test_src.setDir(ra, dec, True, False)
like.addSource(test_src)
like.optimize(0)
ts = -2*(like() - loglike0)
output.write("%3i %3i %.3f %.3f %.5f\n" % (i, j, ra, dec, ts))
output.flush()
like.deleteSource(target_name)
output.close()
def runLikelihood(subdir, tpl_file):
'''This runction runs the likelihood code on a set of pixels in a
subdirectory. It takes as input the subdirectory to work on and a
template counts map. It reads it's configuration from a pickle
file (par.pck) that should be located in the subdirectory and the
pixel locations from another pickle file (pixel.pck). It then
creats an overall likelihood object, does a quick global fit and
then loops over the pixels. At each pixel, it creats a test
source, fits that source, calculates the TS of the source and
writes the results to an output file in the subdirectory called
ts_results.dat.'''
parfile = open("par.pck", "r")
pars = pickle.load(parfile)
pixelfile = open("pixel.pck", "r")
pixels = pickle.load(pixelfile)
pixel_coords = PixelCoords(tpl_file)
obs = UnbinnedObs(resolve_fits_files(pars['evfile']),
resolve_fits_files(pars['scfile']),
expMap='../' + pars['expmap'],
expCube='../' + pars['expcube'],
irfs=pars['irfs'])
like = UnbinnedAnalysis(obs, '../' + pars['srcmdl'], pars['optimizer'])
like.setFitTolType(pars['toltype'])
like.optimize(0)
loglike0 = like()
test_src = getPointSource(like)
target_name = 'testSource'
test_src.setName(target_name)
outfile = 'ts_results.dat'
finished_pixels = []
if os.path.isfile(outfile):
input = open(outfile, 'r')
for line in input:
tokens = line.strip().split()
ij = int(tokens[0]), int(tokens[1])
finished_pixels.append(ij)
input.close()
output = open(outfile, 'a')
for indx, i, j in pixels:
if (i, j) in finished_pixels:
continue
ra, dec = pixel_coords(i, j)
test_src.setDir(ra, dec, True, False)
like.addSource(test_src)
like.optimize(0)
ts = -2 * (like() - loglike0)
output.write("%3i %3i %.3f %.3f %.5f\n" % (i, j, ra, dec, ts))
output.flush()
like.deleteSource(target_name)
output.close()
# N.B A source with a TS value less than 0 should never happen unless the minimization failed. Remove that source and try fitting again and check the return code again.
converganceCheckOne="Zero?: %d" %(likeobj.getRetCode())
converganceCheckTwo="NA"
print converganceCheckOne # This value should be zero, refit using the method below if not
if not likeobj.getRetCode() ==0 or eMin<100000: # If energy is too high, just don't refit because it causes problems
print colors.OKBLUE+"Minimization Failed. Removing sources with TS < 9.0 and refitting!"
# Protip, you can really simplify the model by removing sources with TS levels below 9 (about 3 sigma)
for source,TS in sourceDetails.iteritems():
print source, TS
if (TS < TSThreshold):
print "Deleting..."
like.deleteSource(source)
# Refit
like.fit(verbosity=0,covar=True,optObject=likeobj)
converganceCheckTwo="Zero?: %d" %(likeobj.getRetCode())
print converganceCheckTwo
sourceDetails = {}
for source in like.sourceNames():
sourceDetails[source] = like.Ts(source)
print colors.ENDC
printDictionaryToFile(sourceDetails,'TSValues.txt')
########################################
def main(NAME, RA, DEC, TSTART, TSTOP, EMIN, EMAX, SC, ROIu, xml):
ROIue = float(ROIu) + 10
os.system('ls -1 *PH*.fits > %s_events.list' % (NAME))
my_apps.filter['evclass'] = 128
my_apps.filter['evtype'] = 3
my_apps.filter['ra'] = RA
my_apps.filter['dec'] = DEC
my_apps.filter['rad'] = ROIu
my_apps.filter['emin'] = EMIN
my_apps.filter['emax'] = EMAX
my_apps.filter['zmax'] = 90
my_apps.filter['tmin'] = TSTART
my_apps.filter['tmax'] = TSTOP
my_apps.filter['infile'] = '@%s_events.list' % (NAME)
my_apps.filter['outfile'] = '%s_filtered.fits' % (NAME)
my_apps.filter.run()
# maketime
my_apps.maketime['scfile'] = SC
my_apps.maketime['filter'] = '(DATA_QUAL>0)&&(LAT_CONFIG==1)'
my_apps.maketime['roicut'] = 'no'
my_apps.maketime['evfile'] = '%s_filtered.fits' % (NAME)
my_apps.maketime['outfile'] = '%s_filtered_gti.fits' % (NAME)
my_apps.maketime.run()
#
my_apps.counts_map['evfile'] = '%s_filtered_gti.fits' % (NAME)
my_apps.counts_map['scfile'] = SC
my_apps.counts_map['outfile'] = '%s_CountMap.fits' % (NAME)
# my_apps.counts_map.run()
#
my_apps.expCube['evfile'] = '%s_filtered_gti.fits' % (NAME)
my_apps.expCube['scfile'] = SC
my_apps.expCube['zmax'] = 90
my_apps.expCube['outfile'] = 'expCube.fits'
my_apps.expCube['dcostheta'] = 0.025
my_apps.expCube['binsz'] = 1
my_apps.expCube.run()
my_apps.expMap['evfile'] = '%s_filtered_gti.fits' % (NAME)
my_apps.expMap['scfile'] = SC
my_apps.expMap['expcube'] = 'expCube.fits'
my_apps.expMap['outfile'] = 'expMap.fits'
my_apps.expMap['irfs'] = 'CALDB'
my_apps.expMap['srcrad'] = ROIue
my_apps.expMap['nlong'] = 120
my_apps.expMap['nlat'] = 120
my_apps.expMap['nenergies'] = 20
my_apps.expMap.run()
# sara xml model
roiname = '%s_filtered_gti.fits' % NAME
if float(xml) == 0:
xml_creator_P8_v1.main(NAME, float(RA), float(DEC), float(EMIN), float(EMAX), 15)
xmlmodelname = '%s_model.xml' % NAME
my_apps.diffResps['evfile'] = '%s_filtered_gti.fits' % (NAME)
my_apps.diffResps['scfile'] = SC
my_apps.diffResps['srcmdl'] = xmlmodelname
my_apps.diffResps['irfs'] = 'CALDB'
my_apps.diffResps.run()
xmlfitname = '%s_fit1.xml' % NAME
obs = UnbinnedObs(roiname, SC, expMap='expMap.fits', expCube='expCube.fits', irfs='CALDB')
# like1 = UnbinnedAnalysis(obs,xmlmodelname,optimizer='MINUIT')
like1 = UnbinnedAnalysis(obs, xmlmodelname, optimizer='NewMinuit')
likeobj = pyLike.NewMinuit(like1.logLike)
like1.fit(verbosity=0, optObject=likeobj)
print likeobj.getRetCode()
sourceDetails = {}
for source in like1.sourceNames():
sourceDetails[source] = like1.Ts(source)
for source, TS in sourceDetails.iteritems():
if (TS < 2):
print "Deleting...", source, " TS = ", TS
like1.deleteSource(source)
like1.fit(verbosity=0, optObject=likeobj)
print "0 is converged", likeobj.getRetCode()
like1.logLike.writeXml(xmlfitname)
numl = search(NAME, xmlfitname)
numlg = str(numl + 3)
os.system("sed '" + numlg + "," + numlg + " s/free=\"1\"/free=\"0\"/' " + xmlfitname + " > xml_sed.xml ")
inputs = likeInput(like1, NAME, model="xml_sed.xml", nbins=6, phCorr=1.0)
inputs.plotBins()
inputs.fullFit(CoVar=True)
sed = likeSED(inputs)
sed.getECent()
sed.fitBands()
sed.Plot()
result = like1.model[NAME]
TS = like1.Ts(NAME)
flux = like1.flux(NAME, emin=100)
gamma = like1.model[NAME].funcs['Spectrum'].getParam('Index').value()
cov_gg = like1.model[NAME].funcs['Spectrum'].getParam('Index').error()
# cov_II = like1.model[NAME].funcs['Spectrum'].getParam('Integral').error()
flux_err = like1.fluxError(NAME, emin=100)
like1.plot()
fitsedname = '%s_6bins_likeSEDout.fits' % NAME
sedtool(fitsedname)
print NAME, " TS=", TS
print result
if float(xml) == 1:
xmlmodelname = '%s_model.xml' % NAME
xmlfitname = '%s_fit1.xml' % NAME
obs = UnbinnedObs(roiname, SC, expMap='expMap.fits', expCube='expCube.fits', irfs='CALDB')
# like1 = UnbinnedAnalysis(obs,xmlmodelname,optimizer='MINUIT')
like1 = UnbinnedAnalysis(obs, xmlmodelname, optimizer='NewMinuit')
likeobj = pyLike.NewMinuit(like1.logLike)
like1.fit(verbosity=0, optObject=likeobj)
print likeobj.getRetCode()
sourceDetails = {}
for source in like1.sourceNames():
sourceDetails[source] = like1.Ts(source)
for source, TS in sourceDetails.iteritems():
if (TS < 2):
print "Deleting...", source, " TS = ", TS
like1.deleteSource(source)
like1.fit(verbosity=0, optObject=likeobj)
print "0 is converged", likeobj.getRetCode()
like1.logLike.writeXml(xmlfitname)
numl = search(NAME, xmlfitname)
numlg = str(numl + 3)
os.system("sed '" + numlg + "," + numlg + " s/free=\"1\"/free=\"0\"/' " + xmlfitname + " > xml_sed.xml ")
inputs = likeInput(like1, NAME, model="xml_sed.xml", nbins=6, phCorr=1.0)
inputs.plotBins()
inputs.fullFit(CoVar=True)
sed = likeSED(inputs)
sed.getECent()
sed.fitBands()
sed.Plot()
result = like1.model[NAME]
TS = like1.Ts(NAME)
flux = like1.flux(NAME, emin=100)
gamma = like1.model[NAME].funcs['Spectrum'].getParam('Index').value()
cov_gg = like1.model[NAME].funcs['Spectrum'].getParam('Index').error()
# cov_II = like1.model[NAME].funcs['Spectrum'].getParam('Integral').error()
flux_err = like1.fluxError(NAME, emin=100)
like1.plot()
fitsedname = '%s_6bins_likeSEDout.fits' % NAME
sedtool(fitsedname)
print NAME, " TS=", TS
print result
class quickLike:
""" This is the base class. A usual likelihood analysis will
consists of running the following functions (assuming you have a
configuration file):
* qL = quickLike('MySource', True)
* qL.makeObs()
* qL.initDRM()
* qL.fitDRM()
* qL.initMIN()
* qL.fitMIN()
This will set up all of the objects needed for the analysis and do an
initial fit with one of the DRM optimizers. It'll save these results
and use them for the second fit with one of the Minuit optimizers.
If you do not have a configuration file, you'll need to input all
of the options for this module when you create the quickLike
object (see the various options below). You can create a
configuration file by executing writeConfig().
* qL.writeConfig()
This module will catch any failures from the optimizers and will
report them to the user. There are a few functions that are useful to
use in this case:"""
def __init__(self,
base = 'MySource',
configFile = False,
likelihoodConfig = {"model" : "MySource_model.xml",
"sourcename" : "Source Name",
"drmtol" : 0.1,
"mintol" : 1e-4},
commonConfig = {"base" : 'MySource',
"eventclass" : 2,
"binned" : False,
"irfs" : "P7SOURCE_V6",
"verbosity" : 0}):
commonConfig['base'] = base
self.logger = initLogger(base, 'quickLike')
if(configFile):
try:
commonConfigRead,analysisConfigRead,likelihoodConfigRead,plotConfigRead = readConfig(self.logger,base)
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
try:
commonConfig = checkConfig(self.logger,commonConfig,commonConfigRead)
except(KeyError):
return
try:
likelihoodConfig = checkConfig(self.logger,likelihoodConfig,likelihoodConfigRead)
except(KeyError):
return
self.commonConf = commonConfig
self.likelihoodConf = likelihoodConfig
self.ret = re.compile('\n')
self.fitbit = False
self.Print()
def writeConfig(self):
"""Writes all of the initialization variables to the config
file called <basename>.cfg"""
writeConfig(quickLogger=self.logger,
commonDictionary=self.commonConf,
likelihoodDictionary=self.likelihoodConf)
def Print(self):
"""Prints out information about the various objects to the
terminal and to the log file."""
logString = "Created quickLike object: "
for variable, value in self.commonConf.iteritems():
logString += variable+"="+str(value)+","
for variable, value in self.likelihoodConf.iteritems():
logString += variable+"="+str(value)+","
self.logger.info(logString)
def makeObs(self):
"""Creates either a binned or unbinned observation object for
use in the likelihood analysis. This function checks for all
of the needed files first. If you do not have a needed file,
see the quickAnalysis module for creation. This function
should be run before any of the init or fit functions."""
if(self.commonConf['binned']):
try:
checkForFiles(self.logger,[self.commonConf['base']+'_srcMaps.fits',
self.commonConf['base']+'_ltcube.fits',
self.commonConf['base']+'_BinnedExpMap.fits'])
self.obs = BinnedObs(srcMaps=self.commonConf['base']+'_srcMaps.fits',
expCube=self.commonConf['base']+'_ltcube.fits',
binnedExpMap=self.commonConf['base']+'_BinnedExpMap.fits',
irfs=self.commonConf['irfs'])
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
else:
try:
checkForFiles(self.logger,[self.commonConf['base']+'_filtered_gti.fits',
self.commonConf['base']+'_SC.fits',
self.commonConf['base']+'_expMap.fits',
self.commonConf['base']+'_ltcube.fits'])
self.obs = UnbinnedObs(self.commonConf['base']+'_filtered_gti.fits',
self.commonConf['base']+'_SC.fits',
expMap=self.commonConf['base']+'_expMap.fits',
expCube=self.commonConf['base']+'_ltcube.fits',
irfs=self.commonConf['irfs'])
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
self.logger.info(self.ret.subn(', ',str(self.obs))[0])
def initDRM(self):
"""Initializes the DRM optimizer (either binned or unbinned).
This is usually the second function that you run when using
this module. You need to run makeObs before you run this
function. If it hasn't been run, this function will exit."""
try:
self.obs
except AttributeError:
self.logger.critical("Obs object does not exist. Create it first with the makeObs function")
return
try:
checkForFiles(self.logger,[self.likelihoodConf['model']])
if(self.commonConf['binned']):
self.DRM = BinnedAnalysis(self.obs,self.likelihoodConf['model'],optimizer="DRMNGB")
else:
self.DRM = UnbinnedAnalysis(self.obs,self.likelihoodConf['model'],optimizer="DRMNGB")
self.DRM.tol = float(self.likelihoodConf['drmtol'])
self.logger.info(self.ret.subn(', ',str(self.DRM))[0])
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
def initAltFit(self,opt="MINUIT"):
"""Initiallizes a minuit optimizer to use as a backup to the
DRM optimizer. This function is used internally in the fitDRM
function so you probably will never use it. You need to run
makeObs before you run this function. If it hasn't been run,
this function will exit."""
try:
self.obs
except AttributeError:
self.logger.critical("Obs object does not exist. Create it first with the makeObs function")
return
try:
checkForFiles(self.logger,[self.likelihoodConf['model']])
if(self.commonConf['binned']):
self.ALTFIT = BinnedAnalysis(self.obs,self.likelihoodConf['model'],optimizer=opt)
else:
self.ALTFIT = UnbinnedAnalysis(self.obs,self.likelihoodConf['model'],optimizer=opt)
self.ALTFIT.tol = float(self.likelihoodConf['drmtol'])
self.ALTFITobj = pyLike.Minuit(self.ALTFIT.logLike)
self.logger.info(self.ret.subn(', ',str(self.ALTFIT))[0])
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
def initMIN(self, useBadFit=False):
"""Initiallizes a New Minuit optimizer to use as a backup to
the DRM optimizer. This is usually run after you have
initially run fitDRM and created a <basename>_likeDRM.xml
model file which is used a seed for the New Minuit optimizer.
You can skip the DRM process if you like but you need to have
the proper model file (<basename>_likeDRM.xml) present in the
working directory. You need to run makeObs before you run this
function. If it hasn't been run, this function will exit. If
you want to use the non convergant fit from fitDRM, set
useBadFit to True."""
try:
self.obs
except AttributeError:
self.logger.critical("Obs object does not exist. Create it first with the makeObs function.")
return
if(useBadFit):
model = self.commonConf['base']+'_badDRMFit.xml'
else:
model = self.commonConf['base']+'_likeDRM.xml'
try:
checkForFiles(self.logger,[model])
if(self.commonConf['binned']):
self.MIN = BinnedAnalysis(self.obs,model,optimizer='NewMinuit')
else:
self.MIN = UnbinnedAnalysis(self.obs,model,optimizer='NewMinuit')
self.MIN.tol = float(self.likelihoodConf['mintol'])
self.MINobj = pyLike.NewMinuit(self.MIN.logLike)
self.logger.info(self.ret.subn(', ',str(self.MIN))[0])
except(FileNotFound):
self.logger.critical("One or more needed files do not exist")
return
def fitDRM(self):
"""Performs a DRM inital fit on your data using the
<basename>_model.xml model file. It tries an intial fit and
if that fails, tries a tighter tolerance. If that fails, it
tries a looser tolerance. If that fails, it tries to do this
initial fit with the MINUIT optimizer. If that fails, this
function bails. If the fit converges, it saves the results to
<basename>_likeDRM.xml which will be used in the NewMinuit
fit. If no fit is found, it will save the results to
<basename>_badDRMFit.xml. You can use this in the NewMinuit fit
if you use the useBadFit option in initMIN. You need to have run
initDRM before you run this function."""
try:
self.DRM
except AttributeError:
self.logger.critical("DRM object does not exist. Create it first with the initDRM function.")
return
altfit=False
try:
self.DRM.fit(verbosity=int(self.commonConf['verbosity']))
except:
self.logger.error("Initial DRM Fit Failed")
try:
self.logger.info("Trying tighter tolerance (DRMtol*0.1)")
self.DRM.tol = float(self.likelihoodConf['drmtol']) * 0.1
self.DRM.fit(verbosity= int(self.commonConf['verbosity']))
except:
self.logger.error("Second DRM Fit Failed")
try:
self.logger.info("Trying looser tolerance (drmtol*10.)")
self.DRM.tol = float(self.likelihoodConf['drmtol']) * 10.
self.DRM.fit(verbosity= int(self.commonConf['verbosity']))
except:
self.logger.error("Third DRM Fit Failed")
try:
self.logger.info("Trying alternate fit algorithm (MINUIT)")
self.initAltFit()
self.ALTFIT.fit(verbosity=int(self.commonConf['verbosity']),covar=True,optObject=self.ALTFITobj)
print self.ALTFITobj.getQuality()
altfit = True
except:
self.logger.error("Alternative fit algorithm failed, bailing")
self.logger.error(self.decodeRetCode('Minuit',self.ALTFITobj.getRetCode()))
self.ALTFIT.logLike.writeXml(self.commonConf['base']+'_badDRMFit.xml')
self.logger.info("Saved ALTFIT as "+self.commonConf['base']+"_badDRMFit.xml")
return
if(altfit):
self.logger.info("ALTFIT Fit Finished. Total TS: "+str(self.ALTFIT.logLike.value()))
self.ALTFIT.logLike.writeXml(self.commonConf['base']+'_likeDRM.xml')
self.logger.info("Saved ALTFIT as "+self.commonConf['base']+"_likeDRM.xml")
else:
self.DRM.logLike.writeXml(self.commonConf['base']+'_likeDRM.xml')
self.logger.info("DRM Fit Finished. Total TS: "+str(self.DRM.logLike.value()))
self.logger.info("Saved DRM as "+self.commonConf['base']+"_likeDRM.xml")
def fitMIN(self):
"""Does a New Minuit fit on your data based on the model
output by the fitDRM function. You need to have run initMIN
before running this function. Saves the results to
<basename>_likeMIN.xml if there is convergence. If
convergence is not found, saves the results to
<basename>_badMINFit.xml."""
try:
self.MIN
except AttributeError:
self.logger.critical("MIN object does not exist. Create it first with the initMIN function.")
return
self.MIN.fit(covar=True, optObject=self.MINobj,verbosity=int(self.commonConf['verbosity']))
self.MIN.logLike.writeXml(self.commonConf['base']+'_likeMinuit.xml')
self.logger.info("NEWMINUIT Fit Finished. Total TS: "+str(self.MIN.logLike.value()))
self.logger.info("NEWMINUIT Fit Status: "+str(self.MINobj.getRetCode()))
self.logger.info("NEWMINUIT fit Distance: "+str(self.MINobj.getDistance()))
self.fitbit = True
if(self.MINobj.getRetCode() > 0):
self.logger.error("NEWMINUIT DID NOT CONVERGE!!!")
self.logger.error("The fit failed the following tests: "+self.decodeRetCode('NewMinuit',self.MINobj.getRetCode()))
self.MIN.fit(covar=True, optObject=self.MINobj,verbosity=int(self.commonConf['verbosity']))
self.MIN.logLike.writeXml(self.commonConf['base']+'_badMINFit.xml')
def printSource(self,source,Emin=100,Emax=300000):
"""Prints various details for a source in your model."""
try:
self.MIN
except AttributeError:
self.logger.critical("MIN object does not exist. "+\
"Create it first with the initMIN function and then fit it with the fitMIN function.")
return
if(not self.fitbit):
self.logger.warn("Fit isn't current, these values might not be correct. Fun fitMIN first.")
logString = source
TS = self.MIN.Ts(source)
print "TS: ",TS
logString += " TS: " + str(TS)
NPred = self.MIN.NpredValue(source)
print "Npred: ",NPred
logString += " NPred: " + str(NPred)
flux = self.MIN.flux(source,emin=Emin,emax=Emax)
print "Flux: ",flux
logString += "Flux: "+str(flux)
if(self.fitbit):
fluxErr = self.MIN.fluxError(source,emin=Emin,emax=Emax)
print "Flux Error: ",fluxErr
logString += "Flux Error: "+str(fluxErr)
for paramName in self.MIN.model[source].funcs['Spectrum'].paramNames:
paramValue = self.MIN.model[source].funcs['Spectrum'].getParam(paramName).value()
print paramName,": ",paramValue
logString += paramName + ": " + str(paramValue) + " "
self.logger.info(logString)
def customERange(self,Emin,Emax):
"""Sets a smaller energy range for the fitting of both the DRM
and MIN optimization steps."""
try:
self.DRM
except AttributeError:
self.logger.warn("DRM object doesn't exist. Energy range not modified.")
else:
self.DRM.setEnergyRange(Emin,Emax)
self.logger.info("Set energy range for DRM to "+str(self.DRM.emin)+","+str(self.DRM.emax))
try:
self.MIN
except AttributeError:
self.logger.warn("MIN object doesn't exist. Energy range not modified.")
else:
self.MIN.setEnergyRange(Emin,Emax)
self.logger.info("Set energy range for MIN to "+str(self.MIN.emin)+","+str(self.MIN.emax))
def calcUpper(self,source,Emin=100,Emax=300000):
"""Calculates an upper limit for a source in your model."""
self.ul = UpperLimits(self.MIN)
self.ul[source].compute(emin=Emin,emax=Emax)
print self.ul[source].results
self.logger.info(source+" UL: "+str(self.ul[source].results[0]))
def removeWeak(self,mySource = '',tslimit=0,distlimit=0,RemoveFree=False,RemoveFixed=False):
"""This function has two main uses: it will print out details
on all of the sources in your model and it will remove sources
according to different requirements. If you just want to
print out details, execute it this way:
<obj>.removeWeak(<my_source>)
Where <obj> is the quickLike object you're using here and
<my_source> is the name of your source of interest. You can
then remove some of these sources from the model if you like.
For example, if you want to remove all of the fixed sources
with TS values less than 1, execute it this way:
<obj>.removeWeak(<my_source>,tslimit=1,RemoveFixed=True)
You can mix and match any of the options. You could remove
all sources (fixed and free) that are below a TS value of 3
and are 10 degrees from your source of interest by executing:
<obj>.removeWeak(<my_source>,tslimit=3,distlimit=10,RemoveFree=True,RemoveFixed=True)"""
try:
self.MIN
except AttributeError:
self.logger.critical("MIN object does not exist. "+\
"Create it first with the initMIN function and then fit it with the fitMIN function.")
return
if(not self.fitbit):
self.logger.warn("Fit isn't current, these values might not be correct. Run fitMIN first.")
if(mySource == ''):
mySource = self.likelihoodConf['sourcename']
for name in self.MIN.sourceNames():
remove = False
distance = 0
sourceTS = self.MIN.Ts(name)
if(self.MIN.model[name].src.getType() == 'Point'):
distance = self.MIN._separation(self.MIN.model[mySource].src,self.MIN.model[name].src)
if(self.MIN.freePars(name).size() > 0):
indexFree = "Free"
if( (sourceTS < tslimit) and (distance > distlimit) and RemoveFree ):
remove = True
else:
indexFree = "Fixed"
if( (sourceTS < tslimit) and (distance > distlimit) and RemoveFixed ):
remove = True
if( remove ):
self.logger.info("Removing "+name+", TS: "+str(sourceTS)+", Frozen?: "+str(indexFree)+", Distance: "+str(distance))
self.MIN.deleteSource(name)
else:
self.logger.info("Retaining "+name+", TS: "+str(sourceTS)+", Frozen?: "+str(indexFree)+", Distance: "+str(distance))
def paramsAtLimit(self, limit = 0.1):
"""This function will print out any sources whoes parameters
are close to their limits. You could use this to find sources
that are having issues being fit. This function is useful
when you're having trouble getting convergence from the New
Minuit fit routine. The limit is in percentage difference of a
bound. If one of the bounds is zero it uses the value of the
parameter to check for closeness (absolute instead of percent
differenct). The default is 0.1 (1%) difference for a measure
of closeness."""
try:
self.MIN
except AttributeError:
self.logger.critical("MIN object does not exist. "+\
"Create it first with the initMIN function and then fit it with the fitMIN function.")
return
if(not self.fitbit):
self.logger.warn("Fit isn't current, these values might not be correct. Run fitMIN first.")
for src in self.MIN.sourceNames():
for name in self.MIN.model[src].funcs['Spectrum'].paramNames:
bounds = self.MIN.model[src].funcs['Spectrum'].getParam(name).getBounds()
value = self.MIN.model[src].funcs['Spectrum'].getParam(name).value()
try:
distToLower = abs((value - bounds[0])/bounds[0])
except ZeroDivisionError:
distToLower = abs(value)
try:
distToUpper = abs((value - bounds[1])/bounds[1])
except ZeroDivisionError:
distToUpper = abs(value)
if( distToLower < limit ):
self.logger.error("The "+name+" ("+str(value)+") of "+src+" is close ("\
+str(distToLower)+") to its lower limit ("+str(bounds[0])+")")
if( distToUpper < limit):
self.logger.error("The "+name+" ("+str(value)+") of "+src+" is close ("\
+str(distToUpper)+") to its upper limit ("+str(bounds[1])+")")
def decodeRetCode(self, optimizer, retCode):
"""Decodes the return codes from the Minuit and New Minuit fit
functions. Used in the fitting functions in this module.
You'll probably never use this function."""
if(optimizer == 'NewMinuit'):
retCode -= 100
failure = ""
if(retCode & 1):
failure += " IsAboveMaxEdm"
if(retCode & 2):
failure += " HasCovariance"
if(retCode & 4):
failure += " HesseFailed"
if(retCode & 8):
failure += " HasMadePosDefCovar"
if(retCode & 16):
failure += " HasPosDefCovar"
if(retCode & 32):
failure += " HasAccurateCovar"
if(retCode & 64):
failure += " HasValidCovariance"
if(retCode & 128):
failure += " HasValidParameters"
if(retCode & 256):
failure += " IsValid"
return failure
if(optimizer == 'Minuit'):
if(retCode == 0):
failure = "Error matrix not calculated at all"
if(retCode == 1):
failure = "Diagonal approximation only, not accurate"
if(retCode == 2):
failure = "Full matrix, but forced positive-definite (i.e. not accurate)"
if(retCode == 3):
failure = "Full accurate covariance matrix (After MIGRAD, this is the indication of normal convergence.)"
return failure
def runLike(likeIn,ecent,ftol,tslim,ttype,opt,rescaleAll,lastbinUL,wx):
pts=[]
errs=[]
tsPts=[]
gamma=[]
Src=likeIn.source
NBins=likeIn.NBins
IRFs=likeIn.IRFs
expCube=likeIn.expCube
ft2=likeIn.ft2
bandModel=likeIn.bandModel
nbins=likeIn.nbins
phCorr=likeIn.phCorr
flux=likeIn.ubAn.flux
for i in range(0,nbins): #first, set up for a likelihood run
print ' -Runnng Likelihood for band%i-' %i
ev='%s_%ibins_band%i.fits' %(Src.replace(' ','_'),NBins,i)
em='%s_%ibins_band%i_%s_em.fits' %(Src.replace(' ','_'),NBins,i,IRFs)
band_obs=UnbinnedObs((ev),ft2,irfs=IRFs,expMap=em,expCube=expCube)
band_like=UnbinnedAnalysis(band_obs,bandModel,opt)
band_like.setFitTolType(ttype)
stype=band_like.model.srcs[Src].spectrum().genericName()
emin,emax=band_like.observation.roiCuts().getEnergyCuts()
if phCorr!=1:
for src in band_like.sourceNames():
par=band_like.normPar(src)
par.setValue(par.getValue()*phCorr)
#then set the scale factor to the center of the energy band, make sure it's frozen, and get index for prefactor while you're at it
Ts=band_like.Ts
freeze=band_like.freeze
fit=band_like.fit
DO=1
if stype=='PowerLaw':
scale=getParamIndx(band_like,Src,'Scale') #this is where you have to use PowerLaw, for PowerLaw2 these parameters don't exist and this will cause problems
pref=getParamIndx(band_like,Src,'Prefactor')
freeze(scale)
band_like[scale].setBounds(20,5e5)
band_like[scale].setScale(1)
band_like[scale]=1000.*ecent[i] #put center energies in units of GeV but xml files use MeV
#multiplier=band_like[pref].getScale() #need to get the scale of the prefactor so the values will not be too large
try:
logFlux=log10(flux(Src,emin=emin,emax=emax)/(emax-emin))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
print newScale
band_like[pref].setScale(10.**newScale)
multiplier=10.**newScale
#cycle through the other point sources and adjust parameters of free point sources with PowerLaw2 models
for src in band_like.sourceNames():
spec=band_like[src].funcs['Spectrum']
par=spec.normPar()
if par.isFree()==True and band_like.model.srcs[src].spectrum().genericName()=='PowerLaw2':
HIGH=getParamIndx(band_like,src,'UpperLimit')
LOW=getParamIndx(band_like,src,'LowerLimit')
band_like[HIGH].setBounds(20,5e5) #just in case, make sure no out of range error gets thrown
band_like[LOW].setBounds(20,5e5)
band_like[HIGH].setScale(1.) #just in case, make sure scale is MeV
band_like[LOW].setScale(1.)
band_like[HIGH]=emax
band_like[LOW]=emin
freeze(HIGH) #just in case, make sure these aren't fit values
freeze(LOW)
if rescaleAll==True:
try:
logFlux=log10(flux(src,emin=emin,emax=emax))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
par.setScale(10**newScale)
if rescaleAll==True and src!=Src and par.isFree()==True and band_like.model.srcs[src].spectrum().genericName()=='PowerLaw':
try:
logFlux=log10(flux(src,emin=emin,emax=emax)/(emax-emin))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
par.setScale(10**newScale)
try:
fit(tol=1,verbosity=0,optimizer=opt)
fail=0
except:
try:
fit(tol=1*10,verbosity=0,optimizer=opt)
fail=0
except:
try:
fit(tol=1./10,verbosity=0,optimizer=opt)
fail=0
except:
print "Fit with optimizer %s with tolerances ~1 to look for negative or zero TS sources failed, if error bars are unrealistically small you may need to redo the fit for energy band %i manually" %(opt,i)
fail=1
if fail==0:
for src in band_like.sourceNames():
if src!=Src:
par=band_like.normPar(src)
if band_like[src].type=='PointSource' and Ts(src)<=0 and par.isFree()==True:
band_like.deleteSource(src)
print " -Removing %s from the model" %src
scale=getParamIndx(band_like,Src,'Scale')
pref=getParamIndx(band_like,Src,'Prefactor')
#do the actual fit
try:
fit(tol=ftol,verbosity=0)
except:
try:
print 'Trying lower tolerance of %s for band%i.' %(ftol/10,i)
fit(tol=ftol/10,verbosity=0)
except:
try:
print 'Trying higher tolerance of %s for band%i.' %(ftol*10,i)
fit(tol=ftol*10,verbosity=0)
except:
print 'No convergence for band%i, skipping.' %i
pts+=[0]
errs+=[0]
tsPts+=[0]
pass
#get the prefactor, error, and ts values
val=band_like[pref].value()
err=band_like[pref].error()
TS=Ts(Src)
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)): #calculate 95% upperlimit if source TS<tslim, 25 by default (corresponds to ~5sigma)
try:
freeze(pref)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
except:
try:
print ' Tyring higher tolerance of %s for band %i to get good starting point for upper limit calculations.' %(ftol*10,i)
band_like[pref].setFree(1)
fit(tol=ftol*10,verbosity=0)
TS=Ts(Src)
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)):
freeze(pref)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
else:
val=band_like[pref].value()
err=band_like[pref].error()
except:
try:
print ' Tyring lower tolerance of %s for band %i to get good starting point for upper limit calculations.' %(ftol/10,i)
band_like[pref].setFree(1)
fit(tol=ftol/10,verbosity=0)
TS=Ts(Src)
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)):
freeze(pref)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
else:
val=band_like[pref].value()
err=band_like[pref].error()
except:
err=0
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, TS<%s' %tslim,'but UpperLimits computation failed.'
print ' Quoting best fit value with zero error.'
tsPts+=[TS]
pts+=[val*multiplier/phCorr]
errs+=[err*multiplier/phCorr]
elif stype=='PowerLaw2':
Upper=getParamIndx(band_like,Src,'UpperLimit')
Lower=getParamIndx(band_like,Src,'LowerLimit')
Integral=getParamIndx(band_like,Src,'Integral')
Index=getParamIndx(band_like,Src,'Index')
freeze(Upper)
freeze(Lower)
band_like[Upper].setBounds(20,5e5)
band_like[Lower].setBounds(20,5e5)
band_like[Lower].setScale(1)
band_like[Lower].setScale(1)
band_like[Upper]=emax
band_like[Lower]=emin
#multiplier=band_like[Integral].getScale()
try:
logFlux=log10(flux(Src,emin=emin,emax=emax))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
band_like[Integral].setScale(10**newScale)
multiplier=10**newScale
indxMult=band_like[Index].getScale()
for src in band_like.sourceNames():
if src!=Src:
spec=band_like[src].funcs['Spectrum']
par=spec.normPar()
if par.isFree()==True and band_like.model.srcs[src].spectrum().genericName()=='PowerLaw2':
HIGH=getParamIndx(band_like,src,'UpperLimit')
LOW=getParamIndx(band_like,src,'LowerLimit')
band_like[HIGH].setBounds(20,5e5) #just in case, make sure no out of range error gets thrown
band_like[LOW].setBounds(20,5e5)
band_like[HIGH].setScale(1.) #just in case, make sure scale is MeV
band_like[LOW].setScale(1.)
band_like[HIGH]=emax
band_like[LOW]=emin
freeze(HIGH) #just in case, make sure these aren't fit values
freeze(LOW)
if rescaleAll==True:
try:
logFlux=log10(flux(src,emin=emin,emax=emax))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
par.setScale(10**newScale)
if rescaleAll==True and src!=Src and par.isFree()==True and band_like.model.srcs[src].spectrum().genericName()=='PowerLaw':
try:
logFlux=log10(flux(src,emin=emin,emax=emax)/(emax-emin))
except:
logFlux=-14
newScale=max(int(floor(logFlux)),-14)
par.setScale(10**newScale)
try:
fit(tol=1,verbosity=0,optimizer=opt)
fail=0
except:
try:
fit(tol=1*10,verbosity=0,optimizer=opt)
fail=0
except:
try:
fit(tol=1./10,verbosity=0,optimizer=opt)
fail=0
except:
print "Fit with optimizer %s with tolerances ~1 to look for negative or zero TS sources failed, if error bars are unrealistically small you may need to redo the fit for energy band %i manually" %(opt,i)
fail=1
if not fail:
for src in band_like.sourceNames():
if src!=Src:
par=band_like.normPar(src)
if band_like[src].type=='PointSource' and Ts(src)<=0 and par.isFree()==True:
band_like.deleteSource(src)
print " -Removing %s from the model" %src
Upper=getParamIndx(band_like,Src,'UpperLimit')
Lower=getParamIndx(band_like,Src,'LowerLimit')
Integral=getParamIndx(band_like,Src,'Integral')
Index=getParamIndx(band_like,Src,'Index')
try:
fit(tol=ftol,verbosity=0)
except:
try:
print 'Trying lower tolerance of %s for band%i.' %(ftol/10,i)
fit(tol=ftol/10,verbosity=0)
except:
try:
print 'Trying higher tolerance of %s for band%i.' %(ftol*10,i)
fit(tol=ftol*10,verbosity=0)
except:
print 'No convergence for band%i, skipping.' %i
pts+=[0]
errs+=[0]
tsPts+=[0]
gamma+=[0]
DO=0
if DO:
val=band_like[Integral].value()
err=band_like[Integral].error()
TS=Ts(Src)
gam=band_like[Index].value()*indxMult*-1.
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)): #calculate 95% upperlimit if source TS<9, i.e. less than 3 sigma detection in each energy band
try:
freeze(Integral)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
#need to redo the fit with band_like set to Upper Limit value to get correct spectral index for that value
band_like[Integral]=val
freeze(Integral)
fit(tol=ftol,verbosity=0)
gam=band_like[Index].value()*indxMult*-1.
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
except:
try:
print ' Tyring higher tolerance of %s for band %i to get good starting point for upper limit calculations.' %(ftol*10,i)
band_like[Integral].setFree(1)
fit(tol=ftol*10,verbosity=0)
TS=Ts(Src)
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)):
freeze(Integral)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
band_like[Integral]=val
freeze(Integral)
fit(tol=ftol*10,verbosity=0)
gam=band_like[Index].value()*indxMult*-1.
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
else:
val=band_like[Integral].value()
err=band_like[Integral].error()
gam=band_like[Index].value()*indxMult*-1.
except:
try:
print ' Tyring lower tolerance of %s for band %i to get good starting point for upper limit calculations.' %(ftol/10,i)
band_like[Integral].setFree(1)
fit(tol=ftol/10,verbosity=0)
TS=Ts(Src)
if(TS<tslim or (i==(nbins-1) and lastbinUL==True)):
freeze(Integral)
ul=UpperLimits(band_like)
UL=ul[Src].compute(emin=emin,emax=emax)
val=UL[1]
err=0
band_like[Integral]=val
freeze(Integral)
fit(tol=ftol/10,verbosity=0)
gam=band_like[Index].value()*indxMult*-1.
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, quoting 95% upper limit on flux.'
else:
val=band_like[Integral].value()
err=band_like[Integral].error()
gam=band_like[Index].value()*indxMult*-1.
except:
err=0
print ' NOTE: Band%i,' %i,'with center energy',ecent[i],'GeV, TS<%s' %tslim,'but UpperLimits computation failed.'
print ' Quoting best fit value with zero error.'
tsPts+=[TS]
pts+=[val*multiplier/phCorr]
errs+=[err*multiplier/phCorr]
gamma+=[gam]
else:
print '%s needs to have PowerLaw or PowerLaw2 spectral model, not %s' %(Src,stype)
print 'exiting without running likelihood in the energy bands'
return None,None,None,None
if wx:
band_like.writeXml('%s_%ibins_band%i_fitmodel.xml'%(Src.replace(' ','_'),NBins,i))
del band_like
del band_obs
return pts,errs,tsPts,gamma
