def GenAnalysisObjects(config, verbose=1, xmlfile=""):
#check is the summed likelihood method should be used and get the
#Analysis objects (observation and (Un)BinnedAnalysis objects)
SummedLike = config['Spectrum']['SummedLike']
folder = config['out']
if SummedLike == 'yes':
# Create two obs instances
sys.exit("not yet working")
FitRunnerfront = Analysis(folder, config, tag="FRONT", verbose=verbose)
FitRunnerback = Analysis(folder, config, tag="BACK", verbose=verbose)
if not (xmlfile == ""):
FitRunnerfront.obs.xmlfile = xmlfile
FitRunnerback.obs.xmlfile = xmlfile
FitB = FitRunnerback.CreateLikeObject()
FitF = FitRunnerfront.CreateLikeObject()
import SummedLikelihood
Fit = SummedLikelihood.SummedLikelihood()
Fit.addComponent(FitB)
Fit.addComponent(FitF)
FitRunner = FitRunnerback
else:
# Create one obs instance
FitRunner = Analysis(folder, config, tag="", verbose=verbose)
if not (xmlfile == ""):
FitRunner.obs.xmlfile = xmlfile
Fit = FitRunner.CreateLikeObject()
return FitRunner, Fit
def _GenFit(self):
try:
del self.Fit
except:
pass
if self.config['Spectrum']['SummedLike'] == 'yes':
Obs1 = Observation(self.folder, self.config, tag="FRONT")
Obs2 = Observation(self.folder, self.config, tag="BACK")
self.FitRunnerfront = FitMaker(Obs1, self.config)
self.FitRunnerback = FitMaker(Obs2, self.config)
self.FitRunnerfront.CreateLikeObject()
self.FitRunnerback.CreateLikeObject()
self.Fit = SummedLikelihood.SummedLikelihood()
else:
Obs = Observation(self.folder, self.config, tag="")
self.FitRunner = FitMaker(Obs, self.config) ##Class
#self.Fit = self.FitRunner.CreateLikeObject()
self.Fit = SummedLikelihood.SummedLikelihood()
self.Fit.addComponent(self.FitRunner.CreateLikeObject())
def __init__(self, config=None, **kwargs):
super(AnalysisManager, self).__init__()
self.update_default_config(SelectorTask, group='select')
self.configure(config, **kwargs)
import pprint
pprint.pprint(self.config)
self._like = SummedLikelihood()
def GenAnalysisObjects(config, verbose=1, xmlfile=""):
mes = Loggin.Message()
#check is the summed likelihood method should be used and get the
#Analysis objects (observation and (Un)BinnedAnalysis objects)
folder = config['out']
# If there are no xml files, create it and print a warning
if len(glob.glob(config['file']['xml'].replace('.xml', '*.xml'))) == 0:
mes.warning("Xml not found, creating one for the given config %s" %
config['file']['xml'])
XmlMaker(config)
Fit = SummedLikelihood.SummedLikelihood()
if hasKey(config, 'ComponentAnalysis') == True:
# Create one obs instance for each component
configs = [None] * 4
Fits = [None] * 4
Analyses = [None] * 4
if isKey(config['ComponentAnalysis'], 'FrontBack') == 'yes':
from enrico.data import fermievtypes
mes.info("Breaking the analysis in Front/Back events")
# Set Summed Likelihood to True
oldxml = config['file']['xml']
for k, TYPE in enumerate(["FRONT", "BACK"]):
configs[k] = ConfigObj(config)
configs[k]['event']['evtype'] = fermievtypes[TYPE]
try:
Analyses[k] = Analysis(folder, configs[k], \
configgeneric=config,\
tag=TYPE, verbose = verbose)
if not (xmlfile == ""): Analyses[k].obs.xmlfile = xmlfile
Fits[k] = Analyses[k].CreateLikeObject()
Fit.addComponent(Fits[k])
except RuntimeError, e:
if 'RuntimeError: gtltcube execution failed' in str(e):
mes.warning("Event type %s is empty! Error is %s" %
(TYPE, str(e)))
FitRunner = Analyses[0]
def __init__(self, config):
self.config = config
self.folder = self.config['out']
os.system("mkdir -p "+self.folder+"/TestModel")
convtype = self.config['analysis']['convtype']
if self.config['Spectrum']['SummedLike'] == 'yes':
Obs1 = Observation(self.folder, self.config, convtype=0, tag="FRONT")
Obs2 = Observation(self.folder, self.config, convtype=1, tag="BACK")
FitRunnerfront = FitMaker(Obs1, self.config)
FitRunnerback = FitMaker(Obs2, self.config)
FitRunnerfront.CreateLikeObject()
FitRunnerback.CreateLikeObject()
self.Fit = SummedLikelihood.SummedLikelihood()
else:
Obs = Observation(self.folder, self.config, convtype, tag="")
FitRunner = FitMaker(Obs, self.config)##Class
self.Fit = FitRunner.CreateLikeObject()
# Store the results in a dictionnary
self.Results = {}
self.Results["PowerLaw"] = 0
self.Results["LogParabola"] = 0
self.Results["PLSuperExpCutoff"] = 0
def processAllObs(self, fix_shape=True, delete_below_ts=None,
ul_flux_dflux=0, ul_chi2_ts=None, ul_bayes_ts=4.0,
ul_cl=0.95, verbosity=0, emin=0, emax=0,
interim_save_filename=None):
self.logger.info("Processing all observations.")
for f in self.obsfiles:
lc = dict()
lc['config'] = dict()
lc['config']['fix_shape'] = fix_shape
lc['config']['delete_below_ts'] = delete_below_ts
lc['config']['ul_flux_dflux'] = ul_flux_dflux
lc['config']['ul_chi2_ts'] = ul_chi2_ts
lc['config']['ul_bayes_ts'] = ul_bayes_ts
lc['config']['ul_cl'] = ul_cl
lc['config']['emin'] = emin
lc['config']['emax'] = emax
lc['config']['files'] = f
#lc['config']['argv'] = sys.argv
lc['e_min'] = emin;
lc['e_max'] = emax;
if type(f) != list:
[ obs, like ] = self.loadObs(f,verbosity)
lc['t_min'] = obs.roiCuts().minTime()
lc['t_max'] = obs.roiCuts().maxTime()
if (emin == 0 or emax == 0):
lc['e_min'] = obs.roiCuts().getEnergyCuts()[0];
lc['e_max'] = obs.roiCuts().getEnergyCuts()[1];
else:
lc['t_min'] = None
lc['t_max'] = None
like = SL.SummedLikelihood(self.optimizer)
for ff in f:
[ obs, like1 ] = self.loadObs(ff,verbosity)
tmin = obs.roiCuts().minTime()
tmax = obs.roiCuts().maxTime()
if lc['t_min'] == None or tmin<lc['t_min']:
lc['t_min'] = tmin
if lc['t_max'] == None or tmax>lc['t_max']:
lc['t_max'] = tmax
if (lc['e_min'] == 0 or lc['e_max'] == 0):
ecuts = obs.roiCuts().getEnergyCuts()
lc['e_min'] = ecuts[0]
lc['e_max'] = ecuts[1]
elif (emin == 0 or emax == 0):
ecuts = obs.roiCuts().getEnergyCuts()
lc['e_min'] = max(lc['e_min'], ecuts[0])
lc['e_max'] = min(lc['e_max'], ecuts[1])
like.addComponent(like1)
emin = lc['e_min']
emax = lc['e_max']
like.tol = like.tol*0.01;
if verbosity > 1:
print '- Time:',lc['t_min'],'to',lc['t_max']
src = like[self.likelihoodConf['sourcename']]
if src == None:
raise NameError("No source \""+self.likelihoodConf['sourcename']+"\" in model "+
self.model)
srcfreepar=like.freePars(self.likelihoodConf['sourcename'])
srcnormpar=like.normPar(self.likelihoodConf['sourcename'])
if len(srcfreepar)>0:
like.setFreeFlag(self.likelihoodConf['sourcename'], srcfreepar, 0)
like.syncSrcParams(self.likelihoodConf['sourcename'])
meanvalue = srcnormpar.getValue()
meanerror = srcnormpar.error()
if meanerror == 0:
self.logger.critical("The error on the normalization for your source is 0! You need to do a global fit first (with quickLike) and provide the final XML file (<basename>_likeMinuit.xml) with errors included before you run compute.")
return
lc['original']=dict()
lc['original']['normpar_init_value'] = meanvalue
lc['original']['normpar_name'] = srcnormpar.getName()
lc['original']['nfree'] = len(like.freePars(self.likelihoodConf['sourcename']))
lc['original']['flux'] = like[self.likelihoodConf['sourcename']].flux(emin, emax)
lc['original']['logL'] = like.logLike.value()
if verbosity > 1:
print '- Original log Like:',lc['original']['logL']
if fix_shape:
if verbosity > 1:
print '- Fixing spectral shape parameters'
sync_name = ""
for p in like.params():
if sync_name != "" and sync_name != p.srcName:
like.syncSrcParams(sync_name)
sync_name = ""
if(p.isFree() and p.srcName!=self.likelihoodConf['sourcename'] and
p.getName()!=like.normPar(p.srcName).getName()):
if verbosity > 2:
print '-- '+p.srcName+'.'+p.getName()
p.setFree(False)
sync_name = p.srcName
if sync_name != "" and sync_name != p.srcName:
like.syncSrcParams(sync_name)
sync_name = ""
# ----------------------------- FIT 1 -----------------------------
if verbosity > 1:
print '- Fit 1 - All parameters of',self.likelihoodConf['sourcename'],'fixed'
like.fit(max(verbosity-3, 0))
lc['allfixed'] = dict()
lc['allfixed']['logL'] = like.logLike.value()
fitstat = like.optObject.getRetCode()
if verbosity > 1 and fitstat != 0:
print "- Fit 1 - Minimizer returned with code: ", fitstat
lc['allfixed']['fitstat'] = fitstat
if verbosity > 1:
print '- Fit 1 - log Like:',lc['allfixed']['logL']
if delete_below_ts:
frozensrc = []
if verbosity > 1:
print '- Deleting point sources with TS<'+str(delete_below_ts)
deletesrc = []
for s in like.sourceNames():
freepars = like.freePars(s)
if(s!=self.likelihoodConf['sourcename'] and like[s].src.getType() == 'Point'
and len(freepars)>0):
ts = like.Ts(s)
if ts<delete_below_ts:
deletesrc.append(s)
if verbosity > 2:
print '--',s,'(TS='+str(ts)+')'
if deletesrc:
for s in deletesrc:
like.deleteSource(s)
if verbosity > 1:
print '- Fit 1 - refitting model'
like.fit(max(verbosity-3, 0))
lc['allfixed']['fitstat_initial'] = \
lc['allfixed']['fitstat']
fitstat = like.optObject.getRetCode()
if verbosity > 1 and fitstat != 0:
print "- Fit 1 - Minimizer returned with code: ",\
fitstat
lc['allfixed']['fitstat'] = fitstat
lc['allfixed']['logL'] = like.logLike.value()
if verbosity > 1:
print '- Fit 1 - log Like:',lc['allfixed']['logL']
lc['allfixed']['flux']=like[self.likelihoodConf['sourcename']].flux(emin, emax)
pars = dict()
for pn in like[self.likelihoodConf['sourcename']].funcs['Spectrum'].paramNames:
p = like[self.likelihoodConf['sourcename']].funcs['Spectrum'].getParam(pn)
pars[p.getName()] = dict(name = p.getName(),
value = p.getTrueValue(),
error = p.error()*p.getScale(),
free = p.isFree())
lc['allfixed']['pars'] = pars
# ------------------ N SIGMA PROFILE LIKELIHOOD -------------------
prof_sigma = (-1,-0.5,0,0.5,1.0)
lc['profile'] = dict();
lc['profile']['sigma'] = []
lc['profile']['value'] = []
lc['profile']['logL'] = []
lc['profile']['flux'] = []
lc['profile']['fitstat'] = []
if verbosity > 1:
print '- Fit 1 - generating %d point likelihood profile'%\
len(prof_sigma)
for sigma in prof_sigma:
val = sigma*meanerror+meanvalue
if val < srcnormpar.getBounds()[0]:
val = srcnormpar.getBounds()[0]
if (lc['profile']['value']
and lc['profile']['value'][-1]==val):
continue
lc['profile']['value'].append(val)
lc['profile']['sigma'].append((val-meanvalue)/meanerror)
if(val == meanvalue):
lc['profile']['logL'].append(lc['allfixed']['logL'])
lc['profile']['flux'].append(lc['allfixed']['flux'])
else:
srcnormpar.setValue(val)
like.syncSrcParams(self.likelihoodConf['sourcename'])
like.fit(max(verbosity-3, 0))
fitstat = like.optObject.getRetCode()
if verbosity > 2 and fitstat != 0:
print "- Fit 1 - profile: Minimizer returned code: ",\
fitstat
lc['profile']['fitstat'].append(fitstat)
lc['profile']['logL'].append(like.logLike.value())
lc['profile']['flux'].append(like[self.likelihoodConf['sourcename']].\
flux(emin, emax))
if verbosity > 2:
print '- Fit 1 - profile: %+g, %f -> %f'%\
(sigma,lc['profile']['value'][-1],
lc['profile']['logL'][-1]-lc['allfixed']['logL'])
srcnormpar.setValue(meanvalue)
like.syncSrcParams(self.likelihoodConf['sourcename'])
# ----------------------------- FIT 2 -----------------------------
if verbosity > 1:
print '- Fit 2 - Normalization parameter of',\
self.likelihoodConf['sourcename'],'free'
srcnormpar.setFree(1)
like.syncSrcParams(self.likelihoodConf['sourcename'])
like.fit(max(verbosity-3, 0))
lc['normfree'] = dict()
fitstat = like.optObject.getRetCode()
if verbosity > 1 and fitstat != 0:
print "- Fit 2 - Minimizer returned with code: ", fitstat
lc['normfree']['fitstat'] = fitstat
lc['normfree']['logL'] = like.logLike.value()
lc['normfree']['ts'] = like.Ts(self.likelihoodConf['sourcename'])
lc['normfree']['flux_dflux'] = \
srcnormpar.getValue()/srcnormpar.error()
if verbosity > 1:
print '- Fit 2 - log Like:',lc['normfree']['logL'],\
'(TS='+str(lc['normfree']['ts'])+')'
lc['normfree']['nfree']=len(like.freePars(self.likelihoodConf['sourcename']))
lc['normfree']['flux']=like[self.likelihoodConf['sourcename']].flux(emin, emax)
pars = dict()
for pn in like[self.likelihoodConf['sourcename']].funcs['Spectrum'].paramNames:
p = like[self.likelihoodConf['sourcename']].funcs['Spectrum'].getParam(pn)
pars[p.getName()] = dict(name = p.getName(),
value = p.getTrueValue(),
error = p.error()*p.getScale(),
free = p.isFree())
lc['normfree']['pars'] = pars
ul_type = None
if ul_bayes_ts != None and lc['normfree']['ts'] < ul_bayes_ts:
ul_type = 'bayesian'
[ul_flux, ul_results] = \
IUL.calc_int(like,self.likelihoodConf['sourcename'],cl=ul_cl,
skip_global_opt=True,
verbosity = max(verbosity-2,0),
emin=emin, emax=emax,
poi_values = lc['profile']['value'])
elif ( ul_flux_dflux != None and \
lc['normfree']['flux_dflux'] < ul_flux_dflux ) or \
( ul_chi2_ts != None and lc['normfree']['ts'] < ul_chi2_ts):
ul_type = 'chi2'
[ul_flux, ul_results] = \
IUL.calc_chi2(like,self.likelihoodConf['sourcename'],cl=ul_cl,
skip_global_opt=True,
verbosity = max(verbosity-2,0),
emin=emin, emax=emax)
if ul_type != None:
lc['normfree']['ul'] = dict(flux = ul_flux,
results = ul_results,
type = ul_type)
# ----------------------------- FIT 3 -----------------------------
if verbosity > 1:
print '- Fit 3 - All parameters of',self.likelihoodConf['sourcename'],'free'
like.setFreeFlag(self.likelihoodConf['sourcename'], srcfreepar, 1)
like.syncSrcParams(self.likelihoodConf['sourcename'])
like.fit(max(verbosity-3, 0))
lc['allfree'] = dict()
fitstat = like.optObject.getRetCode()
if verbosity > 1 and fitstat != 0:
print "- Fit 3 - Minimizer returned with code: ", fitstat
lc['allfree']['fitstat'] = fitstat
lc['allfree']['logL'] = like.logLike.value()
lc['allfree']['ts'] = like.Ts(self.likelihoodConf['sourcename'])
if verbosity > 1:
print '- Fit 3 - log Like:',lc['allfree']['logL'],\
'(TS='+str(lc['allfree']['ts'])+')'
lc['allfree']['nfree']=len(like.freePars(self.likelihoodConf['sourcename']))
lc['allfree']['flux']=like[self.likelihoodConf['sourcename']].flux(emin, emax)
pars = dict()
for pn in like[self.likelihoodConf['sourcename']].funcs['Spectrum'].paramNames:
p = like[self.likelihoodConf['sourcename']].funcs['Spectrum'].getParam(pn)
pars[p.getName()] = dict(name = p.getName(),
value = p.getTrueValue(),
error = p.error()*p.getScale(),
free = p.isFree())
lc['allfree']['pars'] = pars
self.lc.append(lc)
if interim_save_filename != None:
self.saveProcessedObs(interim_save_filename)
def GenAnalysisObjects(config, verbose=1, xmlfile=""):
mes = Loggin.Message()
#check is the summed likelihood method should be used and get the
#Analysis objects (observation and (Un)BinnedAnalysis objects)
folder = config['out']
Fit = SummedLikelihood.SummedLikelihood()
EUnBinned = config['ComponentAnalysis']['EUnBinned']
emintotal = float(config['energy']['emin'])
emaxtotal = float(config['energy']['emax'])
evtnum = [config["event"]["evtype"]] #for std analysis
evtold = evtnum[0] #for std analysis
# Create one obs instance for each component
if isKey(config['ComponentAnalysis'], 'FrontBack') == 'yes':
evtnum = [1, 2]
if isKey(config['ComponentAnalysis'], 'PSF') == 'yes':
evtnum = [4, 8, 16, 32]
if isKey(config['ComponentAnalysis'], 'EDISP') == 'yes':
evtnum = [64, 128, 256, 521]
oldxml = config['file']['xml']
for k, evt in enumerate(evtnum):
config['event']['evtype'] = evt
config["file"]["xml"] = oldxml.replace(
".xml", "_" + typeirfs[evt] + ".xml").replace("_.xml", ".xml")
if EUnBinned > emintotal and EUnBinned < emaxtotal:
mes.info(
"Breaking the analysis in Binned (low energy) and Unbinned (high energies)"
)
analysestorun = ["lowE", "highE"]
for k, TYPE in enumerate(analysestorun):
tag = TYPE
if typeirfs[evt] != "":
tag += "_" + typeirfs[evt] # handle name of fits file
# Tune parameters
if TYPE is "lowE":
config['energy']['emin'] = emintotal
config['energy']['emax'] = min(config['energy']['emax'],
EUnBinned)
config['analysis']['likelihood'] = "binned"
config['analysis']['ComputeDiffrsp'] = "no"
elif TYPE is "highE":
config['energy']['emin'] = max(config['energy']['emin'],
EUnBinned)
config['energy']['emax'] = emaxtotal
config['analysis']['likelihood'] = "unbinned"
config['analysis']['ComputeDiffrsp'] = "yes"
Analyse = Analysis(folder, config, \
configgeneric=config,\
tag=TYPE,\
verbose=verbose)
Fit_component = Analyse.CreateLikeObject()
Fit.addComponent(Fit_component)
FitRunner = Analyse
FitRunner.obs.Emin = emintotal
FitRunner.obs.Emax = emaxtotal
else:
Analyse = Analysis(folder, config, \
configgeneric=config,\
tag=typeirfs[evt], verbose = verbose)
# if not(xmlfile ==""): Analyse.obs.xmlfile = xmlfile
Fit_component = Analyse.CreateLikeObject()
Fit.addComponent(Fit_component)
FitRunner = Analyse
config["event"]["evtype"] = evtold
FitRunner.config = config
return FitRunner, Fit
def likelihood_upper_limit3(z):
#Array to hold results of flux upper limit calculation
num_ebins = 51 #1 more than the number of bins due to the fencepost problem
energies = 10**np.linspace(np.log10(6000), np.log10(800000), num_ebins)
ebin_widths = np.diff(energies)
sourcemap = '6gev_srcmap_03_pedestal.fits' #'box_srcmap_artificial_box.fits'#'6gev_srcmap_complete.fits'
box_flux = np.zeros((num_ebins - 1))
box_flux_bayesian = np.zeros((num_ebins - 1))
box_flux_frequentist = np.zeros((num_ebins - 1))
corr = np.zeros((num_ebins - 1))
corr2 = np.zeros((num_ebins - 1))
gll_index = np.zeros((num_ebins - 1))
disk_index = np.zeros((num_ebins - 1))
#reconstructed_spectra = np.zeros((num_ebins-1, num_ebins-1))
#Loop through upper edge of box
for index in range(6, 48):
box_width = energies[index] * 2.0 * np.sqrt(1.0 -
z) / (1 + np.sqrt(1.0 - z))
print "Calculating upper limit in bin " + str(
index) + " at energy " + str(energies[index])
#print "bin " + str(np.argmin(np.abs(energies-energy)))
#window_low, window_high = window(energy, energies)
window_low = index - 6
window_high = index + 2
#Generate two observations (one above the window and one below)
#Make two exposure maps
if index > 6:
exposure_complete = pyfits.open('6gev_exposure.fits')
exposure_complete[0].data = exposure_complete[0].data[:window_low +
1]
a = exposure_complete[0]
exposure_complete[1].data = exposure_complete[1].data[:window_low +
1]
b = exposure_complete[1]
hdulist = pyfits.HDUList([a, b, exposure_complete[2]])
os.system('rm exposure_low.fits')
hdulist.writeto('exposure_low.fits')
exposure_complete.close()
if index < 48:
exposure_complete = pyfits.open('6gev_exposure.fits')
exposure_complete[0].data = exposure_complete[0].data[window_high:]
a = exposure_complete[0]
exposure_complete[1].data = exposure_complete[1].data[window_high:]
b = exposure_complete[1]
hdulist = pyfits.HDUList([a, b, exposure_complete[2]])
os.system('rm exposure_high.fits')
hdulist.writeto('exposure_high.fits')
exposure_complete.close()
#Make two sourcemaps
if index > 6:
srcmap_complete = pyfits.open(sourcemap)
srcmap_complete[0].data = srcmap_complete[0].data[:window_low]
a = srcmap_complete[0]
srcmap_complete[2].data = srcmap_complete[2].data[:window_low]
b = srcmap_complete[2]
srcmap_complete[3].data = srcmap_complete[3].data[:window_low + 1]
c = srcmap_complete[3]
srcmap_complete[4].data = srcmap_complete[4].data[:window_low + 1]
d = srcmap_complete[4]
srcmap_complete[5].data = srcmap_complete[5].data[:window_low + 1]
e = srcmap_complete[5]
srcmap_complete[6].data = srcmap_complete[6].data[:window_low + 1]
f = srcmap_complete[6]
srcmap_complete[7].data = srcmap_complete[7].data[:window_low + 1]
g = srcmap_complete[7]
srcmap_complete[8].data = srcmap_complete[8].data[:window_low + 1]
h = srcmap_complete[8]
srcmap_complete[9].data = srcmap_complete[9].data[:window_low + 1]
m = srcmap_complete[9]
os.system('rm srcmap_low.fits')
b.header['DSVAL4'] = str() + ':' + str()
hdulist = pyfits.HDUList(
[a, srcmap_complete[1], b, c, d, e, f, g, h, m])
hdulist.writeto('srcmap_low.fits')
srcmap_complete.close()
if index < 48:
srcmap_complete = pyfits.open(sourcemap)
srcmap_complete[0].data = srcmap_complete[0].data[window_high:]
a = srcmap_complete[0]
srcmap_complete[2].data = srcmap_complete[2].data[window_high:]
r = 0
for entry in srcmap_complete[2].data:
entry[0] = int(r)
r += 1
#srcmap_complete[2].data[:,0] = np.arange(0, len(srcmap_complete[2].data[:,0]))
b = srcmap_complete[2]
srcmap_complete[3].data = srcmap_complete[3].data[window_high:]
c = srcmap_complete[3]
srcmap_complete[4].data = srcmap_complete[4].data[window_high:]
d = srcmap_complete[4]
srcmap_complete[5].data = srcmap_complete[5].data[window_high:]
e = srcmap_complete[5]
srcmap_complete[6].data = srcmap_complete[6].data[window_high:]
f = srcmap_complete[6]
srcmap_complete[7].data = srcmap_complete[7].data[window_high:]
g = srcmap_complete[7]
srcmap_complete[8].data = srcmap_complete[8].data[window_high:]
h = srcmap_complete[8]
srcmap_complete[9].data = srcmap_complete[9].data[window_high:]
m = srcmap_complete[9]
os.system('rm srcmap_high.fits')
hdulist = pyfits.HDUList(
[a, srcmap_complete[1], b, c, d, e, f, g, h, m])
hdulist.writeto('srcmap_high.fits')
srcmap_complete.close()
summedLike = SummedLikelihood()
if index > 6:
obs_low = BinnedObs(srcMaps='srcmap_low.fits',
expCube='6gev_ltcube.fits',
binnedExpMap='exposure_low.fits',
irfs='CALDB')
like_low = BinnedAnalysis(obs_low,
'xmlmodel_free.xml',
optimizer='NEWMINUIT')
summedLike.addComponent(like_low)
if index < 48:
obs_high = BinnedObs(srcMaps='srcmap_high.fits',
expCube='6gev_ltcube.fits',
binnedExpMap='exposure_high.fits',
irfs='CALDB')
like_high = BinnedAnalysis(obs_high,
'xmlmodel_free.xml',
optimizer='NEWMINUIT')
summedLike.addComponent(like_high)
print "Fitting SummedLikelihood"
summedLike.ftol = 1e-8
summedLike.fit(verbosity=3)
summedLike.writeXml('xmlmodel_free.xml')
for k in range(len(summedLike.params())):
summedLike.freeze(k)
summedLike.writeXml('xmlmodel_fixed.xml')
print "Fitting all data"
calculation = 'poisson'
obs_complete = BinnedObs(srcMaps=sourcemap,
expCube='6gev_ltcube.fits',
binnedExpMap='6gev_exposure.fits',
irfs='CALDB')
edit_box_xml(100000.0, 1e-15, 0.0)
like = BinnedAnalysis(obs_complete,
'xmlmodel_fixed_box.xml',
optimizer='MINUIT')
like.tol = 1e-8
like_obj = pyLike.Minuit(like.logLike)
like.fit(verbosity=3, optObject=like_obj)
like.writeXml('xmlmodel_fixed_box.xml')
#Flucuate the window data
f = pyfits.open(sourcemap)
poisson_data = np.zeros(
(len(range(max(window_low, 0), min(window_high, 49))), 50, 50))
q = 0
for bin in range(max(window_low, 0), min(window_high, 49)):
for source in like.sourceNames():
for j in range(3, 10):
if source == f[j].header['EXTNAME']:
the_index = j
model_counts = np.zeros(
(1, len(f[the_index].data[bin].ravel())))[0]
num_photons = int(
np.round(np.random.poisson(
like._srcCnts(source)[bin]))) #
for photon in range(int(num_photons)):
phot_loc = int(
make_random(np.arange(0, len(model_counts), 1),
f[the_index].data[bin].ravel()))
model_counts[phot_loc] += 1
model_counts = model_counts.reshape(50, 50)
poisson_data[q] += model_counts
q += 1
f[0].data[max(window_low, 0):min(window_high, 49)] = poisson_data
os.system('rm box_srcmap_poisson.fits')
f.writeto('box_srcmap_poisson.fits')
f.close()
obs_poisson = BinnedObs(srcMaps='box_srcmap_poisson.fits',
expCube='6gev_ltcube.fits',
binnedExpMap='6gev_exposure.fits',
irfs='CALDB')
like = BinnedAnalysis(obs_poisson,
'xmlmodel_fixed_box.xml',
optimizer='NEWMINUIT')
like.tol = 1e-8
like_obj = pyLike.Minuit(like.logLike)
like.fit(verbosity=0, optObject=like_obj)
if calculation == 'complete':
obs_calculation = obs_complete
else:
obs_calculation = obs_poisson
print "Finding Upper Limit..."
null_likelihood = loglikelihood3(energies[index], 1.0e-15,
obs_calculation, z)
delta_loglike = 0.0
#increase box flux until likelihood > 2sigma over null likelihood
box_flux[index] = 3.e-15
crit_chi2 = 2.706
while delta_loglike < crit_chi2:
print "delta loglike = " + str(delta_loglike)
print "flux = " + str(box_flux[index]) + " likelihood = " + str(
loglikelihood3(energies[index], box_flux[index],
obs_calculation, z))
box_flux[index] *= 3.0
delta_loglike = 2.0 * (loglikelihood3(
energies[index], box_flux[index], obs_calculation, z) -
null_likelihood)
box_flux[index] *= 1.0 / 3.0
delta_loglike = 2.0 * (loglikelihood3(energies[index], box_flux[index],
obs_calculation, z) -
null_likelihood)
print "Delta loglike = " + str(delta_loglike)
while delta_loglike < crit_chi2:
print "delta loglike = " + str(delta_loglike)
print "flux = " + str(box_flux[index]) + " likelihood = " + str(
loglikelihood3(energies[index], box_flux[index],
obs_calculation, z))
box_flux[index] *= 1.5
delta_loglike = 2.0 * (loglikelihood3(
energies[index], box_flux[index], obs_calculation, z) -
null_likelihood)
box_flux[index] *= 1.0 / 1.5
delta_loglike = 2.0 * (loglikelihood3(energies[index], box_flux[index],
obs_calculation, z) -
null_likelihood)
while delta_loglike < crit_chi2:
print "delta loglike = " + str(delta_loglike)
print "flux = " + str(box_flux[index]) + " likelihood = " + str(
loglikelihood3(energies[index], box_flux[index],
obs_calculation, z))
box_flux[index] *= 1.03
delta_loglike = 2.0 * (loglikelihood3(
energies[index], box_flux[index], obs_calculation, z) -
null_likelihood)
box_flux[index] *= 1.0 / 1.03
delta_loglike = 2.0 * (loglikelihood3(energies[index], box_flux[index],
obs_calculation, z) -
null_likelihood)
print "delta log like = " + str(delta_loglike)
calc_cov = False
if calc_cov:
like1 = BinnedAnalysis(obs_calculation,
'xmlmodel_fixed_box.xml',
optimizer='DRMNFB')
like1.thaw(like1.par_index('Disk Component', 'Index'))
like1.thaw(like1.par_index('Disk Component', 'Prefactor'))
like1.thaw(like1.par_index('Box Component', 'Normalization'))
like1.tol = 1e-5
like1obj = pyLike.Minuit(like1.logLike)
like1.fit(verbosity=0, optObject=like1obj, covar=False)
like1.writeXml('xmlmodel_fixed_box.xml')
like2 = BinnedAnalysis(obs_calculation,
'xmlmodel_fixed_box.xml',
optimizer='NewMinuit')
like2.tol = 1e-8
like2obj = pyLike.Minuit(like1.logLike)
like2.fit(verbosity=3, optObject=like1obj, covar=True)
#ul = UpperLimit(like1,'Box Component')
#ul.compute(emin=100.0,emax=500000, delta=3.91)
#box_flux_bayesian[index] = float(ul.bayesianUL()[0])
#box_flux_frequentist[index] = float(ul.results[0].value)
print like2.covariance
print 'Return code: ' + str(like2obj.getRetCode())
cov = like2.covariance
corr[index] = cov[0][1] / np.sqrt(cov[0][0] * cov[1][1])
corr2[index] = cov[0][2] / np.sqrt(cov[0][0] * cov[2][2])
print "Correlations:"
print corr[index]
print corr2[index]
#if like2obj.getRetCode()!=0:
plot_spectrum(like2, energies, index, window_low, window_high)
if calc_cov:
file = open('correlation_results.pk1', 'wb')
pickle.dump([corr, corr2, box_flux_bayesian, box_flux_frequentist],
file)
file.close()
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(energies[np.nonzero(corr)],
corr[np.nonzero(corr)],
color='blue',
label='Box vs GC Prefactor')
ax.plot(energies[np.nonzero(corr2)],
corr2[np.nonzero(corr2)],
color='red',
label='Box vs GC index')
plt.ylim([-1.0, 1.0])
plt.legend()
plt.xscale('log')
plt.show()
return box_flux
def GenAnalysisObjects(config, verbose=1, xmlfile=""):
import os
import os.path
import math
import SummedLikelihood
from enrico.xml_model import XmlMaker
from enrico.extern.configobj import ConfigObj
from utils import hasKey, isKey
import Loggin
mes = Loggin.Message()
#check is the summed likelihood method should be used and get the
#Analysis objects (observation and (Un)BinnedAnalysis objects)
SummedLike = config['Spectrum']['SummedLike']
folder = config['out']
# If there is no xml file, create it and print a warning
if (not os.path.isfile(config['file']['xml'])):
mes.warning("Xml not found, creating one for the given config %s" %
config['file']['xml'])
XmlMaker(config)
Fit = SummedLikelihood.SummedLikelihood()
if hasKey(config, 'ComponentAnalysis') == True:
# Create one obs instance for each component
configs = [None] * 4
Fits = [None] * 4
Analyses = [None] * 4
if isKey(config['ComponentAnalysis'], 'FrontBack') == 'yes':
from enrico.data import fermievtypes
mes.info("Breaking the analysis in Front/Back events")
# Set Summed Likelihood to True
config['Spectrum']['SummedLike'] = 'yes'
oldxml = config['file']['xml']
for k, TYPE in enumerate(["FRONT", "BACK"]):
configs[k] = ConfigObj(config)
configs[k]['event']['evtype'] = fermievtypes[TYPE]
try:
Analyses[k] = Analysis(folder, configs[k], \
configgeneric=config,\
tag=TYPE, verbose = verbose)
if not (xmlfile == ""): Analyses[k].obs.xmlfile = xmlfile
Fits[k] = Analyses[k].CreateLikeObject()
Fit.addComponent(Fits[k])
except RuntimeError, e:
if 'RuntimeError: gtltcube execution failed' in str(e):
mes.warning("Event type %s is empty! Error is %s" %
(TYPE, str(e)))
FitRunner = Analyses[0]
elif isKey(config['ComponentAnalysis'], 'PSF') == 'yes':
from enrico.data import fermievtypes
mes.info("Breaking the analysis in PSF 0,1,2,3.")
# Clone the configs
# Set Summed Likelihood to True
config['Spectrum']['SummedLike'] = 'yes'
for k, TYPE in enumerate(["PSF0", "PSF1", "PSF2", "PSF3"]):
configs[k] = ConfigObj(config)
configs[k]['event']['evtype'] = fermievtypes[TYPE]
try:
Analyses[k] = Analysis(folder, configs[k], \
configgeneric=config,\
tag=TYPE, verbose = verbose)
if not (xmlfile == ""): Analyses[k].obs.xmlfile = xmlfile
Fits[k] = Analyses[k].CreateLikeObject()
Fit.addComponent(Fits[k])
except RuntimeError, e:
if 'RuntimeError: gtltcube execution failed' in str(e):
mes.warning("Event type %s is empty! Error is %s" %
(TYPE, str(e)))
FitRunner = Analyses[0]
def GenAnalysisObjects(config, verbose = 1, xmlfile =""):
# Array containing the list of analysis objects (needed to produce the individual residual maps)
ListOfAnalysisObjects = []
mes = Loggin.Message()
#check is the summed likelihood method should be used and get the
#Analysis objects (observation and (Un)BinnedAnalysis objects)
folder = config['out']
# If there are no xml files, create it and print a warning <--- Not sure if this is needed here.
Fit = SummedLikelihood.SummedLikelihood()
EUnBinned = config['ComponentAnalysis']['EUnBinned']
emintotal = float(config['energy']['emin'])
emaxtotal = float(config['energy']['emax'])
evtnum = [config["event"]["evtype"]] #for std analysis
evtold = evtnum[0] #for std analysis
# Create one obs instance for each component.
# The first 3 can be combined with splitting in energy. The 4th tries to mimick 4FGL.
if isKey(config['ComponentAnalysis'],'FrontBack') == 'yes':
evtnum = [1, 2]
config['analysis']['likelihood'] = "binned"
elif isKey(config['ComponentAnalysis'],'PSF') == 'yes':
evtnum = [4,8,16,32]
config['analysis']['likelihood'] = "binned"
elif isKey(config['ComponentAnalysis'],'EDISP') == 'yes':
evtnum = [64,128,256,521]
config['analysis']['likelihood'] = "binned"
elif isKey(config['ComponentAnalysis'],'FGL4') == 'yes':
# Special case of the PSF component analysis,
# where up to 15 components (energy+PSF) are created following
# 4FGL prescription.
from catalogComponents import evtnum, energybins, nbinsbins, zmaxbins, ringwidths, pixelsizes
config['analysis']['likelihood'] = "binned"
oldxml = config['file']['xml']
bin_i = 0
roi = 0
# energybins is a dictionary containing an index and a pair of energies
for ebin_i in energybins:
# Restrict the analysis to the specified energy range in all cases.
if emintotal>=energybins[ebin_i][1]:
continue
if emaxtotal<=energybins[ebin_i][0]:
continue
if (roi==0): roi = 2.*ringwidths[ebin_i]+4.
zmax = zmaxbins[ebin_i]
nbinsE = nbinsbins[ebin_i]
energybin = energybins[ebin_i]
for k,evt in enumerate(evtnum):
pixel_size = pixelsizes[ebin_i][k]
if pixel_size<0: continue
tag = "{0}_En{1}".format(typeirfs[evt],ebin_i)
# Approximation, in the 4FGL the core radius changes from src to src!
mes.info("Breaking the analysis in bins ~ 4FGL")
config['event']['evtype'] = evt
config["file"]["xml"] = oldxml.replace(".xml","_")+typeirfs[evt]+"_"+\
"En{0}.xml".format(ebin_i)
config["energy"]["emin"] = max(emintotal,energybin[0])
config["energy"]["emax"] = min(emaxtotal,energybin[1])
config["analysis"]["likelihood"] = "binned"
config["analysis"]["ComputeDiffrsp"] = "no"
config["analysis"]["enumbins_per_decade"] = \
int(1.*nbinsE/math.log10(energybin[1]/energybin[0])+0.5)
config["space"]["rad"] = roi
config["analysis"]["zmax"] = zmax
Analyse = Analysis(folder, config, \
configgeneric=config,\
tag=tag, verbose=verbose)
ListOfAnalysisObjects.append(Analyse)
if not(xmlfile ==""): Analyse.obs.xmlfile = xmlfile
mes.info('Creating Likelihood object for component.')
Fit_component = Analyse.CreateLikeObject()
mes.info('Adding component to the summed likelihood.')
Fit.addComponent(Fit_component)
FitRunner = Analyse
FitRunner.obs.Emin = emintotal
FitRunner.obs.Emax = emaxtotal
config["energy"]["emin"] = emintotal
config["energy"]["emax"] = emaxtotal
config["event"]["evtype"] = evtold
FitRunner.config = config
return FitRunner,Fit,ListOfAnalysisObjects
# Standard (non-4FGL) analysis components
oldxml = config['file']['xml']
for k,evt in enumerate(evtnum):
config['event']['evtype'] = evt
if typeirfs[evt] != "":
config["file"]["xml"] = oldxml.replace(".xml","_"+typeirfs[evt]+".xml")
if EUnBinned>emintotal and EUnBinned<emaxtotal:
mes.info("Breaking the analysis in Binned (low energy) and Unbinned (high energies)")
analysestorun = ["lowE","highE"]
for j,TYPE in enumerate(analysestorun):
tag = TYPE
if typeirfs[evt] != "" : tag += "_"+typeirfs[evt]# handle name of fits file
config["file"]["xml"] = oldxml.replace(".xml","_"+tag+".xml")
# Tune parameters
if TYPE is "lowE":
config['energy']['emin'] = emintotal
config['energy']['emax'] = min(config['energy']['emax'],EUnBinned)
config['analysis']['likelihood'] = "binned"
config['analysis']['ComputeDiffrsp'] = "no"
elif TYPE is "highE":
config['energy']['emin'] = max(config['energy']['emin'],EUnBinned)
config['energy']['emax'] = emaxtotal
config['analysis']['likelihood'] = "unbinned"
config['analysis']['ComputeDiffrsp'] = "yes"
Analyse = Analysis(folder, config, \
configgeneric=config,\
tag=tag,\
verbose=verbose)
ListOfAnalysisObjects.append(Analyse)
mes.info('Creating Likelihood object for component.')
Fit_component = Analyse.CreateLikeObject()
mes.info('Adding component to the summed likelihood.')
Fit.addComponent(Fit_component)
FitRunner = Analyse
FitRunner.obs.Emin = emintotal
FitRunner.obs.Emax = emaxtotal
config["energy"]["emin"] = emintotal
config["energy"]["emax"] = emaxtotal
else:
Analyse = Analysis(folder, config, \
configgeneric=config,\
tag=typeirfs[evt], verbose = verbose)
ListOfAnalysisObjects.append(Analyse)
if not(xmlfile ==""): Analyse.obs.xmlfile = xmlfile
mes.info('Creating Likelihood object for component.')
Fit_component = Analyse.CreateLikeObject()
mes.info('Adding component to the summed likelihood.')
Fit.addComponent(Fit_component)
FitRunner = Analyse
config["event"]["evtype"] = evtold
FitRunner.config = config
return FitRunner,Fit,ListOfAnalysisObjects
