def __init__(self, name, phafile, bkgfile, rspfile, arffile=None): self.name = name # Check that all file exists notExistant = [] inputFiles = [phafile, bkgfile, rspfile] for i in range(3): # The file could contain a {#} specification, like spectrum.pha{3}, # which indicate the 3rd spectrum in the spectrum.pha file inputFiles[i] = file_utils.sanitize_filename(inputFiles[i].split("{")[0]) if not file_utils.file_existing_and_readable(inputFiles[i]): raise IOError("File %s does not exist or is not readable" % (inputFiles[i])) phafile, bkgfile, rspfile = inputFiles # Check the arf, if provided if arffile is not None: arffile = file_utils.sanitize_filename(arffile.split("{")[0]) if not file_utils.file_existing_and_readable(arffile): raise IOError("File %s does not exist or is not readable" % (arffile)) self.phafile = OGIPPHA(phafile, filetype="observed") self.exposure = self.phafile.getExposure() self.bkgfile = OGIPPHA(bkgfile, filetype="background") self.response = Response(rspfile, arffile) # Start with an empty mask (the user will overwrite it using the # setActiveMeasurement method) self.mask = numpy.asarray(numpy.ones(self.phafile.getRates().shape), numpy.bool) # Get the counts for this spectrum self.counts = self.phafile.getRates()[self.mask] * self.exposure # Check that counts is positive idx = self.counts < 0 if numpy.sum(idx) > 0: warnings.warn( "The observed spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning, ) self.counts[idx] = 0 pass # Get the background counts for this spectrum self.bkgCounts = self.bkgfile.getRates()[self.mask] * self.exposure # Check that bkgCounts is positive idx = self.bkgCounts < 0 if numpy.sum(idx) > 0: warnings.warn( "The background spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning, ) self.bkgCounts[idx] = 0 # Check that the observed counts are positive idx = self.counts < 0 if numpy.sum(idx) > 0: raise RuntimeError("Negative counts in observed spectrum %s. Data are corrupted." % (phafile)) # Keep a copy which will never be modified self.counts_backup = numpy.array(self.counts, copy=True) self.bkgCounts_backup = numpy.array(self.bkgCounts, copy=True) # Effective area correction is disabled by default, i.e., # the nuisance parameter is fixed to 1 self.nuisanceParameters = {} self.nuisanceParameters["InterCalib"] = Parameter("InterCalib", 1, min_value=0.9, max_value=1.1, delta=0.01) self.nuisanceParameters["InterCalib"].fix = True
def __init__(self,name,phafile,bkgfile,rspfile): ''' If the input files are PHA2 files, remember to specify the spectrum number, for example: FermiGBMLike("GBM","spectrum.pha{2}","bkgfile.bkg{2}","rspfile.rsp{2}") to load the second spectrum, second background spectrum and second response. ''' self.name = name #Check that all file exists notExistant = [] if(not os.path.exists(phafile.split("{")[0])): notExistant.append(phafile.split("{")[0]) if(not os.path.exists(bkgfile.split("{")[0])): notExistant.append(bkgfile.split("{")[0]) if(not os.path.exists(rspfile.split("{")[0])): notExistant.append(rspfile.split("{")[0]) if(len(notExistant)>0): for nt in notExistant: print("File %s does not exists!" %(nt)) raise IOError("One or more input file do not exist!") self.phafile = OGIPPHA(phafile,filetype='observed') self.exposure = self.phafile.getExposure() self.bkgfile = OGIPPHA(bkgfile,filetype="background") self.response = Response(rspfile) #Start with an empty mask (the user will overwrite it using the #setActiveMeasurement method) self.mask = numpy.asarray( numpy.ones(self.phafile.getRates().shape), numpy.bool) #Get the counts for this spectrum self.counts = ( self.phafile.getRates()[self.mask] * self.exposure ) #Check that counts is positive idx = (self.counts < 0) if(numpy.sum(idx) > 0): warnings.warn("The observed spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning) self.counts[idx] = 0 pass #Get the background counts for this spectrum self.bkgCounts = ( self.bkgfile.getRates()[self.mask] * self.exposure ) #Check that bkgCounts is positive idx = (self.bkgCounts < 0) if(numpy.sum(idx) > 0): warnings.warn("The background spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning) self.bkgCounts[idx] = 0 #Check that the observed counts are positive idx = self.counts < 0 if numpy.sum( idx ) > 0: raise RuntimeError("Negative counts in observed spectrum %s. Data are corrupted." % ( phafile )) #Keep a copy which will never be modified self.counts_backup = numpy.array(self.counts,copy=True) self.bkgCounts_backup = numpy.array(self.bkgCounts,copy=True) #Effective area correction is disabled by default, i.e., #the nuisance parameter is fixed to 1 self.nuisanceParameters = {} self.nuisanceParameters['InterCalib'] = Parameter("InterCalib",1,0.9,1.1,0.01,fixed=True,nuisance=True)
class GenericOGIPLike(PluginPrototype): def __init__(self, name, phafile, bkgfile, rspfile, arffile=None): self.name = name # Check that all file exists notExistant = [] inputFiles = [phafile, bkgfile, rspfile] for i in range(3): # The file could contain a {#} specification, like spectrum.pha{3}, # which indicate the 3rd spectrum in the spectrum.pha file inputFiles[i] = file_utils.sanitize_filename(inputFiles[i].split("{")[0]) if not file_utils.file_existing_and_readable(inputFiles[i]): raise IOError("File %s does not exist or is not readable" % (inputFiles[i])) phafile, bkgfile, rspfile = inputFiles # Check the arf, if provided if arffile is not None: arffile = file_utils.sanitize_filename(arffile.split("{")[0]) if not file_utils.file_existing_and_readable(arffile): raise IOError("File %s does not exist or is not readable" % (arffile)) self.phafile = OGIPPHA(phafile, filetype="observed") self.exposure = self.phafile.getExposure() self.bkgfile = OGIPPHA(bkgfile, filetype="background") self.response = Response(rspfile, arffile) # Start with an empty mask (the user will overwrite it using the # setActiveMeasurement method) self.mask = numpy.asarray(numpy.ones(self.phafile.getRates().shape), numpy.bool) # Get the counts for this spectrum self.counts = self.phafile.getRates()[self.mask] * self.exposure # Check that counts is positive idx = self.counts < 0 if numpy.sum(idx) > 0: warnings.warn( "The observed spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning, ) self.counts[idx] = 0 pass # Get the background counts for this spectrum self.bkgCounts = self.bkgfile.getRates()[self.mask] * self.exposure # Check that bkgCounts is positive idx = self.bkgCounts < 0 if numpy.sum(idx) > 0: warnings.warn( "The background spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning, ) self.bkgCounts[idx] = 0 # Check that the observed counts are positive idx = self.counts < 0 if numpy.sum(idx) > 0: raise RuntimeError("Negative counts in observed spectrum %s. Data are corrupted." % (phafile)) # Keep a copy which will never be modified self.counts_backup = numpy.array(self.counts, copy=True) self.bkgCounts_backup = numpy.array(self.bkgCounts, copy=True) # Effective area correction is disabled by default, i.e., # the nuisance parameter is fixed to 1 self.nuisanceParameters = {} self.nuisanceParameters["InterCalib"] = Parameter("InterCalib", 1, min_value=0.9, max_value=1.1, delta=0.01) self.nuisanceParameters["InterCalib"].fix = True pass def useIntercalibrationConst(self, factorLowBound=0.9, factorHiBound=1.1): self.nuisanceParameters["InterCalib"].free() self.nuisanceParameters["InterCalib"].set_bounds(factorLowBound, factorHiBound) # Check that the parameter is within the provided bounds value = self.nuisanceParameters["InterCalib"].value if value < factorLowBound: warnings.warn( "The intercalibration constant was %s, lower than the provided lower bound." % (value, factorLowBound) + " Setting it equal to the lower bound" ) self.nuisanceParameters["InterCalib"].setValue(float(factorLowBound)) if value > factorHiBound: warnings.warn( "The intercalibration constant was %s, larger than the provided hi bound." % (value, factorHiBound) + " Setting it equal to the hi bound" ) self.nuisanceParameters["InterCalib"].setValue(float(factorHiBound)) def fixIntercalibrationConst(self, value=None): if value is not None: # Fixing the constant to the provided value self.nuisanceParameters["InterCalib"].setValue(float(value)) else: # Do nothing, i.e., leave the constant to the value # it currently has pass self.nuisanceParameters["InterCalib"].fix() def setActiveMeasurements(self, *args): """Set the measurements to be used during the analysis. Use as many ranges as you need, specified as 'emin-emax'. Energies are in keV. Example: setActiveMeasurements('10-12.5','56.0-100.0') which will set the energy range 10-12.5 keV and 56-100 keV to be used in the analysis""" # To implelemnt this we will use an array of boolean index, # which will filter # out the non-used channels during the logLike # Now build the mask: values for which the mask is 0 will be masked mask = numpy.zeros(self.phafile.getRates().shape) for arg in args: ee = map(float, arg.replace(" ", "").split("-")) emin, emax = sorted(ee) idx1 = self.response.energyToChannel(emin) idx2 = self.response.energyToChannel(emax) mask[idx1 : idx2 + 1] = True pass self.mask = numpy.array(mask, numpy.bool) self.counts = self.counts_backup[self.mask] self.bkgCounts = self.bkgCounts_backup[self.mask] print("Now using %s channels out of %s" % (numpy.sum(self.mask), self.phafile.getRates().shape[0])) pass def get_name(self): """ Return a name for this dataset (likely set during the constructor) """ return self.name pass def set_model(self, likelihoodModel): """ Set the model to be used in the joint minimization. """ self.likelihoodModel = likelihoodModel nPointSources = self.likelihoodModel.get_number_of_point_sources() # This is a wrapper which iterates over all the point sources and get # the fluxes # We assume there are no extended sources, since the GBM cannot handle them def diffFlux(energies): fluxes = self.likelihoodModel.get_point_source_fluxes(0, energies) # If we have only one point source, this will never be executed for i in range(1, nPointSources): fluxes += self.likelihoodModel.get_point_source_fluxes(i, energies) return fluxes self.diffFlux = diffFlux # The following integrates the diffFlux function using Simpson's rule # This assume that the intervals e1,e2 are all small, which is guaranteed # for any reasonable response matrix, given that e1 and e2 are Monte-Carlo # energies. It also assumes that the function is smooth in the interval # e1 - e2 and twice-differentiable, again reasonable on small intervals for # decent models. It might fail for models with too sharp features, smaller # than the size of the monte carlo interval. def integral(e1, e2): # Simpson's rule return (e2 - e1) / 6.0 * (self.diffFlux(e1) + 4 * self.diffFlux((e1 + e2) / 2.0) + self.diffFlux(e2)) self.response.setFunction(diffFlux, integral) pass def inner_fit(self): # Effective area correction if self.nuisanceParameters["InterCalib"].free: # A true fit would be an overkill, and slow # Just sample a 100 values and choose the minimum values = numpy.linspace( self.nuisanceParameters["InterCalib"].minValue, self.nuisanceParameters["InterCalib"].maxValue, 100 ) # I do not use getLogLike so I can compute only once the folded model # (which is not going to change during the inner fit) folded = self.getFoldedModel() modelCounts = folded * self.exposure def fitfun(cons): self.nuisanceParameters["InterCalib"].setValue(cons) return (-1) * self._computeLogLike( self.nuisanceParameters["InterCalib"].value * modelCounts + self.bkgCounts ) logLval = map(fitfun, values) idx = numpy.argmax(logLval) self.nuisanceParameters["InterCalib"].setValue(values[idx]) # return logLval[idx] # Now refine with minuit parameters = collections.OrderedDict() parameters[(self.name, "InterCalib")] = self.nuisanceParameters["InterCalib"] minimizer = minimization.MinuitMinimizer(fitfun, parameters) bestFit, mlogLmin = minimizer.minimize() return mlogLmin * (-1) else: return self.get_log_like() def getFoldedModel(self): # Get the folded model for this spectrum # (this is the rate predicted, in cts/s) return self.response.convolve()[self.mask] def getModelAndData(self): e1, e2 = (self.response.ebounds[:, 0], self.response.ebounds[:, 1]) return ( self.response.convolve()[self.mask] * self.exposure + self.bkgCounts, e1[self.mask], e2[self.mask], self.counts, ) def display(self): # Plot the counts spectrum with residuals model, e1, e2, counts = self.getModelAndData() # Try to automagically decide a good constant number # of counts, based on the input total = numpy.sum(counts) trials = [50, 25, 15, 5] choice = None for t in trials: if total / float(t) >= 10: choice = t break # If no choice worked, use the minimum, which is 5 counts # per bin if choice is None: choice = 5 # Bin the data by constant counts binner = Binner.Binner(e1, e2, counts) ne1, ne2, nc, newModel = binner.byConstantCounts(choice, model) # Now plot the results fig = plt.figure() gs = gridspec.GridSpec(2, 1, height_ratios=[2, 1]) gs.update(hspace=0) sub = plt.subplot(gs[0]) # Plot data ec = (ne2 + ne1) / 2.0 de = (ne2 - ne1) / 2.0 y = nc / (de * 2.0) / self.exposure dy = numpy.sqrt(nc) / (de * 2.0) / self.exposure sub.errorbar(ec, y, xerr=de, yerr=dy, fmt=".", capsize=0) # Plot model xx = numpy.append(ne1, [ne2[-1]]) yy = newModel / (2.0 * de) / self.exposure yyy = numpy.append(yy, yy[-1]) sub.step(xx, yyy, where="post") sub.set_xscale("log") sub.set_yscale("log", nonposy="clip") sub.set_ylabel(r"Counts keV$^{-1}$ s$^{-1}$") # Now plot residuals sub1 = plt.subplot(gs[1]) res = (yy - y) / dy sub1.axhline(0, linestyle="--") sub1.errorbar(ec, res, yerr=1.0, xerr=de, fmt=".", capsize=0) sub1.set_xscale("log") # Match the x axis sub.set_xticks([]) sub.set_xlim([(ec - de).min(), (ec + de).max()]) sub1.set_xlim([(ec - de).min(), (ec + de).max()]) sub1.set_xlabel("Energy (keV)") return fig def _getModelCounts(self): # Get the folded model for this spectrum (this is the rate predicted, # in cts/s) folded = self.getFoldedModel() # Model is folded+background (i.e., we assume negligible errors on the # background) modelCounts = self.nuisanceParameters["InterCalib"].value * folded * self.exposure + self.bkgCounts return modelCounts def _computeLogLike(self, modelCounts): idx = modelCounts > 0 return numpy.sum( -modelCounts[idx] + self.counts[idx] * numpy.log(modelCounts[idx]) - logfactorial(self.counts[idx]) ) def get_log_like(self): """ Return the value of the log-likelihood with the current values for the parameters """ modelCounts = self._getModelCounts() logLike = self._computeLogLike(modelCounts) return logLike def get_nuisance_parameters(self): """ Return a list of nuisance parameter names. Return an empty list if there are no nuisance parameters """ return self.nuisanceParameters.keys() pass
class FermiGBMLike(pluginPrototype): def __init__(self,name,phafile,bkgfile,rspfile): ''' If the input files are PHA2 files, remember to specify the spectrum number, for example: FermiGBMLike("GBM","spectrum.pha{2}","bkgfile.bkg{2}","rspfile.rsp{2}") to load the second spectrum, second background spectrum and second response. ''' self.name = name #Check that all file exists notExistant = [] if(not os.path.exists(phafile.split("{")[0])): notExistant.append(phafile.split("{")[0]) if(not os.path.exists(bkgfile.split("{")[0])): notExistant.append(bkgfile.split("{")[0]) if(not os.path.exists(rspfile.split("{")[0])): notExistant.append(rspfile.split("{")[0]) if(len(notExistant)>0): for nt in notExistant: print("File %s does not exists!" %(nt)) raise IOError("One or more input file do not exist!") self.phafile = OGIPPHA(phafile,filetype='observed') self.exposure = self.phafile.getExposure() self.bkgfile = OGIPPHA(bkgfile,filetype="background") self.response = Response(rspfile) #Start with an empty mask (the user will overwrite it using the #setActiveMeasurement method) self.mask = numpy.asarray( numpy.ones(self.phafile.getRates().shape), numpy.bool) #Get the counts for this spectrum self.counts = ( self.phafile.getRates()[self.mask] * self.exposure ) #Check that counts is positive idx = (self.counts < 0) if(numpy.sum(idx) > 0): warnings.warn("The observed spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning) self.counts[idx] = 0 pass #Get the background counts for this spectrum self.bkgCounts = ( self.bkgfile.getRates()[self.mask] * self.exposure ) #Check that bkgCounts is positive idx = (self.bkgCounts < 0) if(numpy.sum(idx) > 0): warnings.warn("The background spectrum for %s " % self.name + "has negative channels! Fixing those to zero.", RuntimeWarning) self.bkgCounts[idx] = 0 #Check that the observed counts are positive idx = self.counts < 0 if numpy.sum( idx ) > 0: raise RuntimeError("Negative counts in observed spectrum %s. Data are corrupted." % ( phafile )) #Keep a copy which will never be modified self.counts_backup = numpy.array(self.counts,copy=True) self.bkgCounts_backup = numpy.array(self.bkgCounts,copy=True) #Effective area correction is disabled by default, i.e., #the nuisance parameter is fixed to 1 self.nuisanceParameters = {} self.nuisanceParameters['InterCalib'] = Parameter("InterCalib",1,0.9,1.1,0.01,fixed=True,nuisance=True) pass def useIntercalibrationConst(self,factorLowBound=0.9,factorHiBound=1.1): self.nuisanceParameters['InterCalib'].free() self.nuisanceParameters['InterCalib'].setBounds(factorLowBound,factorHiBound) #Check that the parameter is within the provided bounds value = self.nuisanceParameters['InterCalib'].value if( value < factorLowBound ): warnings.warn("The intercalibration constant was %s, lower than the provided lower bound." %(value,factorLowBound) + " Setting it equal to the lower bound") self.nuisanceParameters['InterCalib'].setValue(float(factorLowBound)) if( value > factorHiBound): warnings.warn("The intercalibration constant was %s, larger than the provided hi bound." %(value,factorHiBound) + " Setting it equal to the hi bound") self.nuisanceParameters['InterCalib'].setValue(float(factorHiBound)) def fixIntercalibrationConst(self,value=None): if(value is not None): #Fixing the constant to the provided value self.nuisanceParameters['InterCalib'].setValue(float(value)) else: #Do nothing, i.e., leave the constant to the value #it currently has pass self.nuisanceParameters['InterCalib'].fix() def setActiveMeasurements(self,*args): '''Set the measurements to be used during the analysis. Use as many ranges as you need, specified as 'emin-emax'. Energies are in keV. Example: setActiveMeasurements('10-12.5','56.0-100.0') which will set the energy range 10-12.5 keV and 56-100 keV to be used in the analysis''' #To implelemnt this we will use an array of boolean index, #which will filter #out the non-used channels during the logLike #Now build the mask: values for which the mask is 0 will be masked mask = numpy.zeros(self.phafile.getRates().shape) for arg in args: ee = map(float,arg.replace(" ","").split("-")) emin,emax = sorted(ee) idx1 = self.response.energyToChannel(emin) idx2 = self.response.energyToChannel(emax) mask[idx1:idx2+1] = True pass self.mask = numpy.array(mask,numpy.bool) self.counts = self.counts_backup[self.mask] self.bkgCounts = self.bkgCounts_backup[self.mask] print("Now using %s channels out of %s" %( numpy.sum(self.mask), self.phafile.getRates().shape[0] ) ) pass def getName(self): ''' Return a name for this dataset (likely set during the constructor) ''' return self.name pass def setModel(self,likelihoodModel): ''' Set the model to be used in the joint minimization. ''' self.likelihoodModel = likelihoodModel nPointSources = self.likelihoodModel.getNumberOfPointSources() #This is a wrapper which iterates over all the point sources and get #the fluxes #We assume there are no extended sources, since the GBM cannot handle them def diffFlux(energies): fluxes = self.likelihoodModel.getPointSourceFluxes(0,energies) #If we have only one point source, this will never be executed for i in range(1, nPointSources): fluxes += self.likelihoodModel.getPointSourceFluxes(i,energies) return fluxes self.diffFlux = diffFlux #The following integrates the diffFlux function using Simpson's rule #This assume that the intervals e1,e2 are all small, which is guaranteed #for any reasonable response matrix, given that e1 and e2 are Monte-Carlo #energies. It also assumes that the function is smooth in the interval #e1 - e2 and twice-differentiable, again reasonable on small intervals for #decent models. It might fail for models with too sharp features, smaller #than the size of the monte carlo interval. def integral(e1,e2): #Simpson's rule return (e2 - e1) / 6.0 * ( self.diffFlux(e1) + 4 * self.diffFlux( (e1+e2) / 2.0 ) + self.diffFlux(e2) ) self.response.setFunction( diffFlux, integral) pass def innerFit(self): #Effective area correction if(self.nuisanceParameters['InterCalib'].isFree()): #A true fit would be an overkill, and slow #Just sample a 100 values and choose the minimum values = numpy.linspace(self.nuisanceParameters['InterCalib'].minValue, self.nuisanceParameters['InterCalib'].maxValue, 100) #I do not use getLogLike so I can compute only once the folded model #(which is not going to change during the inner fit) folded = self.getFoldedModel() modelCounts = folded * self.exposure def fitfun(cons): self.nuisanceParameters['InterCalib'].setValue( cons ) return (-1) * self._computeLogLike(self.nuisanceParameters['InterCalib'].value * modelCounts + self.bkgCounts) logLval = map(fitfun, values) idx = numpy.argmax(logLval) self.nuisanceParameters['InterCalib'].setValue(values[idx]) #return logLval[idx] #Now refine with minuit parameters = collections.OrderedDict() parameters[ (self.name, 'InterCalib') ] = self.nuisanceParameters['InterCalib'] minimizer = minimization.iMinuitMinimizer(fitfun, parameters) bestFit, mlogLmin = minimizer.minimize() return mlogLmin * (-1) else: return self.getLogLike() def getFoldedModel(self): #Get the folded model for this spectrum #(this is the rate predicted, in cts/s) return self.response.convolve()[self.mask] def getModelAndData(self): e1,e2 = (self.response.ebounds[:,0], self.response.ebounds[:,1]) return ( self.response.convolve()[self.mask] * self.exposure + self.bkgCounts, e1[self.mask], e2[self.mask], self.counts ) def _getModelCounts(self): #Get the folded model for this spectrum (this is the rate predicted, #in cts/s) folded = self.getFoldedModel() #Model is folded+background (i.e., we assume negligible errors on the #background) modelCounts = self.nuisanceParameters['InterCalib'].value * folded * self.exposure + self.bkgCounts return modelCounts def _computeLogLike(self, modelCounts): return numpy.sum(- modelCounts + self.counts * numpy.log(modelCounts) - logfactorial(self.counts) ) def getLogLike(self): ''' Return the value of the log-likelihood with the current values for the parameters ''' modelCounts = self._getModelCounts() logLike = self._computeLogLike( modelCounts ) return logLike def getNuisanceParameters(self): ''' Return a list of nuisance parameter names. Return an empty list if there are no nuisance parameters ''' return self.nuisanceParameters.keys() pass