def Ts2(self, srcName, reoptimize=False, approx=True, tol=None, MaxIterations=10, verbosity=0): """Computes the TS value for a source indicated by "srcName." If "reoptimize=True" is selected this function will reoptimize the model up to "MaxIterations" given the tolerance "tol" (default is the tolerance selected for the overall fit). If "appox=True" is selected (the default) it will renormalize the model (see _renorm). """ saved_state = LikelihoodState(self) if verbosity > 0: print("*** Start Ts_dl ***") source_attributes = self.getExtraSourceAttributes() self.logLike.syncParams() src = self.logLike.getSource(srcName) self._ts_src = src freeParams = pyLike.DoubleVector() self.logLike.getFreeParamValues(freeParams) logLike1 = self.logLike.value() self.scaleSource(srcName, 1E-10) logLike0 = self.logLike.value() if tol is None: tol = self.tol if reoptimize: if verbosity > 0: print("** Do reoptimize") optFactory = pyLike.OptimizerFactory_instance() myOpt = optFactory.create(self.optimizer, self.logLike) Niter = 1 while Niter <= MaxIterations: try: myOpt.find_min(0, tol) break except RuntimeError as e: print(e) if verbosity > 0: print("** Iteration :", Niter) Niter += 1 else: if approx: try: self._renorm() except ZeroDivisionError: pass self.logLike.syncParams() logLike0 = max(self.logLike.value(), logLike0) Ts_value = 2 * (logLike1 - logLike0) self.scaleSource(srcName, 1E10) self.logLike.setFreeParamValues(freeParams) self.model = SourceModel(self.logLike) for src in source_attributes: self.model[src].__dict__.update(source_attributes[src]) saved_state.restore() self.logLike.value() return Ts_value
def _scan_extension(self, name, **kwargs): saved_state = LikelihoodState(self.like) if not hasattr(self.components[0].like.logLike, 'setSourceMapImage'): loglike = self._scan_extension_pylike(name, **kwargs) else: loglike = self._scan_extension_fast(name, **kwargs) saved_state.restore() return loglike
def _scan_position(self, name, **kwargs): saved_state = LikelihoodState(self.like) skydir = kwargs.pop('skydir', self.roi[name].skydir) scan_cdelt = kwargs.pop('scan_cdelt', 0.02) nstep = kwargs.pop('nstep', 5) use_cache = kwargs.get('use_cache', True) use_pylike = kwargs.get('use_pylike', False) optimizer = kwargs.get('optimizer', {}) # Fit without source self.zero_source(name, loglevel=logging.DEBUG) fit_output_nosrc = self._fit(loglevel=logging.DEBUG, **optimizer) self.unzero_source(name, loglevel=logging.DEBUG) saved_state.restore() self.free_norm(name, loglevel=logging.DEBUG) lnlmap = WcsNDMap.create(skydir=skydir, binsz=scan_cdelt, npix=(nstep, nstep), coordsys=wcs_utils.get_coordsys(self.geom.wcs)) src = self.roi.copy_source(name) if use_cache and not use_pylike: self._create_srcmap_cache(src.name, src) coord = MapCoord.create(lnlmap.geom.get_coord(flat=True), coordsys=lnlmap.geom.coordsys) scan_skydir = coord.skycoord.icrs for lon, lat, ra, dec in zip(coord.lon, coord.lat, scan_skydir.ra.deg, scan_skydir.dec.deg): spatial_pars = {'ra': ra, 'dec': dec} self.set_source_morphology(name, spatial_pars=spatial_pars, use_pylike=use_pylike) fit_output = self._fit(loglevel=logging.DEBUG, **optimizer) lnlmap.set_by_coord((lon, lat), fit_output['loglike']) self.set_source_morphology(name, spatial_pars=src.spatial_pars, use_pylike=use_pylike) saved_state.restore() lnlmap.data -= fit_output_nosrc['loglike'] tsmap = WcsNDMap(lnlmap.geom, 2.0 * lnlmap.data) self._clear_srcmap_cache() return tsmap, fit_output_nosrc['loglike']
def _scan_position(self, name, **kwargs): saved_state = LikelihoodState(self.like) skydir = kwargs.pop('skydir', self.roi[name].skydir) scan_cdelt = kwargs.pop('scan_cdelt', 0.02) nstep = kwargs.pop('nstep', 5) use_cache = kwargs.get('use_cache', True) use_pylike = kwargs.get('use_pylike', False) optimizer = kwargs.get('optimizer', {}) # Fit without source self.zero_source(name, loglevel=logging.DEBUG) fit_output_nosrc = self._fit(loglevel=logging.DEBUG, **optimizer) self.unzero_source(name, loglevel=logging.DEBUG) saved_state.restore() self.free_norm(name, loglevel=logging.DEBUG) lnlmap = Map.create(skydir, scan_cdelt, (nstep, nstep), coordsys=wcs_utils.get_coordsys(self._skywcs)) src = self.roi.copy_source(name) if use_cache and not use_pylike: self._create_srcmap_cache(src.name, src) scan_skydir = lnlmap.get_pixel_skydirs().transform_to('icrs') loglike = [] for ra, dec in zip(scan_skydir.ra.deg, scan_skydir.dec.deg): spatial_pars = {'ra': ra, 'dec': dec} self.set_source_morphology(name, spatial_pars=spatial_pars, use_pylike=use_pylike) fit_output = self._fit(loglevel=logging.DEBUG, **optimizer) loglike += [fit_output['loglike']] self.set_source_morphology(name, spatial_pars=src.spatial_pars, use_pylike=use_pylike) saved_state.restore() lnlmap.data = np.array(loglike).reshape((nstep, nstep)).T lnlmap.data -= fit_output_nosrc['loglike'] tsmap = Map(2.0 * lnlmap.data, lnlmap.wcs) self._clear_srcmap_cache() return tsmap, fit_output_nosrc['loglike']
def __init__(self, like, sourcesOfInterest=None, optimizer="Minuit", tol=1e-8, chatter=3, freeze_nuisance_sources_immediately = False, freeze_nuisance_sources_after_optimize = False, nuisance_npred_frac_max = 0.05, nuisance_soi_sep_min_deg = 5.0, calculate_full_ts_for_all = False): self.ver = "$Id: ROILikelihoodOptimizer.py 2795 2011-05-06 14:48:45Z sfegan $" self.res = {} self._like = like self._original_state = LikelihoodState(self._like) self._chatter = 0 self._SOI = [] self._freeze_nuisance_immediately = freeze_nuisance_sources_immediately self._freeze_nuisance_after_optimize = \ freeze_nuisance_sources_after_optimize self._nuisance_npred_frac_max = nuisance_npred_frac_max self._nuisance_soi_sep_min_deg = nuisance_soi_sep_min_deg self._calculate_full_ts_for_all = calculate_full_ts_for_all if sourcesOfInterest is not None: if type(sourcesOfInterest) != list: sourcesOfInterest = [ sourcesOfInterest ] for s in sourcesOfInterest: if s in like.sourceNames(): self._SOI.append(s) else: raise RuntimeError("Invalid source of interest: " + s) if optimizer: self._like.optimizer = optimizer if tol: self._like.tol = tol if chatter: self._chatter = chatter
def Likelihood(like1, modelout, optimizer, statistic, specfile, results, plot, slist, optmodel=(False, ), SkipUL=False, Bayes=False, binedge=None, sedN=None, dellist=None): if statistic != 'BINNED': print '%s method is not implemented in this script' % statistic return None if dellist is not None: for sdel in dellist: if sdel in like1.model.srcNames: like1.deleteSource(sdel) print '%s is deleted' % sdel like1.syncSrcParams() if optmodel[0]: OptimizeModel(like1, slist=slist, TSmax=optmodel[1]) like1.optObject = GetOptimizer(like1, optimizer) if like1.optimizer != optimizer: like1.optimizer = optimizer if isinstance(like1, SummedLikelihood): # not needed, results aren't changed for l in like1.components: l.optimizer = optimizer nloop = 0 fitxml = modelout + '_fit%d' % (nloop + 1) while nloop < 3 and not Fit(like1, fitxml): print print nloop += 1 fitxml = modelout + '_fit%d' % (nloop + 1) like1.optObject = GetOptimizer(like1, optimizer) gc.collect() if nloop == 3: print 'could not converge' fitxml = modelout + '_fit3' # pkl=results+'.pkl' # pkl=pkl.replace('.dat.pkl','.pkl') # with open(pkl, mode='wb') as f: # pickle.dump(like, f) # pickle.dump(likeobj, f) if isinstance(like1, SummedLikelihood): is_esame = True for l in like1.components[1:]: is_esame = (is_esame and np.allclose(like1.components[0].energies, l.energies)) if is_esame: E = like1.components[0].energies else: E = (min([l.energies[0] for l in like1.components]), max([l.energies[-1] for l in like1.components])) else: E = like1.energies emin = E[0] emax = E[-1] print '\nComputing TS values for each extended source\n' print 'Photon fluxes are computed for the energy range ' + repr( emin) + ' to ' + repr(emax) + ' MeV\n' dic = OrderedDict() free = [] for sname in like1.model.srcNames: flag = False src = OrderedDict() func = like1.model[sname].funcs['Spectrum'] for param in func.paramNames: flag = (flag or func.params[param].isFree()) if func.params[param].isFree(): free.append('%s:%s' % (sname, param)) src[param] = (func[param], func.params[param].error()) else: src[param] = func[param] flux = like1.flux(sname, emin=emin, emax=emax) enflux = like1.energyFlux(sname, emin=emin, emax=emax) if flag: if sname.find('gll') < 0 and sname.find('iso') < 0: for param in func.paramNames: src['scale ' + param] = func.params[param].getScale() src['Spectrum'] = func.genericName() src['TS value'] = like1.Ts(sname) eflx = like1.fluxError(sname, emin=emin, emax=emax) eenflx = like1.energyFluxError(sname, emin=emin, emax=emax) src['Flux'] = (flux, eflx) src['EnFlux'] = (enflux, eenflx) else: src['Flux'] = flux src['EnFlux'] = enflux if like1.model[sname].getType() == 'Diffuse': tmplist = [] nbin = len(E) - 1 for ie in range(nbin): tmplist.append(like1.flux(sname, emin=E[ie], emax=E[ie + 1])) src['Diff Flux'] = tmplist dic[sname] = src dic['Energies'] = E if len(free) > 1: dic['Free Parameters'] = free dic['Covariance'] = like1.covariance dic['-LogLike'] = like1() WriteResult(dic, results) PrintResult(dic) ''' Calculate Upper Limit. You can choose bayesian or frequentist algorithm. ''' if SkipUL or len(slist) == 0: print 'skip UL calculation' elif Bayes: ul = {} ullist.append(ul) for sname in slist: print sname flux_ul, ulresults = calc_int(like1, sname, emin=emin, emax=emax) try_ul_calc = 1 par = like1.normPar(sname) while flux_ul == -1 and try_ul_calc <= 10: par.setBounds(par.getBounds()[0], par.getBounds()[1] * 10) like1.syncSrcParams(sname) flux_ul, ulresults = calc_int(like1, sname, emin=emin, emax=emax) try_ul_calc += 1 if flux_ul == -1: print 'could not compute upper limit' else: print '%lg ph/cm^2/s for emin=%.1f, emax=%.1f (Bayesian UL)' % ( flux_ul, ulresults['flux_emin'], ulresults['flux_emax']) ul[sname] = ulresults dic[sname]['dNdE UL'] = ulresults['ul_value'] * par.getScale() ## dN/dE UL at a reference enrergy (ph/cm^2/s) dic[sname]['Flux UL'] = flux_ul saved_state = LikelihoodState(like1) par.setValue(ulresults['ul_value']) dic[sname]['EnFlux UL'] = like1.energyFlux(sname, emin, emax) saved_state.restore() dic[sname]['UL algo'] = 'bayesian' WriteResult(dic, results) else: ul = UpperLimits(like1) ullist.append(ul) for sname in slist: print sname try: flux_ul, pref_ul = ul[sname].compute(emin=emin, emax=emax) print ul[sname].results[-1] dic[sname]['dNdE UL'] = pref_ul * like1.normPar( sname).getScale() dic[sname]['Flux UL'] = flux_ul saved_state = LikelihoodState(like1) like1.normPar(sname).setValue(pref_ul) dic[sname]['EnFlux UL'] = like1.energyFlux(sname, emin, emax) saved_state.restore() dic[sname]['UL algo'] = 'frequentist' dic[sname]['UL dlogL'] = ul[sname].results[-1].delta WriteResult(dic, results) except RuntimeError: print(traceback.format_exc()) print('could not compute upper limit') # with open(pkl, mode='ab') as f: # pickle.dump(ul, f) try: if specfile != 'None' and specfile != '' and specfile is not None: like1.writeCountsSpectra(specfile, len(E) - 1) except NotImplementedError as e: print 'NotImplementedError:', e except RuntimeError: print(traceback.format_exc()) cnt = 0. for name in like1.model.srcNames: cnt += like1.NpredValue(name) print '\nTotal number of observed counts:', int(like1.total_nobs()) print 'Total number of model events:', cnt print '\n-log(Likelihood):', like1() print '\nWriting fitted model to', modelout # like1.logLike.writeXml(modelout) os.rename(fitxml, modelout) ''' Calculate Fermi SED data points. ''' if binedge is not None: ul_alg = 'bayesian' if Bayes else 'frequentist' print '\n%s Upper Limit will be calculated' % ul_alg print 'calculating SED...' for sname in slist: print sname func = like1.model[sname].funcs['Spectrum'] if func.genericName() == 'PowerLaw': idx = func.params['Index'].getTrueValue() elif flagsed2 and func.genericName() == 'LogParabola': print 'LogParabola' idx = [] ebin = np.asarray(binedge) ebin = np.sqrt(ebin[:-1] * ebin[1:]) Alpha = func.params['alpha'].getTrueValue() Beta = func.params['beta'].getTrueValue() Eb = func.params['Eb'].getTrueValue() for valE in ebin: idx.append(-(Alpha + 2 * Beta * np.log(valE / Eb))) print 'approximate PL indices:', idx elif flagsed2 and func.genericName() == 'PLSuperExpCutoff': print 'PLSuperExpCutoff' idx = [] ebin = np.asarray(binedge) ebin = np.sqrt(ebin[:-1] * ebin[1:]) Index1 = func.params['Index1'].getTrueValue() Index2 = func.params['Index2'].getTrueValue() Cutoff = func.params['Cutoff'].getTrueValue() for valE in ebin: idx.append(Index1 - Index2 * pow(valE / Cutoff, Index2)) print 'approximate PL indices:', idx else: idx = -2. GetSED(like1, sname, index=idx, be=binedge, ul_alg=ul_alg, fnheader=sedN) print 'Done!' ''' Plot count graph. ''' if plot: try: print '' like1.setPlotter('mpl') like1.plot() print '' except: print 'could not plot' return dic
def _localize(self, name, **kwargs): nstep = kwargs.get('nstep') dtheta_max = kwargs.get('dtheta_max') update = kwargs.get('update', True) prefix = kwargs.get('prefix', '') use_cache = kwargs.get('use_cache', False) free_background = kwargs.get('free_background', False) free_radius = kwargs.get('free_radius', None) saved_state = LikelihoodState(self.like) if not free_background: self.free_sources(free=False, loglevel=logging.DEBUG) if free_radius is not None: diff_sources = [s.name for s in self.roi.sources if s.diffuse] skydir = self.roi[name].skydir free_srcs = [s.name for s in self.roi.get_sources(skydir=skydir, distance=free_radius, exclude=diff_sources)] self.free_sources_by_name(free_srcs, pars='norm', loglevel=logging.DEBUG) src = self.roi.copy_source(name) skydir = src.skydir skywcs = self._skywcs src_pix = skydir.to_pixel(skywcs) fit0 = self._fit_position_tsmap(name, prefix=prefix, dtheta_max=dtheta_max, zmin=-3.0, use_pylike=False) self.logger.debug('Completed localization with TS Map.\n' '(ra,dec) = (%10.4f,%10.4f) ' '(glon,glat) = (%10.4f,%10.4f)', fit0['ra'], fit0['dec'], fit0['glon'], fit0['glat']) # Fit baseline (point-source) model self.free_norm(name) fit_output = self._fit(loglevel=logging.DEBUG, ** kwargs.get('optimizer', {})) # Save likelihood value for baseline fit loglike0 = fit_output['loglike'] self.logger.debug('Baseline Model Likelihood: %f', loglike0) o = defaults.make_default_tuple(defaults.localize_output) o.name = name o.config = kwargs o.fit_success = True o.loglike_base = loglike0 o.loglike_loc = np.nan o.dloglike_loc = np.nan if fit0['fit_success']: scan_cdelt = 2.0 * fit0['pos_r95'] / (nstep - 1.0) else: scan_cdelt = np.abs(skywcs.wcs.cdelt[0]) self.logger.debug('Refining localization search to ' 'region of width: %.4f deg', scan_cdelt * nstep) fit1 = self._fit_position_scan(name, skydir=fit0['skydir'], scan_cdelt=scan_cdelt, **kwargs) o.loglike_loc = fit1['loglike'] o.dloglike_loc = o.loglike_loc - o.loglike_base o.tsmap = fit0.pop('tsmap') o.tsmap_peak = fit1.pop('tsmap') # o.update(fit1) # Best fit position and uncertainty from fit to TS map o.fit_init = fit0 # Best fit position and uncertainty from pylike scan o.fit_scan = fit1 o.update(fit1) cdelt0 = np.abs(skywcs.wcs.cdelt[0]) cdelt1 = np.abs(skywcs.wcs.cdelt[1]) pix = fit1['skydir'].to_pixel(skywcs) o.pos_offset = skydir.separation(fit1['skydir']).deg o.xpix = float(pix[0]) o.ypix = float(pix[1]) o.deltax = (o.xpix - src_pix[0]) * cdelt0 o.deltay = (o.ypix - src_pix[1]) * cdelt1 o.ra_preloc = skydir.ra.deg o.dec_preloc = skydir.dec.deg o.glon_preloc = skydir.galactic.l.deg o.glat_preloc = skydir.galactic.b.deg if o.pos_offset > dtheta_max: o.fit_success = False self.logger.info('Localization completed with new position:\n' '( ra, dec) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' '(glon,glat) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' 'offset = %8.4f r68 = %8.4f r95 = %8.4f r99 = %8.4f', o.ra, o.ra_err, o.dec, o.dec_err, o.glon, o.glon_err, o.glat, o.glat_err, o.pos_offset, o.pos_r68, o.pos_r95, o.pos_r99) if not o.fit_success: self.logger.warning('Fit to localization contour failed.') elif not o.fit_inbounds: self.logger.warning('Best-fit position outside of search region.') else: self.logger.info('Localization succeeded.') if update and ((not o.fit_success) or (not o.fit_inbounds)): self.logger.warning( 'Localization failed. Keeping existing position.') if update and o.fit_success and o.fit_inbounds: self.logger.info('Updating source %s ' 'to localized position.', name) src = self.delete_source(name) src.set_position(fit1['skydir']) self.add_source(name, src, free=True) fit_output = self.fit(loglevel=logging.DEBUG) o.loglike_loc = fit_output['loglike'] o.dloglike_loc = o.loglike_loc - o.loglike_base src = self.roi.get_source_by_name(name) self.logger.info('LogLike: %12.3f DeltaLogLike: %12.3f', o.loglike_loc, o.dloglike_loc) src['glon_err'] = o.glon_err src['glat_err'] = o.glat_err src['ra_err'] = o.glon_err src['dec_err'] = o.glat_err src['pos_err'] = o.pos_err src['pos_err_semimajor'] = o.pos_err_semimajor src['pos_err_semiminor'] = o.pos_err_semiminor src['pos_r68'] = o.pos_r68 src['pos_r95'] = o.pos_r95 src['pos_r99'] = o.pos_r99 src['pos_angle'] = o.pos_angle src['pos_gal_cov'] = o.pos_gal_cov src['pos_gal_corr'] = o.pos_gal_corr src['pos_cel_cov'] = o.pos_cel_cov src['pos_cel_corr'] = o.pos_cel_corr else: saved_state.restore() self._sync_params(name) self._update_roi() return o
def _make_sed(self, name, **config): bin_index = config['bin_index'] use_local_index = config['use_local_index'] free_background = config['free_background'] free_radius = config['free_radius'] ul_confidence = config['ul_confidence'] cov_scale = config['cov_scale'] loge_bins = config['loge_bins'] if not loge_bins or loge_bins is None: loge_bins = self.log_energies else: loge_bins = np.array(loge_bins) nbins = len(loge_bins) - 1 max_index = 5.0 min_flux = 1E-30 npts = self.config['gtlike']['llscan_npts'] loge_bounds = self.loge_bounds # Output Dictionary o = {'name': name, 'loge_min': loge_bins[:-1], 'loge_max': loge_bins[1:], 'loge_ctr': 0.5 * (loge_bins[:-1] + loge_bins[1:]), 'loge_ref': 0.5 * (loge_bins[:-1] + loge_bins[1:]), 'e_min': 10 ** loge_bins[:-1], 'e_max': 10 ** loge_bins[1:], 'e_ctr': 10 ** (0.5 * (loge_bins[:-1] + loge_bins[1:])), 'e_ref': 10 ** (0.5 * (loge_bins[:-1] + loge_bins[1:])), 'ref_flux': np.zeros(nbins), 'ref_eflux': np.zeros(nbins), 'ref_dnde': np.zeros(nbins), 'ref_dnde_e_min': np.zeros(nbins), 'ref_dnde_e_max': np.zeros(nbins), 'ref_e2dnde': np.zeros(nbins), 'ref_npred': np.zeros(nbins), 'norm': np.zeros(nbins), 'flux': np.zeros(nbins), 'eflux': np.zeros(nbins), 'dnde': np.zeros(nbins), 'e2dnde': np.zeros(nbins), 'index': np.zeros(nbins), 'npred': np.zeros(nbins), 'ts': np.zeros(nbins), 'loglike': np.zeros(nbins), 'norm_scan': np.zeros((nbins, npts)), 'dloglike_scan': np.zeros((nbins, npts)), 'loglike_scan': np.zeros((nbins, npts)), 'fit_quality': np.zeros(nbins), 'fit_status': np.zeros(nbins), 'correlation': {}, 'model_flux': {}, 'config': config } for t in ['norm', 'flux', 'eflux', 'dnde', 'e2dnde']: o['%s_err' % t] = np.zeros(nbins) * np.nan o['%s_err_hi' % t] = np.zeros(nbins) * np.nan o['%s_err_lo' % t] = np.zeros(nbins) * np.nan o['%s_ul95' % t] = np.zeros(nbins) * np.nan o['%s_ul' % t] = np.zeros(nbins) * np.nan saved_state = LikelihoodState(self.like) source = self.components[0].like.logLike.getSource(str(name)) # Perform global spectral fit self._latch_free_params() self.free_sources(False, pars='shape', loglevel=logging.DEBUG) self.free_source(name, pars=config.get('free_pars', None), loglevel=logging.DEBUG) fit_output = self.fit(loglevel=logging.DEBUG, update=False, min_fit_quality=2) o['model_flux'] = self.bowtie(name) spectral_pars = gtutils.get_function_pars_dict(source.spectrum()) o['SpectrumType'] = self.roi[name]['SpectrumType'] o.update(model_utils.pars_dict_to_vectors(o['SpectrumType'], spectral_pars)) param_names = gtutils.get_function_par_names(o['SpectrumType']) npar = len(param_names) o['param_covariance'] = np.empty((npar, npar), dtype=float) * np.nan pmask0 = np.empty(len(fit_output['par_names']), dtype=bool) pmask0.fill(False) pmask1 = np.empty(npar, dtype=bool) pmask1.fill(False) for i, pname in enumerate(param_names): for j, pname2 in enumerate(fit_output['par_names']): if name != fit_output['src_names'][j]: continue if pname != pname2: continue pmask0[j] = True pmask1[i] = True src_cov = fit_output['covariance'][pmask0, :][:, pmask0] o['param_covariance'][np.ix_(pmask1, pmask1)] = src_cov o['param_correlation'] = utils.cov_to_correlation( o['param_covariance']) for i, pname in enumerate(param_names): o['param_covariance'][i, :] *= spectral_pars[pname]['scale'] o['param_covariance'][:, i] *= spectral_pars[pname]['scale'] self._restore_free_params() self.logger.info('Fitting SED') # Setup background parameters for SED self.free_sources(False, pars='shape') self.free_norm(name) if not free_background: self.free_sources(free=False, loglevel=logging.DEBUG) if free_radius is not None: diff_sources = [s.name for s in self.roi.sources if s.diffuse] skydir = self.roi[name].skydir free_srcs = [s.name for s in self.roi.get_sources(skydir=skydir, distance=free_radius, exclude=diff_sources)] self.free_sources_by_name(free_srcs, pars='norm', loglevel=logging.DEBUG) if cov_scale is not None: self._latch_free_params() self.zero_source(name) self.fit(loglevel=logging.DEBUG, update=False) srcNames = list(self.like.sourceNames()) srcNames.remove(name) self.constrain_norms(srcNames, cov_scale) self.unzero_source(name) self._restore_free_params() # Precompute fluxes in each bin from global fit gf_bin_flux = [] gf_bin_index = [] for i, (logemin, logemax) in enumerate(zip(loge_bins[:-1], loge_bins[1:])): emin = 10 ** logemin emax = 10 ** logemax delta = 1E-5 f = self.like[name].flux(emin, emax) f0 = self.like[name].flux(emin * (1 - delta), emin * (1 + delta)) f1 = self.like[name].flux(emax * (1 - delta), emax * (1 + delta)) if f0 > min_flux and f1 > min_flux: g = 1 - np.log10(f0 / f1) / np.log10(emin / emax) gf_bin_index += [g] gf_bin_flux += [f] else: gf_bin_index += [max_index] gf_bin_flux += [min_flux] old_spectrum = source.spectrum() old_pars = copy.deepcopy(self.roi[name].spectral_pars) old_type = self.roi[name]['SpectrumType'] spectrum_pars = { 'Prefactor': {'value': 1.0, 'scale': 1E-13, 'min': 1E-10, 'max': 1E10, 'free': True}, 'Index': {'value': 2.0, 'scale': -1.0, 'min': 0.0, 'max': 5.0, 'free': False}, 'Scale': {'value': 1E3, 'scale': 1.0, 'min': 1., 'max': 1E6, 'free': False}, } self.set_source_spectrum(str(name), 'PowerLaw', spectrum_pars=spectrum_pars, update_source=False) src_norm_idx = -1 free_params = self.get_params(True) for j, p in enumerate(free_params): if not p['is_norm']: continue if p['is_norm'] and p['src_name'] == name: src_norm_idx = j o['correlation'][p['src_name']] = np.zeros(nbins) * np.nan self._fitcache = None for i, (logemin, logemax) in enumerate(zip(loge_bins[:-1], loge_bins[1:])): logectr = 0.5 * (logemin + logemax) emin = 10 ** logemin emax = 10 ** logemax ectr = 10 ** logectr ectr2 = ectr**2 saved_state_bin = LikelihoodState(self.like) if use_local_index: o['index'][i] = -min(gf_bin_index[i], max_index) else: o['index'][i] = -bin_index self.set_norm(name, 1.0, update_source=False) self.set_parameter(name, 'Index', o['index'][i], scale=1.0, update_source=False) self.like.syncSrcParams(str(name)) ref_flux = self.like[name].flux(emin, emax) o['ref_flux'][i] = self.like[name].flux(emin, emax) o['ref_eflux'][i] = self.like[name].energyFlux(emin, emax) o['ref_dnde'][i] = self.like[name].spectrum()(pyLike.dArg(ectr)) o['ref_dnde_e_min'][i] = self.like[ name].spectrum()(pyLike.dArg(emin)) o['ref_dnde_e_max'][i] = self.like[ name].spectrum()(pyLike.dArg(emax)) o['ref_e2dnde'][i] = o['ref_dnde'][i] * ectr2 cs = self.model_counts_spectrum( name, logemin, logemax, summed=True) o['ref_npred'][i] = np.sum(cs) normVal = self.like.normPar(name).getValue() flux_ratio = gf_bin_flux[i] / ref_flux newVal = max(normVal * flux_ratio, 1E-10) self.set_norm(name, newVal, update_source=False) self.set_norm_bounds(name, [newVal * 1E-6, newVal * 1E4]) self.like.syncSrcParams(str(name)) self.free_norm(name) self.logger.debug('Fitting %s SED from %.0f MeV to %.0f MeV' % (name, emin, emax)) self.set_energy_range(logemin, logemax) fit_output = self._fit(**config['optimizer']) free_params = self.get_params(True) for j, p in enumerate(free_params): if not p['is_norm']: continue o['correlation'][p['src_name']][i] = \ fit_output['correlation'][src_norm_idx, j] o['fit_quality'][i] = fit_output['fit_quality'] o['fit_status'][i] = fit_output['fit_status'] flux = self.like[name].flux(emin, emax) eflux = self.like[name].energyFlux(emin, emax) dnde = self.like[name].spectrum()(pyLike.dArg(ectr)) o['norm'][i] = flux / o['ref_flux'][i] o['flux'][i] = flux o['eflux'][i] = eflux o['dnde'][i] = dnde o['e2dnde'][i] = dnde * ectr2 cs = self.model_counts_spectrum(name, logemin, logemax, summed=True) o['npred'][i] = np.sum(cs) o['loglike'][i] = fit_output['loglike'] lnlp = self.profile_norm(name, logemin=logemin, logemax=logemax, savestate=True, reoptimize=True, npts=npts, optimizer=config['optimizer']) o['ts'][i] = max( 2.0 * (fit_output['loglike'] - lnlp['loglike'][0]), 0.0) o['loglike_scan'][i] = lnlp['loglike'] o['dloglike_scan'][i] = lnlp['dloglike'] o['norm_scan'][i] = lnlp['flux'] / ref_flux ul_data = utils.get_parameter_limits( lnlp['flux'], lnlp['dloglike']) o['norm_err_hi'][i] = ul_data['err_hi'] / ref_flux o['norm_err_lo'][i] = ul_data['err_lo'] / ref_flux if np.isfinite(ul_data['err_lo']): o['norm_err'][i] = 0.5 * (ul_data['err_lo'] + ul_data['err_hi']) / ref_flux else: o['norm_err'][i] = ul_data['err_hi'] / ref_flux o['norm_ul95'][i] = ul_data['ul'] / ref_flux ul_data = utils.get_parameter_limits(lnlp['flux'], lnlp['dloglike'], cl_limit=ul_confidence) o['norm_ul'][i] = ul_data['ul'] / ref_flux saved_state_bin.restore() for t in ['flux', 'eflux', 'dnde', 'e2dnde']: o['%s_err' % t] = o['norm_err'] * o['ref_%s' % t] o['%s_err_hi' % t] = o['norm_err_hi'] * o['ref_%s' % t] o['%s_err_lo' % t] = o['norm_err_lo'] * o['ref_%s' % t] o['%s_ul95' % t] = o['norm_ul95'] * o['ref_%s' % t] o['%s_ul' % t] = o['norm_ul'] * o['ref_%s' % t] self.set_energy_range(loge_bounds[0], loge_bounds[1]) self.set_source_spectrum(str(name), old_type, spectrum_pars=old_pars, update_source=False) saved_state.restore() self._sync_params(name) if cov_scale is not None: self.remove_priors() return o
def compute_curve(name,pwndata,phimin, phimax, fit_emin): """Function to compute the points TS vs (phi range)""" TS=np.zeros_like(phimin) print 'First, analyzing unphased data' roi=setup_pwn(name,pwndata,phase=[0,1], quiet=True, fit_emin=fit_emin) print 'bin edges:',roi.bin_edges print roi gtlike = roi_gtlike.Gtlike(roi) like=gtlike.like # for i in range(len(like.model.params)): # like.freeze(i) # like[like.par_index(name, 'Prefactor')].setFree(1) like.fit() like[like.par_index(name, 'Prefactor')].setBounds(1.0e-5,1.0e5) print like.model # freeze index of PWN index=like[like.par_index(name, 'Index')] index.setTrueValue(-2) index.setFree(1) # fix everything in the ROI except prefactor #for i in range(len(like.model.params)): # like.freeze(i) #like[like.par_index(name, 'Prefactor')].setFree(1) print like.model # now, freeze everything else in the ROI: saved_state = LikelihoodState(gtlike.like) print "----------------------------------Begin Loop---------------------------------------" for i,phase in enumerate(zip(phimin,phimax)): print 'Loop %4d/%4d: phase min=%.2f, max=%.2f' % (i+1,len(phimin),phase[0],phase[1]) roi=setup_pwn(name,pwndata,phase, quiet=True, fit_emin=fit_emin) gtlike = roi_gtlike.Gtlike(roi) print gtlike.like.model # give model the same parameters as global fit. saved_state.like = gtlike.like saved_state.restore() for i in range(len(gtlike.like.model.params)): gtlike.like.freeze(i) gtlike.like[gtlike.like.par_index(name, 'Prefactor')].setFree(1) gtlike.like[like.par_index(name, 'Prefactor')].setBounds(1.0e-5,1.0e5) gtlike.like.fit() # n.b. no need to reoptimize since only one parameter # is fit. TS[i] = gtlike.like.Ts(name,reoptimize=False) print 'phase=%s, TS=%s' % (phase,TS[i]) f=open("results_%s.yaml" % name,"w") yaml.dump(dict(TS=TS[0:i+1].tolist(), phimin=phimin[0:i+1].tolist(), phimax=phimax[0:i+1].tolist()), f) f.close() return TS,phimin,phimax
def calc_chi2(like, srcName, cl=0.95, verbosity=0, skip_global_opt=False, freeze_all=False, profile_optimizer = None, emin=100, emax=3e5, poi_values = []): """Calculate an integral upper limit by the profile likelihood (chi2) method. Description: Calculate an upper limit using the likelihood ratio test, i.e. by supposing the Likelihood is distributed as chi-squared of one degree of freedom and finding the point at which the it decreases by the required amount to get an upper limit at a certain confidence limit. This function first uses the optimizer to find the global minimum, then uses the new root finding algorithm to find the point at which the Likelihood decreases by the required amount. The background parameters can be frozen at their values found in the global minimum or optimized freely at each point. Inputs: like -- a binned or unbinned likelihood object which has the desired model. Be careful to freeze the index of the source for which the upper limit is being if you want to quote a limit with a fixed index. srcName -- the name of the source for which to compute the limit. cl -- probability level for the upper limit. verbosity -- verbosity level. A value of zero means no output will be written. With a value of one the function writes some values describing its progress, but the optimizers don't write anything. Values larger than one direct the optimizer to produce verbose output. skip_global_opt -- if the model is already at the global minimum value then you can direct the integrator to skip the initial step to find the minimum. If you specify this option and the model is NOT at the global minimum your results will likely be wrong. freeze_all -- freeze all other parameters at the values of the global minimum. profile_optimizer -- Alternative optimizer to use when computing the profile, after the global minimum has been found. Only set this if you want to use a different optimizer for calculating the profile than for calculating the global minimum. emin, emax -- Bounds on energy range over which the flux should be integrated. poi_values -- Points of interest: values of the normalization parameter corresponding to fluxes of interest to the user. The profile likelihood be evaluated at each of these values and the equivalent probability under the LRT returned in the vector \"results.poi_probs\". This parameter must be a vector, and can be empty. Outputs: (limit, results) limit -- the flux limit found. results -- a dictionary of additional results from the calculation, such as the value of the peak value etc. """ saved_state = LikelihoodState(like) ########################################################################### # # This function has 2 main components: # # 1) Find the global maximum of the likelihood function using ST # 2) Find the point at which it falls by the appropriate amount # ########################################################################### # Optimizer uses verbosity level one smaller than given here optverbosity = max(verbosity-1, 0) ########################################################################### # # 1) Find the global maximum of the likelihood function using ST # ########################################################################### par = like.normPar(srcName) fitstat = None if not skip_global_opt: # Make sure desired parameter is free during global optimization par.setFree(1) like.syncSrcParams(srcName) # Perform global optimization if verbosity: print "Finding global maximum" try: like.fit(optverbosity) fitstat = like.optObject.getRetCode() if verbosity and fitstat != 0: print "Minimizer returned with non-zero code: ",fitstat except RuntimeError: print "Failed to find global maximum, results may be wrong" pass pass original_optimizer = like.optimizer if profile_optimizer != None: like.optimizer = profile_optimizer # Store values of global fit maxval = -like() fitval = par.getValue() fiterr = par.error() limlo, limhi = par.getBounds() if verbosity: print "Maximum of %g with %s = %g +/- %g"\ %(-maxval,srcName,fitval,fiterr) # Freeze all other model parameters if requested (much faster!) if(freeze_all): for i in range(len(like.model.params)): like.model[i].setFree(0) like.syncSrcParams(like[i].srcName) # Freeze the parameter of interest par.setFree(0) like.syncSrcParams(srcName) # Set up the caches for the optimum values and nuisance parameters optvalue_cache = dict() nuisance_cache = dict() optvalue_cache[fitval] = maxval _cache_nuisance(fitval, like, nuisance_cache) # Test if all parameters are frozen (could be true if we froze # them above or if they were frozen in the user's model all_frozen = True for i in range(len(like.model.params)): if like.model[i].isFree(): all_frozen = False break ########################################################################### # # 2) Find the point at which the likelihood has fallen by the # appropriate amount # ########################################################################### delta_log_like = 0.5*scipy.stats.chi2.isf(1-2*(cl-0.5), 1) if verbosity: print "Finding limit (delta log Like=%g)"\ %(delta_log_like) [xunused, xlim, yunused, ylim, exact_root_evals, approx_root_evals] = \ _find_interval(like, par, srcName, all_frozen, maxval, fitval, limlo, limhi, delta_log_like, verbosity, like.tol, True, 5, optvalue_cache, nuisance_cache) if verbosity: print "Limit: %g (%d full fcn evals and %d approx)"\ %(xlim,exact_root_evals,approx_root_evals) ########################################################################### # # Evaluate the probabilities of the "points of interest" using the LRT # ########################################################################### poi_dlogL = []; poi_probs = []; for xval in poi_values: if(xval >= limhi): dlogL = None pval = 1.0 elif(xval <= limlo): dlogL = None pval = 0.0 else: dlogL = _loglike(xval, like, par, srcName, maxval, verbosity, all_frozen, optvalue_cache, nuisance_cache) if(xval<fitval): pval = 0.5*(1-scipy.stats.chi2.cdf(-2*dlogL,1)) else: pval = 0.5*(1+scipy.stats.chi2.cdf(-2*dlogL,1)) if verbosity: print "POI %g: Delta log Like = %g (Pr=%g)"%(xval,dlogL,pval) poi_probs.append(pval) poi_dlogL.append(dlogL) like.optimizer = original_optimizer ########################################################################### # # Calculate the integral flux at the upper limit parameter value # ########################################################################### # Set the parameter value that corresponds to the desired C.L. par.setValue(xlim) # Evaluate the flux corresponding to this upper limit. ul_flux = like[srcName].flux(emin, emax) saved_state.restore() # Pack up all the results results = dict(all_frozen = all_frozen, ul_frac = cl, ul_flux = ul_flux, ul_value = xlim, ul_loglike = maxval+ylim-delta_log_like, ul_dloglike = ylim-delta_log_like, peak_fitstatus = fitstat, peak_value = fitval, peak_dvalue = fiterr, peak_loglike = maxval, poi_values = poi_values, poi_probs = poi_probs, poi_dlogL = poi_dlogL, flux_emin = emin, flux_emax = emax) return ul_flux, results
def calc_int(like, srcName, cl=0.95, verbosity=0, skip_global_opt=False, be_very_careful=False, freeze_all=False, delta_log_like_limits = 10.0, profile_optimizer = None, emin=100, emax=3e5, poi_values = []): """Calculate an integral upper limit by direct integration. Description: Calculate an integral upper limit by integrating the likelihood function up to a point which contains a given fraction of the total probability. This is a fairly standard Bayesian approach to calculating upper limits, which assumes a uniform prior probability. The likelihood function is not assumed to be distributed as chi-squared. This function first uses the optimizer to find the global minimum, then uses the scipy.integrate.quad function to integrate the likelihood function with respect to one of the parameters. During the integration, the other parameters can be frozen at their values found in the global minimum or optimized freely at each point. Inputs: like -- a binned or unbinned likelihood object which has the desired model. Be careful to freeze the index of the source for which the upper limit is being if you want to quote a limit with a fixed index. srcName -- the name of the source for which to compute the limit. cl -- probability level for the upper limit. verbosity -- verbosity level. A value of zero means no output will be written. With a value of one the function writes some values describing its progress, but the optimizers don't write anything. Values larger than one direct the optimizer to produce verbose output. skip_global_opt -- if the model is already at the global minimum value then you can direct the integrator to skip the initial step to find the minimum. If you specify this option and the model is NOT at the global minimum your results will likely be wrong. be_very_careful -- direct the integrator to be even more careful in integrating the function, by telling it to use a higher tolerance and to specifically pay attention to the peak in the likelihood function. More evaluations of the integrand will be made, which WILL be slower and MAY result in a more accurate limit. NOT RECOMMENDED freeze_all -- freeze all other parameters at the values of the global minimum. delta_log_like_limits -- the limits on integration is defined by the region around the global maximum in which the log likelihood is close enough to the peak value. Too small a value will mean the integral does not include a significant amount of the likelihood function. Too large a value may make the integrator miss the peak completely and get a bogus answer (although the \"be_very_careful\" option will help here). profile_optimizer -- Alternative optimizer to use when computing the profile, after the global minimum has been found. Only set this if you want to use a different optimizer for calculating the profile than for calculating the global minimum. emin, emax -- Bounds on energy range over which the flux should be integrated. poi_values -- Points of interest: values of the normalization parameter corresponding to fluxes of interest to the user. The integrator will calculate the integral of the probability distribution to each of these values and return them in the vector \"results.poi_probs\". This parameter must be a vector, and can be empty. Outputs: (limit, results) limit -- the flux limit found. results -- a dictionary of additional results from the calculation, such as the value of the peak, the profile of the likelihood and two profile-likelihood upper-limits. """ saved_state = LikelihoodState(like) ########################################################################### # # This function has 4 main components: # # 1) Find the global maximum of the likelihood function using ST # 2) Define the integration limits by finding the points at which the # log likelihood has fallen by a certain amount # 3) Integrate the function using the QUADPACK adaptive integrator # 4) Calculate the upper limit by re-integrating the function using # the evaluations made by the adaptive integrator. Two schemes are # tried, splines to the function points and trapezoidal quadrature. # ########################################################################### # Optimizer uses verbosity level one smaller than given here optverbosity = max(verbosity-1, 0) ########################################################################### # # 1) Find the global maximum of the likelihood function using ST # ########################################################################### par = like.normPar(srcName) fitstat = None if not skip_global_opt: # Make sure desired parameter is free during global optimization par.setFree(1) like.syncSrcParams(srcName) # Perform global optimization if verbosity: print "Finding global maximum" try: like.fit(optverbosity) fitstat = like.optObject.getRetCode() if verbosity and fitstat != 0: print "Minimizer returned with non-zero code: ",fitstat except RuntimeError: print "Failed to find global maximum, results may be wrong" pass pass original_optimizer = like.optimizer if profile_optimizer != None: like.optimizer = profile_optimizer # Store values of global fit maxval = -like() fitval = par.getValue() fiterr = par.error() limlo, limhi = par.getBounds() if verbosity: print "Maximum of %g with %s = %g +/- %g"\ %(-maxval,srcName,fitval,fiterr) # Freeze all other model parameters if requested (much faster!) if(freeze_all): for i in range(len(like.model.params)): like.model[i].setFree(0) like.syncSrcParams(like[i].srcName) # Freeze the parameter of interest par.setFree(0) like.syncSrcParams(srcName) # Set up the caches for the optimum values and nuisance parameters optvalue_cache = dict() nuisance_cache = dict() optvalue_cache[fitval] = maxval _cache_nuisance(fitval, like, nuisance_cache) # Test if all parameters are frozen (could be true if we froze # them above or if they were frozen in the user's model all_frozen = True for i in range(len(like.model.params)): if like.model[i].isFree(): all_frozen = False break ########################################################################### # # 2) Define the integration limits by finding the points at which the # log likelihood has fallen by a certain amount # ########################################################################### if verbosity: print "Finding integration bounds (delta log Like=%g)"\ %(delta_log_like_limits) [xlo, xhi, ylo, yhi, exact_root_evals, approx_root_evals] = \ _find_interval(like, par, srcName, all_frozen, maxval, fitval, limlo, limhi, delta_log_like_limits, verbosity, like.tol, False, 5, optvalue_cache, nuisance_cache) if poi_values != None and len(poi_values)>0: xlo = max(min(xlo, min(poi_values)/2.0), limlo) xhi = min(max(xhi, max(poi_values)*2.0), limhi) if verbosity: print "Integration bounds: %g to %g (%d full fcn evals and %d approx)"\ %(xlo,xhi,exact_root_evals,approx_root_evals) ########################################################################### # # 3) Integrate the function using the QUADPACK adaptive integrator # ########################################################################### # # Do integration using QUADPACK routine from SciPy -- the "quad" # routine uses adaptive quadrature, which *should* spend more time # evaluating the function where it counts the most. # points = [] epsrel = (1.0-cl)*1e-3 if be_very_careful: # In "be very careful" mode we explicitly tell "quad" that it # should examine more carefully the point at x=fitval, which # is the peak of the likelihood. We also use a tighter # tolerance value, but that seems to have a secondary effect. points = [ fitval ] epsrel = (1.0-cl)*1e-8 if verbosity: print "Integrating probability distribution" nfneval = -len(optvalue_cache) f_of_x = dict() quad_ival, quad_ierr = \ scipy.integrate.quad(_integrand, xlo, xhi,\ args = (f_of_x, like, par, srcName, maxval,\ verbosity, all_frozen, optvalue_cache, nuisance_cache),\ points=points, epsrel=epsrel, epsabs=1) nfneval += len(optvalue_cache) if verbosity: print "Total integral: %g +/- %g (%d fcn evals)"\ %(quad_ival,quad_ierr,nfneval) ########################################################################### # # 4) Calculate the upper limit by re-integrating the function using # the evaluations made by the adaptive integrator. Two schemes are # tried, splines to the function points and trapezoidal quadrature. # ########################################################################### # Calculation of the upper limit requires integrating up to # various test points, and finding the one that contains the # prescribed fraction of the probability. Using the "quad" # function to do this by evaluating the likelihood function # directly would be computationally prohibitive, it is preferable # to use the function evaluations that have been saved in the # "f_of_x" variable. # We try 2 different integration approaches on this data: # trapezoidal quadrature and integration of a fitted spline, with # the expectation that the spline will be better, but that perhaps # the trapezoidal might be more robust if the spline fit goes # crazy. The method whose results are closest to those from "quad" # is picked to do the search. # Organize values computed into two vectors x & y x = f_of_x.keys() x.sort() y=[] logy=[] for xi in x: y.append(f_of_x[xi]) logy.append(math.log(f_of_x[xi])) # Evaluate upper limit using trapezoidal rule trapz_ival = scipy.integrate.trapz(y,x) cint = 0 Cint = [ 0 ] for i in range(len(x)-1): cint += 0.5*(f_of_x[x[i+1]]+f_of_x[x[i]])*(x[i+1]-x[i]) Cint.append(cint) int_irep = scipy.interpolate.interp1d(x, Cint) xlim_trapz = scipy.optimize.brentq(_int1droot, x[0], x[-1], args = (cl*cint, int_irep)) ylim_trapz = int_irep(xlim_trapz).item()/cint # Evaluate upper limit using spline spl_irep = scipy.interpolate.splrep(x,y,xb=xlo,xe=xhi) spl_ival = scipy.interpolate.splint(xlo,xhi,spl_irep) xlim_spl = scipy.optimize.brentq(_splintroot, xlo, xhi, args = (cl*spl_ival, xlo, spl_irep)) ylim_spl = scipy.interpolate.splint(xlo,xlim_spl,spl_irep)/spl_ival # Test which is closest to QUADPACK adaptive method: TRAPZ or SPLINE if abs(spl_ival - quad_ival) < abs(trapz_ival - quad_ival): # Evaluate upper limit using spline if verbosity: print "Using spline integral: %g (delta=%g)"\ %(spl_ival,abs(spl_ival/quad_ival-1)) xlim = xlim_spl ylim = ylim_spl if verbosity: print "Spline search: %g (P=%g)"%(xlim,ylim) else: # Evaluate upper limit using trapezoidal rule if verbosity: print "Using trapezoidal integral: %g (delta=%g)"\ %(trapz_ival,abs(trapz_ival/quad_ival-1)) xlim = xlim_trapz ylim = ylim_trapz if verbosity: print "Trapezoidal search: %g (P=%g)"%(xlim,cl) like.optimizer = original_optimizer ########################################################################### # # Since we have computed the profile likelihood, calculate the # right side of the 2-sided confidence region at the CL% and # 2*(CL-50)% levels under the assumption that the likelihood is # distributed as chi^2 of 1 DOF. Again, use the root finder on a # spline and linear representation of logL. # ########################################################################### profile_dlogL1 = -0.5*scipy.stats.chi2.isf(1-cl, 1) profile_dlogL2 = -0.5*scipy.stats.chi2.isf(1-2*(cl-0.5), 1) # The spline algorithm is prone to noise in the fitted logL, # especially in "be_very_careful" mode, so fall back to a linear # interpolation if necessary spl_drep = scipy.interpolate.splrep(x,logy,xb=xlo,xe=xhi) spl_pflux1 = scipy.optimize.brentq(_splevroot, fitval, xhi, args = (profile_dlogL1, spl_drep)) spl_pflux2 = scipy.optimize.brentq(_splevroot, fitval, xhi, args = (profile_dlogL2, spl_drep)) int_drep = scipy.interpolate.interp1d(x,logy) int_pflux1 = scipy.optimize.brentq(_int1droot, max(min(x),fitval), max(x), args = (profile_dlogL1, int_drep)) int_pflux2 = scipy.optimize.brentq(_int1droot, max(min(x),fitval), max(x), args = (profile_dlogL2, int_drep)) if (2.0*abs(int_pflux1-spl_pflux1)/abs(int_pflux1+spl_pflux1) > 0.05 or \ 2.0*abs(int_pflux2-spl_pflux2)/abs(int_pflux2+spl_pflux2) > 0.05): if verbosity: print "Using linear interpolation for profile UL estimate" profile_flux1 = int_pflux1 profile_flux2 = int_pflux2 else: if verbosity: print "Using spline interpolation for profile UL estimate" profile_flux1 = spl_pflux1 profile_flux2 = spl_pflux2 ########################################################################### # # Evaluate the probabilities of the "points of interest" using the integral # ########################################################################### poi_probs = []; poi_dlogL_interp = []; poi_chi2_equiv = []; for xval in poi_values: dLogL = None if(xval >= xhi): pval = 1.0 elif(xval <= xlo): pval = 0.0 # Same test as above to decide between TRAPZ and SPLINE elif abs(spl_ival - quad_ival) < abs(trapz_ival - quad_ival): pval = scipy.interpolate.splint(xlo,xval,spl_irep)/spl_ival dlogL = scipy.interpolate.splev(xval, spl_drep) else: pval = int_irep(xval).item()/cint dlogL = int_drep(xval).item() poi_probs.append(pval) poi_dlogL_interp.append(dlogL) poi_chi2_equiv.append(scipy.stats.chi2.isf(1-pval,1)) ########################################################################### # # Calculate the integral flux at the upper limit parameter value # ########################################################################### # Set the parameter value that corresponds to the desired C.L. par.setValue(xlim) # Evaluate the flux corresponding to this upper limit. ul_flux = like[srcName].flux(emin, emax) saved_state.restore() # Pack up all the results results = dict(all_frozen = all_frozen, ul_frac = cl, ul_flux = ul_flux, ul_value = xlim, ul_trapz = xlim_trapz, ul_spl = xlim_spl, int_limits = [xlo, xhi], profile_x = x, profile_y = y, peak_fitstatus = fitstat, peak_value = fitval, peak_dvalue = fiterr, peak_loglike = maxval, prof_ul_frac1 = cl, prof_ul_dlogL1 = profile_dlogL1, prof_ul_value1 = profile_flux1, prof_ul_frac2 = 2*(cl-0.5), prof_ul_dlogL2 = profile_dlogL2, prof_ul_value2 = profile_flux2, poi_values = poi_values, poi_probs = poi_probs, poi_dlogL_interp = poi_dlogL_interp, poi_chi2_equiv = poi_chi2_equiv, flux_emin = emin, flux_emax = emax) return ul_flux, results
def _find_interval(like, par, srcName, no_optimizer, maxval, fitval, limlo, limhi, delta_log_like_limits = 2.71/2, verbosity = 0, tol = 0.01, no_lo_bound_search = False, nloopmax = 5, optvalue_cache = dict(), nuisance_cache = dict()): """Internal function to search for interval of the normalization parameter in which the log Likelihood is larger than predefined value. Used to find the upper limit in the profile method and to find sensible limits of integration in the Bayesian method. Use the SciPy Brent method root finder to do the search. Use new fast method for up to nloopmax iterations then fall back to old method.""" subval = maxval - delta_log_like_limits search_xtol = limlo*0.1 search_ytol = tol # 2010-06-11: NEW and FASTER algorithm to find integration # limits. Instead of evaluating the real function while searching # for the root (which requires calling the optimizer) we now # evaluate an approximate function, in which all the background # parameters are kept constant. When we find the root (flux) of # the approximate function then optimize at that flux to evaluate # how close the real function is there. Then repeat this up to # "nloopmax" times, after which revert to old method if we haven't # converged. Each time the real function is evaluated at the root # of the approximate it forces the approximate function in the # next iteration to equal the real function at that point (since # the background parameters values are changed to those optimized # at that point) and so the real and approximate functions get # closer and closer around the region of the roots. # 2009-04-16: modified to do logarithmic search before calling # Brent because the minimizer does not converge very well when it # is called alternatively at extreme ends of the flux range, # because the "nuisance" parameters are very far from their # optimal values from call to call. THIS COMMENT IS OBSOLETED # BY PREVIOUS COMMENT EXCEPT IF/WHEN NEW METHOD FAILS. exact_root_evals = -len(optvalue_cache) approx_root_evals = 0 temp_saved_state = LikelihoodState(like) # HI BOUND xlft = fitval xrgt = limhi xtst = fitval ytst = delta_log_like_limits iloop = 0 while (iloop<nloopmax) and (xrgt>xlft) and (abs(ytst)>search_ytol): approx_cache = dict() approx_cache[xtst] = ytst if _approxroot(xrgt,approx_cache,like,par,srcName,subval,verbosity)<0: xtst = scipy.optimize.brentq(_approxroot, xlft, xrgt, xtol=search_xtol, args = (approx_cache,like,par, srcName,subval,verbosity)) else: xtst = xrgt ytst = _root(xtst, like, par,srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache) if ytst<=0: xrgt=xtst else: xlft=xtst iloop += 1 approx_root_evals += len(approx_cache)-1 pass xhi = xtst yhi = ytst if (xrgt>xlft) and (abs(ytst)>search_ytol): xlft = fitval for ix in optvalue_cache: if(optvalue_cache[ix]-subval>0 and ix>xlft): xlft = ix xrgt = limhi for ix in optvalue_cache: if(optvalue_cache[ix]-subval<0 and ix<xrgt): xrgt = ix if(xrgt > max(xlft*10.0, xlft+(limhi-limlo)*1e-4)): xtst = max(xlft*10.0, xlft+(limhi-limlo)*1e-4) while(xtst<xrgt and\ _root(xtst, like,par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache)>=0): xtst *= 10.0 if(xtst<xrgt): xrgt = xtst if xrgt>limhi: xrgt=limhi if xrgt<limhi or \ _root(xrgt, like, par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache)<0: xhi = scipy.optimize.brentq(_root, xlft, xrgt, xtol=search_xtol, args = (like,par,srcName,\ subval,verbosity,no_optimizer, optvalue_cache,nuisance_cache)) pass yhi = _root(xhi, like, par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache) pass temp_saved_state.restore() # LO BOUND if(no_lo_bound_search): xlo = fitval ylo = maxval exact_root_evals += len(optvalue_cache) return [xlo, xhi, ylo, yhi, exact_root_evals, approx_root_evals] xlft = limlo xrgt = fitval xtst = fitval ytst = delta_log_like_limits iloop = 0 while (iloop<nloopmax) and (xrgt>xlft) and (abs(ytst)>search_ytol): approx_cache = dict() approx_cache[xtst] = ytst if _approxroot(xlft,approx_cache,like,par,srcName,subval,verbosity)<0: xtst = scipy.optimize.brentq(_approxroot, xlft, xrgt, xtol=search_xtol, args = (approx_cache,like,par, srcName,subval,verbosity)) else: xtst = xlft ytst = _root(xtst, like, par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache) if ytst<=0: xlft=xtst else: xrgt=xtst approx_root_evals += len(approx_cache)-1 iloop += 1 pass xlo = xtst ylo = ytst if (xrgt>xlft) and (abs(ytst)>search_ytol): xrgt = fitval for ix in optvalue_cache: if(optvalue_cache[ix]-subval>0 and ix<xrgt): xrgt = ix xlft = limlo for ix in optvalue_cache: if(optvalue_cache[ix]-subval<0 and ix<xlft): xlft = ix if(xlft < min(xrgt*0.1, xrgt-(limhi-limlo)*1e-4)): xtst = min(xrgt*0.1, xrgt-(limhi-limlo)*1e-4) while(xtst>xlft and\ _root(xtst, like,par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache)>=0): xtst *= 0.1 if(xtst>xlft): xlft = xtst if xlft<limlo: xlft=limlo if xlft>limlo or \ _root(xlft, like, par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache)<0: xlo = scipy.optimize.brentq(_root, xlft, xrgt, xtol=search_xtol, args = (like,par,srcName,\ subval,verbosity,no_optimizer, optvalue_cache,nuisance_cache)) pass ylo = _root(xlo, like, par, srcName, subval, verbosity, no_optimizer, optvalue_cache, nuisance_cache) pass temp_saved_state.restore() exact_root_evals += len(optvalue_cache) return [xlo, xhi, ylo, yhi, exact_root_evals, approx_root_evals]
def saveCurrentFit(self, negLogLike=None): self.saved_state = LikelihoodState(self, negLogLike)
def _extension(self, name, **kwargs): spatial_model = kwargs['spatial_model'] width_min = kwargs['width_min'] width_max = kwargs['width_max'] width_nstep = kwargs['width_nstep'] width = kwargs['width'] free_background = kwargs['free_background'] free_radius = kwargs.get('free_radius', None) fix_shape = kwargs.get('fix_shape', False) make_tsmap = kwargs.get('make_tsmap', False) update = kwargs['update'] sqrt_ts_threshold = kwargs['sqrt_ts_threshold'] if kwargs['psf_scale_fn']: def psf_scale_fn(t): return 1.0 + np.interp(np.log10(t), kwargs['psf_scale_fn'][0], kwargs['psf_scale_fn'][1]) else: psf_scale_fn = None saved_state = LikelihoodState(self.like) loglike_init = -self.like() self.logger.debug('Initial Model Log-Likelihood: %f', loglike_init) if not free_background: self.free_sources(free=False, loglevel=logging.DEBUG) if free_radius is not None: diff_sources = [s.name for s in self.roi.sources if s.diffuse] skydir = self.roi[name].skydir free_srcs = [s.name for s in self.roi.get_sources(skydir=skydir, distance=free_radius, exclude=diff_sources)] self.free_sources_by_name(free_srcs, pars='norm', loglevel=logging.DEBUG) # Fit baseline model self.free_source(name, loglevel=logging.DEBUG) if fix_shape: self.free_source(name, free=False, pars='shape', loglevel=logging.DEBUG) fit_output = self._fit(loglevel=logging.DEBUG, **kwargs['optimizer']) src = self.roi.copy_source(name) # Save likelihood value for baseline fit saved_state_base = LikelihoodState(self.like) loglike_base = fit_output['loglike'] self.logger.debug('Baseline Model Log-Likelihood: %f', loglike_base) if not width: width = np.logspace(np.log10(width_min), np.log10(width_max), width_nstep) width = np.array(width) width = width[width > 0] width = np.concatenate(([0.0], np.array(width))) o = defaults.make_default_tuple(defaults.extension_output) o.name = name o.width = width o.dloglike = np.zeros(len(width) + 1) o.loglike = np.zeros(len(width) + 1) o.loglike_base = loglike_base o.loglike_init = loglike_init o.config = kwargs o.ebin_ext = np.ones(self.enumbins) * np.nan o.ebin_ext_err = np.ones(self.enumbins) * np.nan o.ebin_ext_err_lo = np.ones(self.enumbins) * np.nan o.ebin_ext_err_hi = np.ones(self.enumbins) * np.nan o.ebin_ext_ul95 = np.ones(self.enumbins) * np.nan o.ebin_ts_ext = np.ones(self.enumbins) * np.nan o.ebin_loglike = np.ones((self.enumbins, len(width))) * np.nan o.ebin_dloglike = np.ones((self.enumbins, len(width))) * np.nan o.ebin_loglike_ptsrc = np.ones(self.enumbins) * np.nan o.ebin_loglike_ext = np.ones(self.enumbins) * np.nan o.ebin_e_min = self.energies[:-1] o.ebin_e_max = self.energies[1:] o.ebin_e_ctr = np.sqrt(o.ebin_e_min * o.ebin_e_max) self.logger.debug('Width scan vector:\n %s', width) if kwargs['fit_position']: ext_fit = self._fit_extension_full(name, spatial_model=spatial_model, optimizer=kwargs['optimizer']) else: ext_fit = self._fit_extension(name, spatial_model=spatial_model, optimizer=kwargs['optimizer'], psf_scale_fn=psf_scale_fn) o.update(ext_fit) # Fit with the best-fit extension model self.logger.info('Fitting extended-source model.') self.set_source_morphology(name, spatial_model=spatial_model, spatial_pars={'ra': o['ra'], 'dec': o['dec'], 'SpatialWidth': o['ext']}, use_pylike=False, psf_scale_fn=psf_scale_fn) # Perform scan over width parameter o.loglike = self._scan_extension(name, spatial_model=spatial_model, width=width, optimizer=kwargs['optimizer'], psf_scale_fn=psf_scale_fn) self.set_source_morphology(name, spatial_model=spatial_model, spatial_pars={'ra': o['ra'], 'dec': o['dec'], 'SpatialWidth': o['ext']}, use_pylike=False, psf_scale_fn=psf_scale_fn) fit_output = self._fit(loglevel=logging.DEBUG, update=False, **kwargs['optimizer']) o.source_fit = self.get_src_model(name, reoptimize=True, optimizer=kwargs['optimizer']) o.loglike_ext = fit_output['loglike'] if kwargs['fit_ebin']: self._fit_extension_ebin(name, o, **kwargs) if kwargs['save_model_map']: o.ext_tot_map = self.model_counts_map() o.ext_src_map = self.model_counts_map(name) o.ext_bkg_map = self.model_counts_map(exclude=[name]) if make_tsmap: tsmap_model = {'SpatialModel': 'RadialDisk', 'SpatialWidth': 0.1 * 0.8246211251235321} tsmap_model.update(src.spectral_pars) self.logger.info('Generating TS map.') tsmap = self.tsmap(model=tsmap_model, map_skydir=SkyCoord( o['ra'], o['dec'], unit='deg'), map_size=max(1.0, 4.0 * o['ext']), exclude=[name], write_fits=False, write_npy=False, use_pylike=False, make_plots=False, loglevel=logging.DEBUG) o.tsmap = tsmap['ts'] self.logger.info('Testing point-source model.') # Test point-source hypothesis self.set_source_morphology(name, spatial_model='PointSource', use_pylike=False, psf_scale_fn=psf_scale_fn) # Fit a point-source saved_state_base.restore() self.logger.debug('Fitting point-source model.') fit_output = self._fit(loglevel=logging.DEBUG, **kwargs['optimizer']) if src['SpatialModel'] == 'PointSource' and kwargs['fit_position']: loc = self.localize(name, update=False) o.loglike_ptsrc = loc['loglike_loc'] else: o.loglike_ptsrc = fit_output['loglike'] o.dloglike = o.loglike - o.loglike_ptsrc o.ts_ext = 2 * (o.loglike_ext - o.loglike_ptsrc) self.logger.debug('Point-Source Model Likelihood: %f', o.loglike_ptsrc) if kwargs['save_model_map']: o.ptsrc_tot_map = self.model_counts_map() o.ptsrc_src_map = self.model_counts_map(name) o.ptsrc_bkg_map = self.model_counts_map(exclude=[name]) if update and (sqrt_ts_threshold is None or np.sqrt(o['ts_ext']) > sqrt_ts_threshold): src = self.delete_source(name, loglevel=logging.DEBUG) src.set_spatial_model(spatial_model, {'ra': o.ra, 'dec': o.dec, 'SpatialWidth': o.ext}) # FIXME: Issue with source map cache with source is # initialized as fixed. self.add_source(name, src, free=True, loglevel=logging.DEBUG) self.free_source(name, loglevel=logging.DEBUG) if fix_shape: self.free_source(name, free=False, pars='shape', loglevel=logging.DEBUG) fit_output = self.fit(loglevel=logging.DEBUG, **kwargs['optimizer']) o.loglike_ext = fit_output['loglike'] src = self.roi.get_source_by_name(name) if kwargs['fit_position']: for k in ['ra_err', 'dec_err', 'glon_err', 'glat_err', 'pos_err', 'pos_err_semimajor', 'pos_err_semiminor', 'pos_r68', 'pos_r95', 'pos_r99', 'pos_angle']: src[k] = o[k] else: self.set_source_morphology(name, spatial_model=src['SpatialModel'], spatial_pars=src.spatial_pars, update_source=False) # Restore ROI to previous state saved_state.restore() self._sync_params(name) self._update_roi() self.logger.info('Best-fit extension: %6.4f + %6.4f - %6.4f' % (o['ext'], o['ext_err_hi'], o['ext_err_lo'])) self.logger.info('TS_ext: %.3f' % o['ts_ext']) self.logger.info('Extension UL: %6.4f' % o['ext_ul95']) self.logger.info('LogLike: %12.3f DeltaLogLike: %12.3f', o.loglike_ext, o.loglike_ext - o.loglike_init) if kwargs['fit_position']: self.logger.info('Position:\n' '( ra, dec) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' '(glon,glat) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' 'offset = %8.4f r68 = %8.4f r95 = %8.4f r99 = %8.4f', o.ra, o.ra_err, o.dec, o.dec_err, o.glon, o.glon_err, o.glat, o.glat_err, o.pos_offset, o.pos_r68, o.pos_r95, o.pos_r99) return o
def _calculate(self, like): """ Compute the flux data points for each energy. """ name = self.name verbosity = self.verbosity init_energies = like.energies[[0, -1]] # Freeze all sources except one to make sed of. all_sources = like.sourceNames() if name not in all_sources: raise Exception("Cannot find source %s in list of sources" % name) # make copy of parameter values + free parameters saved_state = LikelihoodState(like) if self.freeze_background: if verbosity: print 'Freezing all parameters' # freeze all other sources for i in range(len(like.model.params)): like.freeze(i) # convert source to a PowerLaw of frozen index source = like.logLike.getSource(name) old_spectrum = source.spectrum() like.setSpectrum(name, 'PowerLaw') index = like[like.par_index(name, 'Index')] index.setTrueValue(self.powerlaw_index) index.setFree(0) like.syncSrcParams(name) # assume a canonical dnde=1e-11 at 1GeV index 2 starting value dnde_start = 1e-11 * (self.energy / 1e3)**(-2) optverbosity = max(verbosity - 1, 0) # see IntegralUpperLimit.py for i, (e, lower, upper) in enumerate( zip(self.energy, self.lower_energy, self.upper_energy)): if verbosity: print 'Calculating spectrum from %.0dMeV to %.0dMeV' % (lower, upper) # goot starting guess for source prefactor = like[like.par_index(name, 'Prefactor')] prefactor.setScale(dnde_start[i]) prefactor.setValue(1) prefactor.setBounds(1e-10, 1e10) scale = like[like.par_index(name, 'Scale')] scale.setScale(1) scale.setValue(e) like.syncSrcParams(name) like.setEnergyRange(float(lower) + 1, float(upper) - 1) try: like.fit(optverbosity, covar=True) except Exception, ex: if verbosity: print 'ERROR gtlike fit: ', ex self.ts[i] = like.Ts(name, reoptimize=self.reoptimize_ts) prefactor = like[like.par_index(name, 'Prefactor')] self.dnde[i] = prefactor.getTrueValue() if self.do_minos: if verbosity: print 'Calculating minos errors from %.0dMeV to %.0dMeV' % ( lower, upper) self.dnde_lower_err[i], self.dnde_upper_err[ i] = like.minosError(name, 'Prefactor') self.dnde_lower_err[i] *= ( -1) * prefactor.getScale() # make lower errors positive self.dnde_upper_err[i] *= prefactor.getScale() self.dnde_err[i] = (self.dnde_upper_err[i] + self.dnde_lower_err[i]) / 2 else: self.dnde_err[i] = prefactor.parameter.error( ) * prefactor.parameter.getScale() self.flux[i] = like.flux(name, lower, upper) self.flux_err[i] = like.fluxError(name, lower, upper) self.eflux[i] = like.energyFluxError(name, lower, upper) self.eflux_err[i] = like.energyFluxError(name, lower, upper) if self.ts[i] < self.min_ts or self.always_upper_limit: if verbosity: print 'Calculating upper limit from %.0dMeV to %.0dMeV' % ( lower, upper) self.dnde_ul[i], self.flux_ul[i], self.eflux_ul[ i] = SED.upper_limit(like, name, self.ul_algorithm, lower, upper, confidence=self.ul_confidence, verbosity=verbosity) if verbosity: print lower, upper, self.dnde[i], self.dnde_err[i], self.ts[ i], self.dnde_ul[i] self.npred[i] = like.NpredValue(name)
class SummedLikelihood(AnalysisBase): def __init__(self, optimizer='Minuit'): self.composite = pyLike.SummedLikelihood() #the C++ SummedLikelihood has many if not all the properties #of a logLike object self.logLike = self.composite self.components = [] self.covariance = None self.covar_is_current = False self.optObject = None self.optimizer = optimizer self.tolType = pyLike.ABSOLUTE self.tol = 1e-2 self.saved_state = None def sourceNames(self): return self.components[0].sourceNames() def addComponent(self, like): self.composite.addComponent(like.logLike) self.components.append(like) if len(self.components) == 1: self.model = self.components[0].model def syncSrcParams(self, src=None): if src is not None: for comp in self.components: comp.logLike.syncSrcParams(src) else: for comp in self.components: for src in self.sourceNames(): comp.logLike.syncSrcParams(src) def fit(self, verbosity=3, tol=None, optimizer=None, covar=False, optObject=None): if tol is None: tol = self.tol errors = self._errors(optimizer, verbosity, tol, covar=covar, optObject=optObject) negLogLike = -self.composite.value() self.saveBestFit(negLogLike) return negLogLike def optimize(self, verbosity=3, tol=None, optimizer=None): self._syncParams() if optimizer is None: optimizer = self.optimizer if tol is None: tol = self.tol optFactory = pyLike.OptimizerFactory_instance() myOpt = optFactory.create(optimizer, self.composite) myOpt.find_min_only(verbosity, tol, self.tolType) self.saveBestFit() def normPar(self, source): return Parameter([like.normPar(source) for like in self.components]) def __call__(self): negLogLike = -self.composite.value() self.saveBestFit(negLogLike) return negLogLike def sourceNames(self): return self.components[0].sourceNames() def params(self): my_params = [] for i, like in enumerate(self.components): for j, par in enumerate(like.params()): if i == 0: my_params.append(Parameter([par])) else: my_params[j].addParam(par) return my_params def nFreeParams(self): '''Count the number of free parameters in the active model.''' nF = 0 pars = self.params() for par in pars: if par.isFree(): nF += 1 return nF def saveBestFit(self, negLogLike=None): if negLogLike is None: negLogLike = -self.composite.value() if (self.saved_state is None or negLogLike <= self.saved_state.negLogLike): self.saveCurrentFit(negLogLike) def saveCurrentFit(self, negLogLike=None): self.saved_state = LikelihoodState(self, negLogLike) def restoreBestFit(self): if (self.saved_state is not None and self() > self.saved_state.negLogLike): self.saved_state.restore() else: self.saveCurrentFit() def NpredValue(self, src, weighted=False): return self.composite.NpredValue(src, weighted) def total_nobs(self, weighted=False): return sum([x.total_nobs(weighted) for x in self.components]) def __repr__(self): return str(self.components[0].model) def _syncParams(self): for component in self.components: component.logLike.syncParams() def __getitem__(self, name): item = self.model[name] try: item.type return item except AttributeError: par = Parameter([item]) for comp in self.components[1:]: par.addParam(comp[name]) return par def __setitem__(self, name, value): for component in self.components: component[name] = value component.syncSrcParams(self.model[name].srcName) def thaw(self, i): for component in self.components: component.thaw(i) self.saved_state = None def freeze(self, i): for component in self.components: component.freeze(i) self.saved_state = None def _errors(self, optimizer=None, verbosity=0, tol=None, useBase=False, covar=False, optObject=None): self._syncParams() if optimizer is None: optimizer = self.optimizer if tol is None: tol = self.tol if optObject is None: optFactory = pyLike.OptimizerFactory_instance() myOpt = optFactory.create(optimizer, self.composite) else: myOpt = optObject self.optObject = myOpt myOpt.find_min(verbosity, tol, self.tolType) errors = myOpt.getUncertainty(useBase) if covar: self.covariance = myOpt.covarianceMatrix() self.covar_is_current = True else: self.covar_is_current = False self._set_errors(errors) return errors def _set_errors(self, errors): source_attributes = self.components[0].getExtraSourceAttributes() my_errors = list(errors) self.composite.setErrors(my_errors) for component in self.components: component.model = SourceModel(component.logLike) for src in source_attributes: component.model[src].__dict__.update(source_attributes[src]) def minosError(self, srcname, parname, level=1): freeParams = pyLike.ParameterVector() self.composite.getFreeParams(freeParams) saved_values = [par.getValue() for par in freeParams] par_index = self.components[0].par_index(srcname, parname) index = self.composite.findIndex(par_index) if index == -1: raise RuntimeError("Invalid parameter specification") try: errors = self.optObject.Minos(index, level) self.composite.setFreeParamValues(saved_values) return errors except RuntimeError, message: print "Minos error encountered for parameter %i" % index self.composite.setFreeParamValues(saved_values)
def wrapper(self, *args, **kwargs): saved_state = LikelihoodState(self.like) o = func(self, *args, **kwargs) saved_state.restore() return o
def _tsmap_pylike(self, prefix, **kwargs): """Evaluate the TS for an additional source component at each point in the ROI. This is the brute force implementation of TS map generation that runs a full pyLikelihood fit at each point in the ROI.""" logLike0 = -self.like() self.logger.info('LogLike: %f' % logLike0) saved_state = LikelihoodState(self.like) # Get the ROI geometry # Loop over pixels w = copy.deepcopy(self._skywcs) # w = create_wcs(self._roi.skydir,cdelt=self._binsz,crpix=50.5) data = np.zeros((self.npix, self.npix)) # self.free_sources(free=False) xpix = (np.linspace(0, self.npix - 1, self.npix)[:, np.newaxis] * np.ones(data.shape)) ypix = (np.linspace(0, self.npix - 1, self.npix)[np.newaxis, :] * np.ones(data.shape)) radec = wcs_utils.pix_to_skydir(xpix, ypix, w) radec = (np.ravel(radec.ra.deg), np.ravel(radec.dec.deg)) testsource_dict = { 'ra': radec[0][0], 'dec': radec[1][0], 'SpectrumType': 'PowerLaw', 'Index': 2.0, 'Scale': 1000, 'Prefactor': {'value': 0.0, 'scale': 1e-13}, 'SpatialModel': 'PSFSource', } # src = self.roi.get_source_by_name('tsmap_testsource') for i, (ra, dec) in enumerate(zip(radec[0], radec[1])): testsource_dict['ra'] = ra testsource_dict['dec'] = dec # src.set_position([ra,dec]) self.add_source('tsmap_testsource', testsource_dict, free=True, init_source=False, save_source_maps=False) # for c in self.components: # c.update_srcmap_file([src],True) self.set_parameter('tsmap_testsource', 'Prefactor', 0.0, update_source=False) self.fit(loglevel=logging.DEBUG, update=False) logLike1 = -self.like() ts = max(0, 2 * (logLike1 - logLike0)) data.flat[i] = ts # print i, ra, dec, ts # print self.like() # print self.components[0].like.model['tsmap_testsource'] self.delete_source('tsmap_testsource') saved_state.restore() outfile = os.path.join(self.config['fileio']['workdir'], 'tsmap.fits') fits_utils.write_fits_image(data, w, outfile)
class ROILikelihoodOptimizer: """Class to optimize Likelihood of ROI model (a replacement for gtlike).""" def __init__(self, like, sourcesOfInterest=None, optimizer="Minuit", tol=1e-8, chatter=3, freeze_nuisance_sources_immediately = False, freeze_nuisance_sources_after_optimize = False, nuisance_npred_frac_max = 0.05, nuisance_soi_sep_min_deg = 5.0, calculate_full_ts_for_all = False): self.ver = "$Id: ROILikelihoodOptimizer.py 2795 2011-05-06 14:48:45Z sfegan $" self.res = {} self._like = like self._original_state = LikelihoodState(self._like) self._chatter = 0 self._SOI = [] self._freeze_nuisance_immediately = freeze_nuisance_sources_immediately self._freeze_nuisance_after_optimize = \ freeze_nuisance_sources_after_optimize self._nuisance_npred_frac_max = nuisance_npred_frac_max self._nuisance_soi_sep_min_deg = nuisance_soi_sep_min_deg self._calculate_full_ts_for_all = calculate_full_ts_for_all if sourcesOfInterest is not None: if type(sourcesOfInterest) != list: sourcesOfInterest = [ sourcesOfInterest ] for s in sourcesOfInterest: if s in like.sourceNames(): self._SOI.append(s) else: raise RuntimeError("Invalid source of interest: " + s) if optimizer: self._like.optimizer = optimizer if tol: self._like.tol = tol if chatter: self._chatter = chatter def _nuisanceSourceNames(self, SOI = None, sep_min_deg = None, npred_frac_max = None): if SOI is None: SOI = self._SOI if type(SOI) != list: SOI = [ SOI ] if npred_frac_max is None: npred_frac_max = self._nuisance_npred_frac_max if sep_min_deg is None: sep_min_deg = self._nuisance_soi_sep_min_deg sep_min_rad = sep_min_deg/180.0*math.pi nuisance = [] npred_total = 0 for sn in self._like.sourceNames(): npred_total += self._like.NpredValue(sn) npred_thresh = npred_total * npred_frac_max for sn in self._like.sourceNames(): if sn in SOI: continue s = self._like[sn] if s.type != 'PointSource': continue ps = pyLike.PointSource_cast(s.src) if ps.fixedSpectrum(): continue if self._like.NpredValue(sn) > npred_thresh: continue d = ps.getDir() min_sep = math.pi for soi_sn in SOI: soi_s = self._like[soi_sn] soi_ps = pyLike.PointSource_cast(soi_s.src) soi_d = soi_ps.getDir() sep = d.difference(soi_d) if(sep < min_sep): min_sep = sep if min_sep < sep_min_rad: continue nuisance.append(sn) return nuisance def _freeze_sources(self): nuisance_sources = self._nuisanceSourceNames() for sn in nuisance_sources: srcfreepar = self._like.freePars(sn) self._like.setFreeFlag(sn, srcfreepar, False) self._like.syncSrcParams(sn) return nuisance_sources def restoreOriginalState(): self._original_state.restore() def run(self, noFit=False): L = self._like start_state = LikelihoodState(L) if(self._freeze_nuisance_immediately): nuisance_sources = self._freeze_sources() if self._chatter > 0: for sn in nuisance_sources: print "Freezing source:",sn if(self._freeze_nuisance_after_optimize): L.optimize() nuisance_sources = self._freeze_sources() if self._chatter > 0: for sn in nuisance_sources: print "Freezing source:",sn if not noFit: L.fit() res = {} res['version'] = self.ver res['like_val'] = L.logLike.value() res['fit_state'] = L.optObject.getRetCode() res['cov_matrix'] = L.optObject.covarianceMatrix() res['src'] = {} npred_total = 0 nfree_param = 0 for sn in L.sourceNames(): s = L[sn] ss = s.src src_info = {} src_info["type"] = ss.getType() src_info["fixed"] = ss.fixedSpectrum() if not src_info["fixed"] and src_info["type"] == 'Point': if self._chatter > 0: print "Calculating approximate TS for:",sn src_info['TS_approx'] = L.Ts(sn,reoptimize=False) if sn in self._SOI or self._calculate_full_ts_for_all: if self._chatter > 0: print "Calculating full TS for:",sn src_info['TS'] = L.Ts(sn,reoptimize=True) # Spectrum parameters spec = ss.spectrum() spec_param_names = s.funcs['Spectrum'].paramNames src_info["spec_type"] = spec.genericName() src_info["spec_free_par"] = {} spec_info = {} for pn in spec_param_names: param = spec.getParam(pn) param_info = {} param_info['free'] = param.isFree() if param.isFree(): src_info["spec_free_par"][pn] = nfree_param param_info['free_iparam'] = nfree_param param_info['error'] = param.error() param_info['true_error'] = param.error()*param.getScale() nfree_param+=1 param_info['true_value'] = param.getTrueValue() param_info['value'] = param.getValue() param_info['scale'] = param.getScale() param_info['bounds'] = param.getBounds() spec_info[pn] = param_info cov_m = [] for ipn in src_info["spec_free_par"]: cov = {} cov_v = [] for jpn in src_info["spec_free_par"]: iparam = spec_info[ipn]['free_iparam'] jparam = spec_info[jpn]['free_iparam'] cov[jpn] = res['cov_matrix'][iparam][jparam] cov_v.append(cov[jpn]) spec_info[ipn]['cov'] = cov cov_m.append(cov_v) src_info["spec_par"] = spec_info src_info["spec_cov"] = cov_m cov_m = numpy.matrix(cov_m) # Flux value, derivatives and error info = {} info['value'] = ss.flux() if not ss.fixedSpectrum(): info["deriv"] = {} v=[] for pn in src_info["spec_free_par"]: x = ss.fluxDeriv(pn) info["deriv"][pn] = x v.append(x) v = numpy.matrix(v) src_info["error"] = math.sqrt(v*cov_m*v.transpose()) src_info["flux"] = info # Energy flux value, derivatives and error info = {} info['value'] = ss.energyFlux() if not ss.fixedSpectrum(): info["deriv"] = {} v=[] for pn in src_info["spec_free_par"]: x = ss.energyFluxDeriv(pn) info["deriv"][pn] = x v.append(x) v = numpy.matrix(v) info["error"] = math.sqrt(v*cov_m*v.transpose()) src_info["energy_flux"] = info # Npred value, derivatives and error npred = L.NpredValue(sn) npred_total += npred info = {} info['value'] = npred # if not ss.fixedSpectrum(): # info["deriv"] = {} # v = [] # for pn in src_info["spec_free_par"]: # x = 0 #ss.NpredDeriv(pn) CRASH # info["deriv"][pn] = x # v.append(x) # v = numpy.matrix(v) # info["error"] = math.sqrt(v*cov_m*v.transpose()) src_info["npred"] = info # Spectral plots # Set source results res['src'][sn] = src_info res['total_nobs'] = L.total_nobs(); res['total_npred'] = npred_total self.res = res
def _localize(self, name, **kwargs): nstep = kwargs.get('nstep') dtheta_max = kwargs.get('dtheta_max') update = kwargs.get('update', True) prefix = kwargs.get('prefix', '') use_cache = kwargs.get('use_cache', False) free_background = kwargs.get('free_background', False) free_radius = kwargs.get('free_radius', None) fix_shape = kwargs.get('fix_shape', False) saved_state = LikelihoodState(self.like) loglike_init = -self.like() self.logger.debug('Initial Model Log-Likelihood: %f', loglike_init) if not free_background: self.free_sources(free=False, loglevel=logging.DEBUG) if free_radius is not None: diff_sources = [s.name for s in self.roi.sources if s.diffuse] skydir = self.roi[name].skydir free_srcs = [ s.name for s in self.roi.get_sources( skydir=skydir, distance=free_radius, exclude=diff_sources) ] self.free_sources_by_name(free_srcs, pars='norm', loglevel=logging.DEBUG) src = self.roi.copy_source(name) skydir = src.skydir skywcs = self._skywcs src_pix = skydir.to_pixel(skywcs) fit0 = self._fit_position_tsmap(name, prefix=prefix, dtheta_max=dtheta_max, zmin=-3.0, use_pylike=False) self.logger.debug( 'Completed localization with TS Map.\n' '(ra,dec) = (%10.4f,%10.4f) ' '(glon,glat) = (%10.4f,%10.4f)', fit0['ra'], fit0['dec'], fit0['glon'], fit0['glat']) # Fit baseline (point-source) model self.free_source(name, loglevel=logging.DEBUG) if fix_shape: self.free_source(name, free=False, pars='shape', loglevel=logging.DEBUG) fit_output = self._fit(loglevel=logging.DEBUG, **kwargs.get('optimizer', {})) # Save likelihood value for baseline fit loglike_base = fit_output['loglike'] self.logger.debug('Baseline Model Log-Likelihood: %f', loglike_base) o = defaults.make_default_tuple(defaults.localize_output) o.name = name o.config = kwargs o.fit_success = True o.loglike_init = loglike_init o.loglike_base = loglike_base o.loglike_loc = np.nan o.dloglike_loc = np.nan if fit0['fit_success']: scan_cdelt = 2.0 * fit0['pos_r95'] / (nstep - 1.0) else: scan_cdelt = np.abs(skywcs.wcs.cdelt[0]) self.logger.debug( 'Refining localization search to ' 'region of width: %.4f deg', scan_cdelt * nstep) fit1 = self._fit_position_scan(name, skydir=fit0['skydir'], scan_cdelt=scan_cdelt, **kwargs) o.loglike_loc = fit1['loglike'] o.dloglike_loc = o.loglike_loc - o.loglike_base o.tsmap = fit0.pop('tsmap') o.tsmap_peak = fit1.pop('tsmap') # o.update(fit1) # Best fit position and uncertainty from fit to TS map o.fit_init = fit0 # Best fit position and uncertainty from pylike scan o.fit_scan = fit1 o.update(fit1) cdelt0 = np.abs(skywcs.wcs.cdelt[0]) cdelt1 = np.abs(skywcs.wcs.cdelt[1]) pix = fit1['skydir'].to_pixel(skywcs) o.pos_offset = skydir.separation(fit1['skydir']).deg o.xpix = float(pix[0]) o.ypix = float(pix[1]) o.deltax = (o.xpix - src_pix[0]) * cdelt0 o.deltay = (o.ypix - src_pix[1]) * cdelt1 o.ra_preloc = skydir.ra.deg o.dec_preloc = skydir.dec.deg o.glon_preloc = skydir.galactic.l.deg o.glat_preloc = skydir.galactic.b.deg if o.pos_offset > dtheta_max: o.fit_success = False if not o.fit_success: self.logger.warning('Fit to localization contour failed.') elif not o.fit_inbounds: self.logger.warning('Best-fit position outside of search region.') else: self.logger.info('Localization succeeded.') if update and ((not o.fit_success) or (not o.fit_inbounds)): self.logger.warning( 'Localization failed. Keeping existing position.') if update and o.fit_success and o.fit_inbounds: self.logger.info('Updating source %s ' 'to localized position.', name) src = self.delete_source(name) src.set_position(fit1['skydir']) self.add_source(name, src, free=True) self.free_source(name, loglevel=logging.DEBUG) if fix_shape: self.free_source(name, free=False, pars='shape', loglevel=logging.DEBUG) fit_output = self.fit(loglevel=logging.DEBUG) o.loglike_loc = fit_output['loglike'] o.dloglike_loc = o.loglike_loc - o.loglike_base src = self.roi.get_source_by_name(name) src['glon_err'] = o.glon_err src['glat_err'] = o.glat_err src['ra_err'] = o.glon_err src['dec_err'] = o.glat_err src['pos_err'] = o.pos_err src['pos_err_semimajor'] = o.pos_err_semimajor src['pos_err_semiminor'] = o.pos_err_semiminor src['pos_r68'] = o.pos_r68 src['pos_r95'] = o.pos_r95 src['pos_r99'] = o.pos_r99 src['pos_angle'] = o.pos_angle src['pos_gal_cov'] = o.pos_gal_cov src['pos_gal_corr'] = o.pos_gal_corr src['pos_cel_cov'] = o.pos_cel_cov src['pos_cel_corr'] = o.pos_cel_corr else: saved_state.restore() self._sync_params(name) self._update_roi() self.logger.info( 'Localization completed with new position:\n' '( ra, dec) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' '(glon,glat) = (%10.4f +/- %8.4f,%10.4f +/- %8.4f)\n' 'offset = %8.4f r68 = %8.4f r95 = %8.4f r99 = %8.4f', o.ra, o.ra_err, o.dec, o.dec_err, o.glon, o.glon_err, o.glat, o.glat_err, o.pos_offset, o.pos_r68, o.pos_r95, o.pos_r99) self.logger.info('LogLike: %12.3f DeltaLogLike: %12.3f', o.loglike_loc, o.loglike_loc - o.loglike_init) return o