def set(self): """Set the PSF for Sherpa.""" import sherpa.astro.ui as sau from ..scripts.image_fit import read_json read_json(self.pars, sau.set_model) sau.load_psf('psf', sau.get_model()) self.center_psf() sau.set_psf('psf')
def set(self): """Set the PSF for Sherpa.""" import sherpa.astro.ui as sau # from morphology.utils import read_json read_json(self.pars, sau.set_model) sau.load_psf('psf', sau.get_model()) sau.set_psf('psf') self.center_psf()
def set(self): """Set the PSF for Sherpa.""" import sherpa.astro.ui as sau # from gammapy.image.models.utils import read_json read_json(self.pars, sau.set_model) sau.load_psf('psf', sau.get_model()) self.center_psf() sau.set_psf('psf')
def loadSrcModel(id=1, readFrom='srcPowerLaw.json'): """ """ with open(readFrom, 'r') as f: parDict = json.load(f) srcModel = ui.get_model(id=id) for p in srcModel.pars: p.val = parDict[p.fullname]
def sherpa_spec_fit(in_files, model, noplot, manual, reproj, do_conf): """Reads a set of pha files and performs a maximum likelihood fit. """ import logging logging.basicConfig(level=logging.DEBUG, format='%(levelname)s - %(message)s') # from sherpa.astro.ui import * # TEST if needed (only when outside sherpa env) import sherpa.astro.ui as sau from . import load_model from . import make_plot from .specsource import SpecSource logger = logging.getLogger("sherpa") logger.setLevel(logging.ERROR) # Read and load the data and model: list_data = [] emax = 9e10 # Maximum energy taken into account for the fit --> Compute properly (as a function of the max E event?) # We don't use a specific maximum reduced statistic value since we don't expect the cstat to be anywhere near the large number limit sau.set_conf_opt("max_rstat", 10000) p1 = load_model.load_model(model[0]) # [-1] #load model returns an array of model components # where the last component is the total model spec = SpecSource('SRC', in_files) if reproj == 0: myspec = spec elif reproj == 3: myspec = spec.reproject(nbins={'offset': 5, 'eff': 10, 'zen': 10}) elif reproj == 2: myspec = spec.reproject(nbins={'offset': 5, 'eff': 20, 'zen': 12}) elif reproj == 1: myspec = spec.reproject(nbins={'offset': 25, 'eff': 40, 'zen': 30}) elif reproj == 4: myspec = spec.reproject(nbins={'offset': 1, 'eff': 1, 'zen': 1}) myspec.set_source(p1) if manual: load_model.set_manual_model(sau.get_model(datid)) # Parameters for all runs are linked else: print('Using default initial values for model parameters') myspec.fit(do_conf=do_conf) if noplot: quit() make_plot.make_plot(list_data, p1) raw_input('Press <ENTER> to continue')
def test_set_analysis_wave_fabrizio(self): rmf = self.datadir + '/ciao4.3/fabrizio/Data/3c273.rmf' arf = self.datadir + '/ciao4.3/fabrizio/Data/3c273.arf' ui.set_model("fabrizio", "xspowerlaw.p1") ui.fake_pha("fabrizio", arf, rmf, 10000) model = ui.get_model("fabrizio") bare_model, _ = ui._session._get_model_status("fabrizio") y = bare_model.calc([1, 1], model.xlo, model.xhi) y_m = numpy.mean(y) ui.set_analysis("fabrizio", "wave") model2 = ui.get_model("fabrizio") bare_model2, _ = ui._session._get_model_status("fabrizio") y2 = bare_model2.calc([1, 1], model2.xlo, model2.xhi) y2_m = numpy.mean(y2) self.assertAlmostEqual(y_m, y2_m)
def test_set_analysis_wave_fabrizio(self): rmf = self.datadir + '/ciao4.3/fabrizio/Data/3c273.rmf' arf = self.datadir + '/ciao4.3/fabrizio/Data/3c273.arf' ui.set_model("fabrizio", "xspowerlaw.p1") ui.fake_pha("fabrizio", arf, rmf, 10000) model = ui.get_model("fabrizio") bare_model, _ = ui._session._get_model_status("fabrizio") y = bare_model.calc([1,1], model.xlo, model.xhi) y_m = numpy.mean(y) ui.set_analysis("fabrizio","wave") model2 = ui.get_model("fabrizio") bare_model2, _ = ui._session._get_model_status("fabrizio") y2 = bare_model2.calc([1,1], model2.xlo, model2.xhi) y2_m = numpy.mean(y2) self.assertAlmostEqual(y_m, y2_m)
def test_set_analysis_wave_fabrizio(self): rmf = self.make_path('3c273.rmf') arf = self.make_path('3c273.arf') ui.set_model("fabrizio", "xspowerlaw.p1") ui.fake_pha("fabrizio", arf, rmf, 10000) parvals = [1, 1] model = ui.get_model("fabrizio") bare_model, _ = ui._session._get_model_status("fabrizio") y = bare_model.calc(parvals, model.xlo, model.xhi) y_m = numpy.mean(y) ui.set_analysis("fabrizio", "wave") model2 = ui.get_model("fabrizio") bare_model2, _ = ui._session._get_model_status("fabrizio") y2 = bare_model2.calc(parvals, model2.xlo, model2.xhi) y2_m = numpy.mean(y2) self.assertAlmostEqual(y_m, y2_m)
def test_set_analysis_wave_fabrizio(clean_astro_ui, make_data_path): rmf = make_data_path('3c273.rmf') arf = make_data_path('3c273.arf') ui.set_model("fabrizio", "xspowerlaw.p1") ui.fake_pha("fabrizio", arf, rmf, 10000) parvals = [1, 1] model = ui.get_model("fabrizio") bare_model, _ = ui._session._get_model_status("fabrizio") y = bare_model.calc(parvals, model.xlo, model.xhi) y_m = numpy.mean(y) ui.set_analysis("fabrizio", "wave") model2 = ui.get_model("fabrizio") bare_model2, _ = ui._session._get_model_status("fabrizio") y2 = bare_model2.calc(parvals, model2.xlo, model2.xhi) y2_m = numpy.mean(y2) assert y2_m == pytest.approx(y_m)
def saveSrcModel(id=1, writeTo='srcPowerLaw.json', stat=True, info={}): """ """ srcModel = ui.get_model(id=id) parDict = {p.fullname: p.val for p in srcModel.pars} if stat: fsrc, *_ = ui.get_stat_info() for i in ['statname', 'numpoints', 'dof', 'qval', 'rstat', 'statval']: parDict[i] = getattr(fsrc, i) for key, val in info.items(): parDict[key] = val with open(writeTo, 'w') as f: json.dump(parDict, f)
def sherpa_spec_fit(in_files, model, noplot, manual, reproj, do_conf): """Reads a set of pha files and performs a maximum likelihood fit. """ # from sherpa.astro.ui import * # TEST if needed (only when outside sherpa env) import sherpa.astro.ui as sau from . import load_model from . import make_plot from .specsource import SpecSource # Read and load the data and model: list_data = [] emax = 9e10 # Maximum energy taken into account for the fit --> Compute properly (as a function of the max E event?) # We don't use a specific maximum reduced statistic value since we don't expect the cstat to be anywhere near the large number limit sau.set_conf_opt("max_rstat", 10000) p1 = load_model.load_model( model[0]) # [-1] #load model returns an array of model components # where the last component is the total model spec = SpecSource('SRC', in_files) if reproj == 0: myspec = spec elif reproj == 3: myspec = spec.reproject(nbins={'offset': 5, 'eff': 10, 'zen': 10}) elif reproj == 2: myspec = spec.reproject(nbins={'offset': 5, 'eff': 20, 'zen': 12}) elif reproj == 1: myspec = spec.reproject(nbins={'offset': 25, 'eff': 40, 'zen': 30}) elif reproj == 4: myspec = spec.reproject(nbins={'offset': 1, 'eff': 1, 'zen': 1}) myspec.set_source(p1) if manual: load_model.set_manual_model( sau.get_model(datid)) # Parameters for all runs are linked else: print('Using default initial values for model parameters') myspec.fit(do_conf=do_conf) if noplot: quit() make_plot.make_plot(list_data, p1) raw_input('Press <ENTER> to continue')
def save_components(filename, compIDstosave=None): if os.path.isfile(filename): raise Exception("Model already exists.") with open(filename, 'w') as f: print('Saving ' + filename) print(shp.get_model()) if compIDstosave is None: compIDstosave = [c for c in shp.list_model_components() if c not in ['emap', 'psf']] f.write('components: ' + str(len(compIDstosave)) + '\n') for compID in compIDstosave: if compID not in ['psf', 'emap']: comp = shp.get_model_component(compID) f.write(comp.name.replace('.', ' ') + ' ' + str(len(comp.pars)) + '\n') for par in comp.pars: f.write(par.name + ' ' + str(par.min) + ' ' + str(par.val) + ' ' + str(par.max) + '\n') f.close()
def get_source_qq_data(id=None): """Get data for a quantile-quantile plot of the source data and model. *id* The dataset id for which to get the data; defaults if unspecified. Returns: An ndarray of shape ``(3, npts)``. The first slice is the energy axis in keV; the second is the observed values in each bin (counts, or rate, or rate per keV, etc.); the third is the corresponding model value in each bin. The inputs are implicit; the data are obtained from the current state of the Sherpa ``ui`` module. """ sdata = ui.get_data(id=id) kev = sdata.get_x() obs_data = sdata.counts model_data = ui.get_model(id=id)(kev) return np.vstack((kev, obs_data, model_data))
def get_model_parts(id = None): '''obtain a list of strings for sherpa models Iterate through all components which are part of the Sherpa model and return their identifiers. Ignore all composite models. Example ------- >>> from sherpa.ui import * >>> load_arrays(1, [1,2,3], [1,2,3]) # Set some dummy data >>> set_model('const1d.c + gauss1d.lineg1 + gauss1d.lineg2 + gauss1d.lineg3') >>> show_model() # doctest: +SKIP Model: 1 (((const1d.c + gauss1d.lineg1) + gauss1d.lineg2) + gauss1d.lineg3) ... >>> get_model_parts() # doctest: +SKIP {'c', 'lineg1', 'lineg2', 'lineg3'} ''' try: return set([par.modelname for par in get_model(id).pars]) except IdentifierErr: return set([])
def get_plot_arrays(self, data_list): """Construct arrays of model count rates.""" sample_model = sau.get_model(data_list[0].name) self.get_binning(sample_model) # do this only once assuming that true energy # binning does not change from run to run obs_exc = np.zeros_like(self.bcenter) obs_err = np.zeros_like(self.bcenter) tot_on = np.zeros_like(self.bcenter) tot_off = np.zeros_like(self.bcenter) mod_cnts = np.zeros_like(self.bcenter) exp_tot = np.zeros_like(self.etrue_center) mod_tot = np.zeros_like(self.etrue_center) for dat in data_list: datid = dat.name exposure = dat.data.exposure on_cnt_rate = dat.data.get_y() c_bkg = sau.get_bkg(datid) bg_cnt_rate = c_bkg.get_y() backscal = c_bkg.get_backscal() c_mod = sau.get_model(datid) arf = c_mod.arf arf_vals = arf.get_y() # Excess bw_expo = self.b_width * exposure on_cnts = on_cnt_rate * bw_expo off_cnts = bg_cnt_rate * bw_expo / backscal c_exc = on_cnts - off_cnts # excess counts c_exc_err2 = on_cnts + off_cnts / backscal # errors # model counts c_modcnts = c_mod.calc(self.para, 2.) # second parameter is dummy... # Consider only noticed bins valid = dat.data.get_noticed_channels().astype(int) valid -= np.ones_like(valid) # Channel id's start at 1! obs_exc[valid] = obs_exc[valid] + c_exc[valid] # Total excess in noticed bins obs_err[valid] = obs_err[valid] + c_exc_err2[valid] # Total error square tot_on[valid] = tot_on[valid] + on_cnts[valid] tot_off[valid] = tot_off[valid] + off_cnts[valid] mod_cnts[valid] = mod_cnts[valid] + c_modcnts[valid] # Total noticed model counts valid_arf = self.ener_map[valid].sum(0) > 0 # valid pixels in true energy self.get_mod_val(self.totmodel, self.etrue_center) # Add run exposure*area*model for valid true energy bins only exp_tot[valid_arf] = exp_tot[valid_arf] + \ arf_vals[valid_arf] * self.mod_val[valid_arf] * exposure ''' Not used, may be useful to produce upper limits #significance per bin: signis = significance(n_observed=tot_on, mu_background=tot_off, method='lima') some_significant = False #makeUL = [] for i,signi in enumerate(signis): if signi<2: print('WARNING: Energy bin from', round(binmin[i]/1e9,1), 'to', \ round(binmax[i]/1e9,1), 'TeV has', round(signi,2), 'sigma only.') print('...may want to convert to upper limit') # NOT YET IMPLEMENTED continue #makeUL.append(True) if np.isinf(signi) or np.isnan(signi): #isinf when Non = Noff = 0? if some_significant: # otherwise we are probably below threshold print('WARNING: Energy bin from', round(binmin[i]/1e9,1), 'to', \ round(binmax[i]/1e9,1), 'TeV contains no events.') continue else: some_significant = True ''' # compute average exposure (time*area) in each measured energy bin mean_expo = np.zeros(obs_exc.shape) for i in range(obs_exc.shape[0]): mean_expo[i] = exp_tot[self.ener_map[i, :]].sum() / \ self.mod_val[self.ener_map[i, :]].sum() bw_meanexpo = self.b_width * mean_expo # get flux and error per cm^2/s/TeV self.mean_flux = 1e9 * obs_exc / bw_meanexpo self.mean_flux[np.isnan(self.mean_flux)] = 0 self.mean_err = 1e9 * np.sqrt(obs_err) / bw_meanexpo # mean_flux/signis # Compute residuals where model counts >0 self.resid = (-mod_cnts + obs_exc) / np.sqrt(obs_err) # Model spectral points self.bcenter /= 1e9 # keV? Nope, real high energy...
def renorm(id=None, cpt=None, bkg_id=None, names=None, limscale=1000.0): """Change the normalization of a model to match the data. The idea is to change the normalization to be a better match to the data, so that the search can be quicker. It can be considered to be like the `guess` command, but for the normalization. It is *only* intended to change the normalization to a value near the correct one; it *should not* be used for any sort of calculation without first doing a fit. It is also only going to give reasonable results for models where the predicted data of a model is linearly related to the normalization. Parameters ---------- id : None, int, or str The data set identifier to use. A value of ``None`` uses the default identifier. cpt If not ``None``, the model component to use. When ``None``, the full source expression for the data set is used. There is no check that the ``id`` argument matches the component (i.e. that the component is included in the source model for the data set) bkg_id : None, int If not None then change the normalization of the model to the given background dataset. names : None or array of str The parameter names that should be changed (a case-insensitive comparison is made, and the name does not include the model name). If ``None`` then the default set of ``['ampl', 'norm']`` is used. limscale : float The min and max range of the normalization is set to the calculated value divided and multiplied by ``limscale``. These limits will be modified to match the hard limits of the parameter if they exceed them. See Also -------- guess, ignore, notice, set_par Notes ----- The normalization is computed so that the predicted model counts matches the observed counts for the currently-noticed data range, as long as parameter names match the ``names`` argument (or ['ampl', 'norm'] if that is ``None``) and the parameter is not frozen. If no matches are found, then no changes are made. Otherwise, a scale factor is created by summing up the data counts and dividing this by the model sum over the currently-noticed range. This scale factor is divided by the number of matching parameters, and then the parameter values are multiplied by this value. If a model contains multiple parameters matching the contents of the ``names`` argument then each one will be changed by this routine. It is not intended for use with source expressions created with `set_full_model`, and may not work well with image models that use a PSF (one set with `set_psf`). Examples -------- Adjust the normalization of the gal component before fitting. >>> load_pha('src.pi') >>> subtract() >>> notice(0.5, 7) >>> set_source(xsphabs.galabs * xsapec.gal) >>> renorm() Change the normalization of a 2D model using the 'src' dataset. Only the ``src`` component is changed since the default value for the ``names`` parameter - that is ['ampl', 'norm'] - does not match the normalization parameter of the `const2d` model. >>> load_image('src', 'img.fits') >>> set_source('src', gauss2d.src + const2d.bgnd) >>> renorm('src') The names parameter is set so that both components are adjusted, and each component is assumed to contribute half the signal. >>> load_image(12, 'img.fits') >>> notice2d_id(12, 'srcfit.reg') >>> set_source(12, gauss2d.src12 + const2d.bgnd12) >>> renorm(12, names=['ampl', 'c0']) Change the minimum and maximum values of the normalization parameter to be the calculated value divided by and multiplied by 1e4 respectively (these changes are made to the soft limits). >>> renorm(limscale=1e4) """ if names is None: matches = ['ampl', 'norm'] elif names == []: raise ArgumentErr('bad', 'names argument', '[]') else: matches = [n.lower() for n in names] if bkg_id is None: d = ui.get_data(id=id) m = ui.get_model(id=id) else: d = ui.get_bkg(id=id, bkg_id=id) m = ui.get_bkg_model(id=id, bkg_id=bkg_id) if cpt is not None: # In this case the get_[bkg_]model call is not needed above, # but leave in as it at least ensures there's a model defined # for the data set. m = cpt pars = [p for p in m.pars if p.name.lower() in matches and not p.frozen] npars = len(pars) if npars == 0: wmsg = "no thawed parameters found matching: {}".format( ", ".join(matches)) warn(wmsg) return yd = d.get_dep(filter=True).sum() ym = d.eval_model_to_fit(m).sum() # argh; these are numpy floats, and they do not throw a # ZeroDivisionError, rather you get a RuntimeWarning message. # So explicitly convert to Python float. # try: scale = float(yd) / float(ym) / npars except ZeroDivisionError: error("model sum evaluated to 0; no re-scaling attempted") return for p in pars: newval = p.val * scale newmin = newval / limscale newmax = newval * limscale # Could do the limit/range checks and then call set_par, # but only do so if there's a problem. # try: ui.set_par(p, val=newval, min=newmin, max=newmax) except ParameterErr: # The following is not guaranteed to catch all cases; # e.g if the new value is outside the hard limits. # minflag = newmin < p.hard_min maxflag = newmax > p.hard_max if minflag: newmin = p.hard_min if maxflag: newmax = p.hard_max ui.set_par(p, val=newval, min=newmin, max=newmax) # provide informational message after changing the # parameter if minflag and maxflag: reason = "to hard min and max limits" elif minflag: reason = "to the hard minimum limit" elif maxflag: reason = "to the hard maximum limit" else: # this should be impossible reason = "for an unknown reason" info("Parameter {} is restricted ".format(p.fullname) + reason)
def test_pileup_model(make_data_path, clean_astro_ui): """Basic check of setting a pileup model. It is more to check we can set a pileup model, not to check the model works. """ infile = make_data_path('3c273.pi') ui.load_pha('pileup', infile) ui.subtract('pileup') ui.notice(0.3, 7) ui.set_stat('chi2datavar') # pick xswabs as it is unlikely to change with XSPEC ui.set_source('pileup', ui.xswabs.amdl * ui.powlaw1d.pl) # get close to the best fit, but don't need to fit pl.ampl = 1.82e-4 pl.gamma = 1.97 amdl.nh = 0.012 stat0 = ui.calc_stat('pileup') # We want to compare the data to the pileup model, # which should be run with the higher-energy bins included, # but that's not relevant here where I am just # checking the statistic value. ui.set_pileup_model('pileup', ui.jdpileup.jdp) # pick some values to make the model change the data jdp.ftime = 3.2 jdp.fracexp = 1 jdp.alpha = 0.95 jdp.f = 0.91 # Check pileup is added to get_model # mlines = str(ui.get_model('pileup')).split('\n') assert mlines[0] == 'apply_rmf(jdpileup.jdp((xswabs.amdl * powlaw1d.pl)))' assert mlines[3].strip( ) == 'jdp.alpha thawed 0.95 0 1' assert mlines[5].strip( ) == 'jdp.f thawed 0.91 0.9 1' assert mlines[11].strip( ) == 'pl.gamma thawed 1.97 -10 10' # Ensure that the statistic has got worse (technically # it coud get better, but not for this case). # stat1 = ui.calc_stat('pileup') assert stat1 > stat0 # As a test, check the actual statistic values # (evaluated with XSPEC 12.11.0 and Sherpa with the # master branch 2020/07/29, on Linux). # assert stat0 == pytest.approx(35.99899827358692) assert stat1 == pytest.approx(36.58791181460404) # Can we remove the pileup model? # ui.delete_pileup_model('pileup') # Check pileup is not in get_model # mlines = str(ui.get_model('pileup')).split('\n') # Numeric display depends on Python and/or NumPy # for the exposure time. I should have set the exposure # time to an integer value. # # assert mlines[0] == 'apply_rmf(apply_arf((38564.608926889 * (xswabs.amdl * powlaw1d.pl))))' toks = mlines[0].split() assert len(toks) == 5 assert toks[0].startswith('apply_rmf(apply_arf((38564.608') assert toks[1] == '*' assert toks[2] == '(xswabs.amdl' assert toks[3] == '*' assert toks[4] == 'powlaw1d.pl))))' assert mlines[4].strip( ) == 'pl.gamma thawed 1.97 -10 10' stat2 = ui.calc_stat('pileup') assert stat2 == stat0
# inner radii chosen for r in [20, 24, 28, 32]: # from KP15, (ra, dec) = (266.4172 deg, -29.00716 deg) # systematic shifting error = 6" = 2.4 px # ellip = .52 # theta = 57 deg (from positive north) shp.set_par(chxe.xpos, 497.4, 497.4 - 4, 497.4 + 4) shp.set_par(chxe.ypos, 499.1, 499.1 - 4, 499.1 + 4) shp.set_par(chxe.fwhm, 30, 1, 200) shp.set_par(chxe.ellip, .5) shp.set_par(chxe.theta, -.9948) shp.set_par(chxe.ampl, 1e-5) shp.thaw(chxe.ellip, chxe.xpos, chxe.ypos) print(shp.get_model()) shp.ignore2d('circle(500,500,1000)') shp.notice2d('circle(500,500,60)') shp.ignore2d('circle(500,500,' + str(r) + ')') shp.fit() print(shp.get_model()) shp.set_conf_opt('numcores', 3) shp.set_conf_opt('maxiters', 50) shp.set_conf_opt('fast', True) shp.set_conf_opt('remin', 10000.0) shp.set_conf_opt('soft_limits', True) shp.freeze(chxe.xpos, chxe.ypos)
def fit_with_sherpa(model, data, trans, rows, ranges=[], errs=None, shmod=1, method='levmar'): """ This is probably going to be one of the slightly more complicated functions, so it'll probably need a relatively good and comprehensive docstring. Parameters ----------- model : pandas DataFrame object. Must either have a 3-level index as created by grism, or at least a one-level index identifying the different components. If a three-level index is passed, it must either contain only one value of the Transition and Row values, or the 'trans' and 'row' kwargs must be passed in the call. data : numpy array or pandas Series or DataFrame. The data is assumed to have at least two columns containing first wavelength, then data. An optional third column can contain errors. If no such column exists, the errors are assumed to be 1. trans : string The desired value of the 'Transition' level of the model dataframe. rows : string The desired value of the 'Rows' level of the model index. shmod : integer Identifier for the Sherpa model in case one wants to have more models loaded in memory simultaneously. If not, it will be overwritten each time fit_with_sherpa() is run. Default: 1 ranges : list of (min, max) tuples. The determines which wavelength ranges will be included in the fit. If an empty list is passed, the entire range of the data passed will be used. Default: [] (Empty list). method : string optimization to be used by Sherpa. Can be either 'levmar', 'neldermead' or 'moncar'. See Sherpa documentation for more detail. Default: 'levmar'. """ # Should this perhaps be an instancemethod of one of the major classes # instead? On the upside it would mean direct access to all said class's # attributes, which is good because it means less mandatory input # parameters. On the downside, it is not generally useable. I want things # to be as general as possible. But not at any price. Cost/benifit analysis # not yet conclusive. # First of all, check if Sherpa is even installed. try: import sherpa.astro.ui as ai import sherpa.models as sm except ImportError: print " ".join("The Sherpa fitting software must be installed to use \ this functionality.".split()) raise # Sherpa isn't good at staying clean, need to help it. for i in ai.list_model_ids(): ai.delete_model(shmod) ai.delete_data(shmod) # Load data first, 'cause Sherpa wants it so. if data.shape[0] == 2: ai.load_arrays(shmod, data[:, 0], data[:, 1]) if data.shape[0] > 2: ai.load_arrays(shmod, data[:, 0], data[:, 1], data[:, 2]) # Initialize model by setting continuum Contin = sm.Const1D('Contin') Contin.c0 = model.xs('Contin')['Ampl'] ai.set_model(shmod, Contin) for i in model.index: if i == 'Contin': continue else: # use the identifier as letter (good idea?) name = model.ix[i]['Identifier'] comp = ai.gauss1d(name) comp.ampl = model.ix[i]['Ampl'] comp.pos = model.ix[i]['Pos'] comp.fwhm = model.ix[i]['Sigma'] ai.set_model(shmod, ai.get_model(shmod) + comp) ai.show_model(shmod) # For testing... print ' ' print ai.get_model(shmod) # Set ranges included in fit. # First, unset all. ai.ignore_id(shmod) if len(ranges) == 0: ai.notice_id(shmod) else: for r in ranges: ai.notice_id(shmod, r[0], r[1]) # Set optimization algorithm ai.set_method(method) # Create copy of model new_model = model.copy() # Perform the fit: ai.fit(shmod) print ai.get_fit_results() print model return new_model
def fit(self): # try a PCA decomposition of this spectrum initial = self.decompose() ui.set_method('neldermead') bkgmodel = PCAModel('pca%s' % self.id, data=self.pca) self.bkgmodel = bkgmodel response = get_identity_response(self.id) convbkgmodel = response(bkgmodel) ui.set_bkg_full_model(self.id, convbkgmodel) for p, v in zip(bkgmodel.pars, initial): p.val = v srcmodel = ui.get_model(self.id) ui.set_full_model(self.id, srcmodel) initial_v = self.calc_bkg_stat() # print('before fit: stat: %s' % (initial_v)) ui.fit_bkg(id=self.id) # print('fit: first full fit done') final = [p.val for p in ui.get_bkg_model(self.id).pars] # print('fit: parameters: %s' % (final)) initial_v = self.calc_bkg_stat() # print('fit: stat: %s' % (initial_v)) # lets try from zero # logf.info('fit: second full fit from zero') for p in bkgmodel.pars: p.val = 0 ui.fit_bkg(id=self.id) initial_v0 = self.calc_bkg_stat() # logf.info('fit: parameters: %s' % (final)) # logf.info('fit: stat: %s' % (initial_v0)) # pick the better starting point if initial_v0 < initial_v: # logf.info('fit: using zero-fit') initial_v = initial_v0 final = [p.val for p in ui.get_bkg_model(self.id).pars] else: # logf.info('fit: using decomposed-fit') for p, v in zip(bkgmodel.pars, final): p.val = v # start with the full fit and remove(freeze) parameters print('%d parameters, stat=%.2f' % (len(initial), initial_v)) results = [(2 * len(final) + initial_v, final, len(final), initial_v)] for i in range(len(initial) - 1, 0, -1): bkgmodel.pars[i].val = 0 bkgmodel.pars[i].freeze() ui.fit_bkg(id=self.id) final = [p.val for p in ui.get_bkg_model(self.id).pars] v = self.calc_bkg_stat() print('--> %d parameters, stat=%.2f' % (i, v)) results.insert(0, (v + 2 * i, final, i, v)) print() print('Background PCA fitting AIC results:') print('-----------------------------------') print() print('stat Ncomp AIC') for aic, params, nparams, val in results: print('%-05.1f %2d %-05.1f' % (val, nparams, aic)) aic, final, nparams, val = min(results) for p, v in zip(bkgmodel.pars, final): p.val = v for i in range(nparams): bkgmodel.pars[i].thaw() print() print('Increasing parameters again...') # now increase the number of parameters again # results = [(aic, final, nparams, val)] last_aic, last_final, last_nparams, last_val = aic, final, nparams, val for i in range(last_nparams, len(bkgmodel.pars)): next_nparams = i + 1 bkgmodel.pars[i].thaw() for p, v in zip(bkgmodel.pars, last_final): p.val = v ui.fit_bkg(id=self.id) next_final = [p.val for p in ui.get_bkg_model(self.id).pars] v = self.calc_bkg_stat() next_aic = v + 2 * next_nparams if next_aic < last_aic: # accept print('%d parameters, aic=%.2f ** accepting' % (next_nparams, next_aic)) last_aic, last_final, last_nparams, last_val = next_aic, next_final, next_nparams, v else: print('%d parameters, aic=%.2f' % (next_nparams, next_aic)) # stop if we are 3 parameters ahead what we needed if next_nparams >= last_nparams + 3: break print('Final choice: %d parameters, aic=%.2f' % (last_nparams, last_aic)) # reset to the last good solution for p, v in zip(bkgmodel.pars, last_final): p.val = v last_model = convbkgmodel for i in range(10): print('Adding Gaussian#%d' % (i + 1)) # find largest discrepancy ui.set_analysis(self.id, "ener", "rate") m = ui.get_bkg_fit_plot(self.id) y = m.dataplot.y.cumsum() z = m.modelplot.y.cumsum() diff_rate = np.abs(y - z) ui.set_analysis(self.id, "ener", "counts") m = ui.get_bkg_fit_plot(self.id) x = m.dataplot.x y = m.dataplot.y.cumsum() z = m.modelplot.y.cumsum() diff = np.abs(y - z) i = np.argmax(diff) energies = x e = x[i] print( 'largest remaining discrepancy at %.3fkeV[%d], need %d counts' % (x[i], i, diff[i])) # e = x[i] power = diff_rate[i] # lets try to inject a gaussian there g = ui.xsgaussian('g_%d_%d' % (self.id, i)) print('placing gaussian at %.2fkeV, with power %s' % (e, power)) # we work in energy bins, not energy g.LineE.min = energies[0] g.LineE.max = energies[-1] g.LineE.val = e if i > len(diff) - 2: i = len(diff) - 2 if i < 2: i = 2 g.Sigma = (x[i + 1] - x[i - 1]) g.Sigma.min = (x[i + 1] - x[i - 1]) / 3 g.Sigma.max = x[-1] - x[0] g.norm.min = power * 1e-6 g.norm.val = power convbkgmodel2 = response(g) next_model = last_model + convbkgmodel2 ui.set_bkg_full_model(self.id, next_model) ui.fit_bkg(id=self.id) next_final = [p.val for p in ui.get_bkg_model(self.id).pars] next_nparams = len(next_final) v = self.calc_bkg_stat() next_aic = v + 2 * next_nparams print('with Gaussian:', next_aic, '; change: %.1f (negative is good)' % (next_aic - last_aic)) if next_aic < last_aic: print('accepting') last_model = next_model last_aic, last_final, last_nparams, last_val = next_aic, next_final, next_nparams, v else: print('not significant, rejecting') ui.set_bkg_full_model(self.id, last_model) for p, v in zip(last_model.pars, last_final): p.val = v if v == 0: # the parameter was frozen. ui.freeze(p) break self.cstat, self.dof = self.calc_bkg_stat( dof=True) # Save the final cstat and dof (dof = ihi - ilo) self.filter_energy = ui.get_filter( ) # Save the filter for background fitting. ui.set_analysis('channel') ui.ignore() ui.notice(self.filter0) # restore filter ui.set_analysis('energy')
def get_plot_arrays(self, data_list): """Construct arrays of model count rates.""" sample_model = sau.get_model(data_list[0].name) self.get_binning( sample_model) # do this only once assuming that true energy # binning does not change from run to run obs_exc = np.zeros_like(self.bcenter) obs_err = np.zeros_like(self.bcenter) tot_on = np.zeros_like(self.bcenter) tot_off = np.zeros_like(self.bcenter) mod_cnts = np.zeros_like(self.bcenter) exp_tot = np.zeros_like(self.etrue_center) mod_tot = np.zeros_like(self.etrue_center) for dat in data_list: datid = dat.name exposure = dat.data.exposure on_cnt_rate = dat.data.get_y() c_bkg = sau.get_bkg(datid) bg_cnt_rate = c_bkg.get_y() backscal = c_bkg.get_backscal() c_mod = sau.get_model(datid) arf = c_mod.arf arf_vals = arf.get_y() # Excess bw_expo = self.b_width * exposure on_cnts = on_cnt_rate * bw_expo off_cnts = bg_cnt_rate * bw_expo / backscal c_exc = on_cnts - off_cnts # excess counts c_exc_err2 = on_cnts + off_cnts / backscal # errors # model counts c_modcnts = c_mod.calc(self.para, 2.) # second parameter is dummy... # Consider only noticed bins valid = dat.data.get_noticed_channels().astype(int) valid -= np.ones_like(valid) # Channel id's start at 1! obs_exc[valid] = obs_exc[valid] + c_exc[ valid] # Total excess in noticed bins obs_err[valid] = obs_err[valid] + c_exc_err2[ valid] # Total error square tot_on[valid] = tot_on[valid] + on_cnts[valid] tot_off[valid] = tot_off[valid] + off_cnts[valid] mod_cnts[valid] = mod_cnts[valid] + c_modcnts[ valid] # Total noticed model counts valid_arf = self.ener_map[valid].sum( 0) > 0 # valid pixels in true energy self.get_mod_val(self.totmodel, self.etrue_center) # Add run exposure*area*model for valid true energy bins only exp_tot[valid_arf] = exp_tot[valid_arf] + \ arf_vals[valid_arf] * self.mod_val[valid_arf] * exposure ''' Not used, may be useful to produce upper limits #significance per bin: signis = significance(n_observed=tot_on, mu_background=tot_off, method='lima') some_significant = False #makeUL = [] for i,signi in enumerate(signis): if signi<2: print('WARNING: Energy bin from', round(binmin[i]/1e9,1), 'to', \ round(binmax[i]/1e9,1), 'TeV has', round(signi,2), 'sigma only.') print('...may want to convert to upper limit') # NOT YET IMPLEMENTED continue #makeUL.append(True) if np.isinf(signi) or np.isnan(signi): #isinf when Non = Noff = 0? if some_significant: # otherwise we are probably below threshold print('WARNING: Energy bin from', round(binmin[i]/1e9,1), 'to', \ round(binmax[i]/1e9,1), 'TeV contains no events.') continue else: some_significant = True ''' # compute average exposure (time*area) in each measured energy bin mean_expo = np.zeros(obs_exc.shape) for i in range(obs_exc.shape[0]): mean_expo[i] = exp_tot[self.ener_map[i, :]].sum() / \ self.mod_val[self.ener_map[i, :]].sum() bw_meanexpo = self.b_width * mean_expo # get flux and error per cm^2/s/TeV self.mean_flux = 1e9 * obs_exc / bw_meanexpo self.mean_flux[np.isnan(self.mean_flux)] = 0 self.mean_err = 1e9 * np.sqrt( obs_err) / bw_meanexpo # mean_flux/signis # Compute residuals where model counts >0 self.resid = (-mod_cnts + obs_exc) / np.sqrt(obs_err) # Model spectral points self.bcenter /= 1e9 # keV? Nope, real high energy...
parameters += zparameters else: prefix += 'zfree_' parameters += [redshift] priors += [priorfuncs.create_uniform_prior_for(redshift)] assert len(priors) == len( parameters), 'priors: %d parameters: %d' % (len(priors), len(parameters)) ################ # set model # find background automatically using PCA method print('setting source and background model ...') set_model(id, model * galabso) convmodel = get_model(id) bkg_model = auto_background(id) set_full_model(id, get_response(id)(model) + bkg_model * get_bkg_scale(id)) #plot_bkg_fit(id) ## we allow the background normalisation to be a free fitting parameter p = bkg_model.pars[0] p.max = p.val + 2 p.min = p.val - 2 parameters.append(p) priors += [priorfuncs.create_uniform_prior_for(p)] priorfunction = priorfuncs.create_prior_function(priors=priors) assert not numpy.isnan( calc_stat(id)), 'NaN on calc_stat, probably a bad RMF/ARF file for PC'