def calcChiSquared(self): ''' Calculate the chi_squared value for given models and data. If integrated fluxes are available, they are used without the line blends. If not, the modeled, convolved spectrum is compared directly with the data, where the model is not just 0 flux. ''' inst = self.instrument for istar,star in enumerate(self.star_grid): this_id = star['LAST_%s_MODEL'%inst.instrument.upper()] #-- Get all integrated flux chi2s, add them up for a single model # and divide by the amount of comparisons. Line blends not incl. # If no integrated flux available, set chi2_inttot[this_id] to 0 all_chi2s = [] [all_chi2s.extend(dd[this_id]) for dd in self.chi2_intsi.values()] if not all_chi2s: self.chi2_inttot[this_id] = 0 else: self.chi2_inttot[this_id] = sum(all_chi2s)/len(all_chi2s) #-- Calculate chi2 based on the convolved model with respect to the # continuum-subtracted data. all_dflux = [] all_mflux = [] all_dstd = [] for fn,dflux in zip(inst.data_filenames,inst.data_flux_list): dstd = self.data_stats[fn]['std'] #-- Cannot return empty list as the selection of existing # convolutions is done in Statistics.setModels() mflux = inst.getSphinxConvolution(star,fn)[1] all_dflux.extend(dflux[mflux>0]) all_mflux.extend(mflux[mflux>0]) all_dstd.extend([dstd]*len(mflux[mflux>0])) self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,all_mflux,\ all_dstd)
def calcChiSquared(self): ''' Calculate the chi_squared value for given models and data. If integrated fluxes are available, they are used without the line blends. If not, the modeled, convolved spectrum is compared directly with the data, where the model is not just 0 flux. ''' inst = self.instrument for istar, star in enumerate(self.star_grid): this_id = star['LAST_%s_MODEL' % inst.instrument.upper()] #-- Get all integrated flux chi2s, add them up for a single model # and divide by the amount of comparisons. Line blends not incl. # If no integrated flux available, set chi2_inttot[this_id] to 0 all_chi2s = [] [all_chi2s.extend(dd[this_id]) for dd in self.chi2_intsi.values()] if not all_chi2s: self.chi2_inttot[this_id] = 0 else: self.chi2_inttot[this_id] = sum(all_chi2s) / len(all_chi2s) #-- Calculate chi2 based on the convolved model with respect to the # continuum-subtracted data. all_dflux = [] all_mflux = [] all_dstd = [] for fn, dflux in zip(inst.data_filenames, inst.data_flux_list): dstd = self.data_stats[fn]['std'] #-- Cannot return empty list as the selection of existing # convolutions is done in Statistics.setModels() mflux = inst.getSphinxConvolution(star, fn)[1] all_dflux.extend(dflux[mflux > 0]) all_mflux.extend(mflux[mflux > 0]) all_dstd.extend([dstd] * len(mflux[mflux > 0])) self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,all_mflux,\ all_dstd)
def __setIntRatios(self,ifn,fn,chi2_type='normal'): ''' Calculate ratios of integrated intensities, if requested. Assumes there is a linefit filename available in the Instrument() object. Only done for those line present in this file, based on Doppler shifted wavelength. @param ifn: Index of the data band in self.instrument lists. @type ifn: int @param fn: The filename of the data set. Needed for book keeping. @type fn: string @keyword chi2_type: The type of chi-squared calculated for integrated fluxes. 'normal' for the usual kind, 'log' for chi2 of the log of the integrated fluxes and noise. (default: normal) @type chi2_type: string ''' #-- Get some data properties, and extract data wavelength and flux inst = self.instrument self.int_ratios[fn] = dict() self.int_ratios_err[fn] = dict() self.chi2_intsi[fn] = dict() #-- Comparing integrated intensities between PACS and models. # Comparisons only made per filename! inst.intIntMatch(trans_list=self.sample_trans[fn],ifn=ifn) for star in self.star_grid: #-- From here on, we start extracting the model specific int ints. this_id = star['LAST_%s_MODEL'%inst.instrument.upper()] mtrans = array([star.getTransition(t) for t in self.sample_trans[fn]]) these_ratios = [] these_errs = [] self.chi2_intsi[fn][this_id] = [] for mt,st in zip(mtrans,self.sample_trans[fn]): # 4) No trans == sample_trans found for this model, or sample # trans does not contain a PACS integrated intensity. if mt is None or st.getIntIntUnresolved(fn)[0] is None or \ st.getIntIntUnresolved(fn)[0] == 'inblend': these_ratios.append(None) these_errs.append(None) # 5) Match found with a wave_fit value. Get int ratio # m/d. If dintint is negative, it is a blend due to large # FWHM! If blends is not None, multiple sample trans have # been found in the wavelength resolution bin of the # fitted line and also indicates a blend. else: dintint, dintinterr, blends = st.getIntIntUnresolved(fn) if blends is None: mintint = mt.getIntIntIntSphinx() else: #-- blends is a list of sample transitions that refers # to the transitions involved in the blend, so get # these from the model grid, add them up and make # sure the ratio will be negative to indicate a blend blendlines = [star.getTransition(t) for t in blends if star.getTransition(t) <> None] mintint = sum([t.getIntIntIntSphinx() for t in blendlines]) dintint = -1.*abs(dintint) if dintint > 0 and not mt.sphinx.nans_present: if chi2_type == 'log': ichi2 = bs.calcChiSquared(log10(dintint),\ log10(mintint),\ log10(dintint*dintinterr)) else: ichi2 = bs.calcChiSquared(dintint,\ mintint,\ dintint*dintinterr) #ichi2 = bs.calcLoglikelihood(dintint,\ # mintint,\ # dintint*dintinterr) self.chi2_intsi[fn][this_id].append(ichi2) this_ratio = mintint/dintint these_ratios.append(this_ratio) these_errs.append(abs(this_ratio)*dintinterr) self.int_ratios[fn][this_id] = these_ratios self.int_ratios_err[fn][this_id] = these_errs
def calcChi2(self,ndf=0,fns=None,cwave=0.0,phot_ivs=1,sort=1,\ chi2_method='diff'): ''' Calculate, save and return the chi^2 values for a given set of models. For now, only photometry (both with and without photometric bands available, the latter being associated with the Photometric_IvS file) is taken into account. You can specify the number of degrees of freedom for the reduced chi^2 in case you are comparing models probing a grid across ndf different parameters. The method overrides previously calculated chi2 in the object. Write the chi^2 to a file before running the method if you want to save the values with self.writeChi2() @keyword ndf: Number of degrees of freedom. Default in case of calculating for one single model. Typically the number of variable grid parameters in a grid calculation. (default: 0) @type ndf: int @keyword fns: The photometric filenames to be included in the chi^2 calc Default if all are to be included. Empty list if none are to be included. (default: None) @type fns: list(str) @keyword cwave: A cut-off wavelength in micron used to calculate the chi^2 stat for all photometry above or equal to the wavelength in micron. Use a negative value to grab all wavelengths lower than cwave. Default if no cut-off. (default: 0.0) @type cwave: float @keyword phot_ivs: Include the Photometric_IvS photometry as well. (default: 1) @type phot_ivs: bool @keyword sort: Sort the chi^2 values from lowest to highest. (default: 1) @type sort: bool @keyword chi2_method: Method for calculating chi^2. Can be diff or log (see BasicStats for more information) (default: 'diff') @type chi2_method: str @return: The list of chi^2 values with the same length as the model grid of valid MCMax models. @rtype: list ''' #-- Don't use [[]]*len(self.star_grid). This only multiplies the pointer # to the same list, it doesn't do copy([]). mphot = [[] for s in self.star_grid] dphot = [] ephot = [] if fns is None: fns = self.dphot_other.keys() print('No phot files for chi2 given. Using files in usr/Sed.dat.') else: fns = [fn if os.path.split(fn)[0] else os.path.join(cc.path.dsed,fn) for fn in fns] fns = [fn for fn in fns if fn in self.dphot_other.keys()] #-- When no valid filenames are given, and IvS phot is not requested, # just forget the filename selection. if not fns and not phot_ivs: print('No valid phot files given. Make sure to include them in '+\ 'usr/Sed.dat! Taking files from usr/Sed.dat now instead.') fns = self.dphot_other.keys() for fn in fns: #-- Includes cwave == 0.0, in which case everything is selected if cwave >= 0.0: keep = self.dphot_other[fn]['wave'] >= cwave else: keep = self.dphot_other[fn]['wave'] < -cwave #-- Check if anything is kept at all, otherwise move on if not keep.any(): continue #-- Add the model and data points to the list for i,iphot in enumerate(self.mphot_other[fn]): mphot[i].extend(iphot[keep]) dphot.extend(self.dphot_other[fn]['phot'][keep]) ephot.extend(self.dphot_other[fn]['ephot'][keep]) if phot_ivs and self.photbands.size: if cwave >= 0.0: keep = self.dphot_ivs['wave'] >= cwave else: keep = self.dphot_ivs['wave'] < -cwave #-- Check if anything is kept at all, otherwise move on if keep.any(): for i,iphot in enumerate(self.mphot_ivs): mphot[i].extend(iphot[keep]) dphot.extend(self.dphot_ivs['phot'][keep]) ephot.extend(self.dphot_ivs['ephot'][keep]) #-- If no data are selected at all, return an empty array if not dphot: self.chi2 = np.empty(0) return self.chi2 self.chi2 = [] for iphot in mphot: self.chi2.append(BasicStats.calcChiSquared(dphot,iphot,ephot,ndf,\ chi2_method)) self.chi2 = np.array(self.chi2) return np.sort(self.chi2) if sort else self.chi2
def calcChiSquared(self,chi2_method='diff',ndf=0,filename=None): ''' Calculate the chi_squared value for given models and data. If integrated fluxes are available, they are used without the line blends. LINES FLAGGED AS A BLEND ARE EXCLUDED FROM THE CHI2 CALCULATION. If not, the modeled, convolved spectrum is compared directly with the data, where the model is not just 0 flux. @keyword chi2_method: The type of chi-squared calculated for integrated fluxes. 'diff' for standard chi^2 kind, 'log' for redistributing the data/model ratios on an absolute logarithmic scale before calculating the chi2 (default: 'diff') @type chi2_method: string @keyword ndf: Number of degrees of freedom. Default in case of calculating for one single model. Typically the number of variable grid parameters in a grid calculation. (default: 0) @type ndf: int @keyword filename: the filename for which you want to return the list, if None, all filenames are used and the lists are merged into one (default: None) @type filename: string ''' if chi2_method not in ['diff','log']: chi2_method = 'diff' print "WARNING: STAT_CHI2 not recognized. Using default 'diff'." #-- For now excluding blends inst = self.instrument all_dints = self.getRatios(sel_type='dint_bands',data_type='dint_bands',\ filename=filename) all_derrs = self.getRatios(sel_type='dint_bands',data_type='derr_bands',\ filename=filename) for istar,star in enumerate(self.star_grid): this_id = star['LAST_%s_MODEL'%inst.instrument.upper()] #-- Get all integrated fluxes and errors for this particular model, # no matter the filename of the data (ie no matter the band). # If no integrated flux available, set chi2_inttot[this_id] to 0 all_mints = self.getRatios(this_id=this_id,sel_type='dint_bands',\ data_type='mint_bands',filename=filename) sel = np.isfinite(all_mints)*np.isfinite(all_dints) if not all_mints[sel].size: self.chi2_inttot[this_id] = 0 else: chi2 = bs.calcChiSquared(data=all_dints,model=all_mints,\ noise=all_derrs*all_dints,\ mode=chi2_method,ndf=ndf) self.chi2_inttot[this_id] = chi2 #-- Calculate chi2 based on the convolved model with respect to the # continuum-subtracted data. all_dflux = [] all_mflux = [] all_dstd = [] if not filename: fns = inst.data_filenames fluxes = inst.data_flux_list else: fns = filename is None and inst.data_filenames or [filename] fluxes = [inst.data_flux_list[inst.data_filenames.index(fn)] for fn in fns] for fn,dflux in zip(fns,fluxes): dstd = self.data_stats[fn]['std'] #-- Cannot return empty list as the selection of existing # convolutions is done in Statistics.setModels() mflux = inst.getSphinxConvolution(star,fn)[1] #-- Make sure all points are positive. Some data points may be # <0 due to continuum subtraction. mflux = mflux[dflux>0] dflux = dflux[dflux>0] all_dflux.extend(dflux[mflux>0]) all_mflux.extend(mflux[mflux>0]) all_dstd.extend([dstd]*len(mflux[mflux>0])) self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,\ all_mflux,all_dstd,\ mode=chi2_method,ndf=ndf)
def __setIntRatios(self, ifn, fn, chi2_type='normal'): ''' Calculate ratios of integrated intensities, if requested. Assumes there is a linefit filename available in the Instrument() object. Only done for those line present in this file, based on Doppler shifted wavelength. @param ifn: Index of the data band in self.instrument lists. @type ifn: int @param fn: The filename of the data set. Needed for book keeping. @type fn: string @keyword chi2_type: The type of chi-squared calculated for integrated fluxes. 'normal' for the usual kind, 'log' for chi2 of the log of the integrated fluxes and noise. (default: normal) @type chi2_type: string ''' #-- Get some data properties, and extract data wavelength and flux inst = self.instrument self.int_ratios[fn] = dict() self.int_ratios_err[fn] = dict() self.chi2_intsi[fn] = dict() #-- Comparing integrated intensities between PACS and models. # Comparisons only made per filename! inst.intIntMatch(trans_list=self.sample_trans[fn], ifn=ifn) for star in self.star_grid: #-- From here on, we start extracting the model specific int ints. this_id = star['LAST_%s_MODEL' % inst.instrument.upper()] mtrans = array( [star.getTransition(t) for t in self.sample_trans[fn]]) these_ratios = [] these_errs = [] self.chi2_intsi[fn][this_id] = [] for mt, st in zip(mtrans, self.sample_trans[fn]): # 4) No trans == sample_trans found for this model, or sample # trans does not contain a PACS integrated intensity. if mt is None or st.getIntIntUnresolved(fn)[0] is None or \ st.getIntIntUnresolved(fn)[0] == 'inblend': these_ratios.append(None) these_errs.append(None) # 5) Match found with a wave_fit value. Get int ratio # m/d. If dintint is negative, it is a blend due to large # FWHM! If blends is not None, multiple sample trans have # been found in the wavelength resolution bin of the # fitted line and also indicates a blend. else: dintint, dintinterr, blends = st.getIntIntUnresolved(fn) if blends is None: mintint = mt.getIntIntIntSphinx() else: #-- blends is a list of sample transitions that refers # to the transitions involved in the blend, so get # these from the model grid, add them up and make # sure the ratio will be negative to indicate a blend blendlines = [ star.getTransition(t) for t in blends if star.getTransition(t) <> None ] mintint = sum( [t.getIntIntIntSphinx() for t in blendlines]) dintint = -1. * abs(dintint) if dintint > 0 and not mt.sphinx.nans_present: if chi2_type == 'log': ichi2 = bs.calcChiSquared(log10(dintint),\ log10(mintint),\ log10(dintint*dintinterr)) else: ichi2 = bs.calcChiSquared(dintint,\ mintint,\ dintint*dintinterr) #ichi2 = bs.calcLoglikelihood(dintint,\ # mintint,\ # dintint*dintinterr) self.chi2_intsi[fn][this_id].append(ichi2) this_ratio = mintint / dintint these_ratios.append(this_ratio) these_errs.append(abs(this_ratio) * dintinterr) self.int_ratios[fn][this_id] = these_ratios self.int_ratios_err[fn][this_id] = these_errs
def calcChiSquared(self, chi2_method='diff', ndf=0, filename=None): ''' Calculate the chi_squared value for given models and data. If integrated fluxes are available, they are used without the line blends. LINES FLAGGED AS A BLEND ARE EXCLUDED FROM THE CHI2 CALCULATION. If not, the modeled, convolved spectrum is compared directly with the data, where the model is not just 0 flux. @keyword chi2_method: The type of chi-squared calculated for integrated fluxes. 'diff' for standard chi^2 kind, 'log' for redistributing the data/model ratios on an absolute logarithmic scale before calculating the chi2 (default: 'diff') @type chi2_method: string @keyword ndf: Number of degrees of freedom. Default in case of calculating for one single model. Typically the number of variable grid parameters in a grid calculation. (default: 0) @type ndf: int @keyword filename: the filename for which you want to return the list, if None, all filenames are used and the lists are merged into one (default: None) @type filename: string ''' if chi2_method not in ['diff', 'log']: chi2_method = 'diff' print "WARNING: STAT_CHI2 not recognized. Using default 'diff'." #-- For now excluding blends inst = self.instrument all_dints = self.getRatios(sel_type='dint_bands',data_type='dint_bands',\ filename=filename) all_derrs = self.getRatios(sel_type='dint_bands',data_type='derr_bands',\ filename=filename) for istar, star in enumerate(self.star_grid): this_id = star['LAST_%s_MODEL' % inst.instrument.upper()] #-- Get all integrated fluxes and errors for this particular model, # no matter the filename of the data (ie no matter the band). # If no integrated flux available, set chi2_inttot[this_id] to 0 all_mints = self.getRatios(this_id=this_id,sel_type='dint_bands',\ data_type='mint_bands',filename=filename) sel = np.isfinite(all_mints) * np.isfinite(all_dints) if not all_mints[sel].size: self.chi2_inttot[this_id] = 0 else: chi2 = bs.calcChiSquared(data=all_dints,model=all_mints,\ noise=all_derrs*all_dints,\ mode=chi2_method,ndf=ndf) self.chi2_inttot[this_id] = chi2 #-- Calculate chi2 based on the convolved model with respect to the # continuum-subtracted data. all_dflux = [] all_mflux = [] all_dstd = [] if not filename: fns = inst.data_filenames fluxes = inst.data_flux_list else: fns = filename is None and inst.data_filenames or [filename] fluxes = [ inst.data_flux_list[inst.data_filenames.index(fn)] for fn in fns ] for fn, dflux in zip(fns, fluxes): dstd = self.data_stats[fn]['std'] #-- Cannot return empty list as the selection of existing # convolutions is done in Statistics.setModels() mflux = inst.getSphinxConvolution(star, fn)[1] #-- Make sure all points are positive. Some data points may be # <0 due to continuum subtraction. mflux = mflux[dflux > 0] dflux = dflux[dflux > 0] all_dflux.extend(dflux[mflux > 0]) all_mflux.extend(mflux[mflux > 0]) all_dstd.extend([dstd] * len(mflux[mflux > 0])) self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,\ all_mflux,all_dstd,\ mode=chi2_method,ndf=ndf)