def calcChiSquared(self):
'''
Calculate the chi_squared value for given models and data.
If integrated fluxes are available, they are used without the line
blends.
If not, the modeled, convolved spectrum is compared directly with the
data, where the model is not just 0 flux.
'''
inst = self.instrument
for istar,star in enumerate(self.star_grid):
this_id = star['LAST_%s_MODEL'%inst.instrument.upper()]
#-- Get all integrated flux chi2s, add them up for a single model
# and divide by the amount of comparisons. Line blends not incl.
# If no integrated flux available, set chi2_inttot[this_id] to 0
all_chi2s = []
[all_chi2s.extend(dd[this_id]) for dd in self.chi2_intsi.values()]
if not all_chi2s:
self.chi2_inttot[this_id] = 0
else:
self.chi2_inttot[this_id] = sum(all_chi2s)/len(all_chi2s)
#-- Calculate chi2 based on the convolved model with respect to the
# continuum-subtracted data.
all_dflux = []
all_mflux = []
all_dstd = []
for fn,dflux in zip(inst.data_filenames,inst.data_flux_list):
dstd = self.data_stats[fn]['std']
#-- Cannot return empty list as the selection of existing
# convolutions is done in Statistics.setModels()
mflux = inst.getSphinxConvolution(star,fn)[1]
all_dflux.extend(dflux[mflux>0])
all_mflux.extend(mflux[mflux>0])
all_dstd.extend([dstd]*len(mflux[mflux>0]))
self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,all_mflux,\
all_dstd)
def calcChiSquared(self):
'''
Calculate the chi_squared value for given models and data.
If integrated fluxes are available, they are used without the line
blends.
If not, the modeled, convolved spectrum is compared directly with the
data, where the model is not just 0 flux.
'''
inst = self.instrument
for istar, star in enumerate(self.star_grid):
this_id = star['LAST_%s_MODEL' % inst.instrument.upper()]
#-- Get all integrated flux chi2s, add them up for a single model
# and divide by the amount of comparisons. Line blends not incl.
# If no integrated flux available, set chi2_inttot[this_id] to 0
all_chi2s = []
[all_chi2s.extend(dd[this_id]) for dd in self.chi2_intsi.values()]
if not all_chi2s:
self.chi2_inttot[this_id] = 0
else:
self.chi2_inttot[this_id] = sum(all_chi2s) / len(all_chi2s)
#-- Calculate chi2 based on the convolved model with respect to the
# continuum-subtracted data.
all_dflux = []
all_mflux = []
all_dstd = []
for fn, dflux in zip(inst.data_filenames, inst.data_flux_list):
dstd = self.data_stats[fn]['std']
#-- Cannot return empty list as the selection of existing
# convolutions is done in Statistics.setModels()
mflux = inst.getSphinxConvolution(star, fn)[1]
all_dflux.extend(dflux[mflux > 0])
all_mflux.extend(mflux[mflux > 0])
all_dstd.extend([dstd] * len(mflux[mflux > 0]))
self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,all_mflux,\
all_dstd)
def __setIntRatios(self,ifn,fn,chi2_type='normal'):
'''
Calculate ratios of integrated intensities, if requested.
Assumes there is a linefit filename available in the Instrument()
object. Only done for those line present in this file, based on
Doppler shifted wavelength.
@param ifn: Index of the data band in self.instrument lists.
@type ifn: int
@param fn: The filename of the data set. Needed for book keeping.
@type fn: string
@keyword chi2_type: The type of chi-squared calculated for integrated
fluxes. 'normal' for the usual kind, 'log' for chi2
of the log of the integrated fluxes and noise.
(default: normal)
@type chi2_type: string
'''
#-- Get some data properties, and extract data wavelength and flux
inst = self.instrument
self.int_ratios[fn] = dict()
self.int_ratios_err[fn] = dict()
self.chi2_intsi[fn] = dict()
#-- Comparing integrated intensities between PACS and models.
# Comparisons only made per filename!
inst.intIntMatch(trans_list=self.sample_trans[fn],ifn=ifn)
for star in self.star_grid:
#-- From here on, we start extracting the model specific int ints.
this_id = star['LAST_%s_MODEL'%inst.instrument.upper()]
mtrans = array([star.getTransition(t)
for t in self.sample_trans[fn]])
these_ratios = []
these_errs = []
self.chi2_intsi[fn][this_id] = []
for mt,st in zip(mtrans,self.sample_trans[fn]):
# 4) No trans == sample_trans found for this model, or sample
# trans does not contain a PACS integrated intensity.
if mt is None or st.getIntIntUnresolved(fn)[0] is None or \
st.getIntIntUnresolved(fn)[0] == 'inblend':
these_ratios.append(None)
these_errs.append(None)
# 5) Match found with a wave_fit value. Get int ratio
# m/d. If dintint is negative, it is a blend due to large
# FWHM! If blends is not None, multiple sample trans have
# been found in the wavelength resolution bin of the
# fitted line and also indicates a blend.
else:
dintint, dintinterr, blends = st.getIntIntUnresolved(fn)
if blends is None:
mintint = mt.getIntIntIntSphinx()
else:
#-- blends is a list of sample transitions that refers
# to the transitions involved in the blend, so get
# these from the model grid, add them up and make
# sure the ratio will be negative to indicate a blend
blendlines = [star.getTransition(t)
for t in blends
if star.getTransition(t) <> None]
mintint = sum([t.getIntIntIntSphinx()
for t in blendlines])
dintint = -1.*abs(dintint)
if dintint > 0 and not mt.sphinx.nans_present:
if chi2_type == 'log':
ichi2 = bs.calcChiSquared(log10(dintint),\
log10(mintint),\
log10(dintint*dintinterr))
else:
ichi2 = bs.calcChiSquared(dintint,\
mintint,\
dintint*dintinterr)
#ichi2 = bs.calcLoglikelihood(dintint,\
# mintint,\
# dintint*dintinterr)
self.chi2_intsi[fn][this_id].append(ichi2)
this_ratio = mintint/dintint
these_ratios.append(this_ratio)
these_errs.append(abs(this_ratio)*dintinterr)
self.int_ratios[fn][this_id] = these_ratios
self.int_ratios_err[fn][this_id] = these_errs
def calcChi2(self,ndf=0,fns=None,cwave=0.0,phot_ivs=1,sort=1,\
chi2_method='diff'):
'''
Calculate, save and return the chi^2 values for a given set of models.
For now, only photometry (both with and without photometric bands
available, the latter being associated with the Photometric_IvS file)
is taken into account.
You can specify the number of degrees of freedom for the reduced chi^2
in case you are comparing models probing a grid across ndf different
parameters.
The method overrides previously calculated chi2 in the object. Write the
chi^2 to a file before running the method if you want to save the values
with self.writeChi2()
@keyword ndf: Number of degrees of freedom. Default in case of
calculating for one single model. Typically the number of
variable grid parameters in a grid calculation.
(default: 0)
@type ndf: int
@keyword fns: The photometric filenames to be included in the chi^2 calc
Default if all are to be included. Empty list if none are
to be included.
(default: None)
@type fns: list(str)
@keyword cwave: A cut-off wavelength in micron used to calculate the
chi^2 stat for all photometry above or equal to the
wavelength in micron. Use a negative value to grab all
wavelengths lower than cwave. Default if no cut-off.
(default: 0.0)
@type cwave: float
@keyword phot_ivs: Include the Photometric_IvS photometry as well.
(default: 1)
@type phot_ivs: bool
@keyword sort: Sort the chi^2 values from lowest to highest.
(default: 1)
@type sort: bool
@keyword chi2_method: Method for calculating chi^2. Can be diff or
log (see BasicStats for more information)
(default: 'diff')
@type chi2_method: str
@return: The list of chi^2 values with the same length as the model grid
of valid MCMax models.
@rtype: list
'''
#-- Don't use [[]]*len(self.star_grid). This only multiplies the pointer
# to the same list, it doesn't do copy([]).
mphot = [[] for s in self.star_grid]
dphot = []
ephot = []
if fns is None:
fns = self.dphot_other.keys()
print('No phot files for chi2 given. Using files in usr/Sed.dat.')
else:
fns = [fn if os.path.split(fn)[0] else os.path.join(cc.path.dsed,fn)
for fn in fns]
fns = [fn for fn in fns if fn in self.dphot_other.keys()]
#-- When no valid filenames are given, and IvS phot is not requested,
# just forget the filename selection.
if not fns and not phot_ivs:
print('No valid phot files given. Make sure to include them in '+\
'usr/Sed.dat! Taking files from usr/Sed.dat now instead.')
fns = self.dphot_other.keys()
for fn in fns:
#-- Includes cwave == 0.0, in which case everything is selected
if cwave >= 0.0:
keep = self.dphot_other[fn]['wave'] >= cwave
else:
keep = self.dphot_other[fn]['wave'] < -cwave
#-- Check if anything is kept at all, otherwise move on
if not keep.any():
continue
#-- Add the model and data points to the list
for i,iphot in enumerate(self.mphot_other[fn]):
mphot[i].extend(iphot[keep])
dphot.extend(self.dphot_other[fn]['phot'][keep])
ephot.extend(self.dphot_other[fn]['ephot'][keep])
if phot_ivs and self.photbands.size:
if cwave >= 0.0:
keep = self.dphot_ivs['wave'] >= cwave
else:
keep = self.dphot_ivs['wave'] < -cwave
#-- Check if anything is kept at all, otherwise move on
if keep.any():
for i,iphot in enumerate(self.mphot_ivs):
mphot[i].extend(iphot[keep])
dphot.extend(self.dphot_ivs['phot'][keep])
ephot.extend(self.dphot_ivs['ephot'][keep])
#-- If no data are selected at all, return an empty array
if not dphot:
self.chi2 = np.empty(0)
return self.chi2
self.chi2 = []
for iphot in mphot:
self.chi2.append(BasicStats.calcChiSquared(dphot,iphot,ephot,ndf,\
chi2_method))
self.chi2 = np.array(self.chi2)
return np.sort(self.chi2) if sort else self.chi2
def calcChiSquared(self,chi2_method='diff',ndf=0,filename=None):
'''
Calculate the chi_squared value for given models and data.
If integrated fluxes are available, they are used without the line
blends. LINES FLAGGED AS A BLEND ARE EXCLUDED FROM THE CHI2 CALCULATION.
If not, the modeled, convolved spectrum is compared directly with the
data, where the model is not just 0 flux.
@keyword chi2_method: The type of chi-squared calculated for integrated
fluxes. 'diff' for standard chi^2 kind, 'log' for
redistributing the data/model ratios on an
absolute logarithmic scale before calculating the
chi2
(default: 'diff')
@type chi2_method: string
@keyword ndf: Number of degrees of freedom. Default in case of calculating
for one single model. Typically the number of variable grid
parameters in a grid calculation.
(default: 0)
@type ndf: int
@keyword filename: the filename for which you want to return the list,
if None, all filenames are used and the lists are
merged into one
(default: None)
@type filename: string
'''
if chi2_method not in ['diff','log']:
chi2_method = 'diff'
print "WARNING: STAT_CHI2 not recognized. Using default 'diff'."
#-- For now excluding blends
inst = self.instrument
all_dints = self.getRatios(sel_type='dint_bands',data_type='dint_bands',\
filename=filename)
all_derrs = self.getRatios(sel_type='dint_bands',data_type='derr_bands',\
filename=filename)
for istar,star in enumerate(self.star_grid):
this_id = star['LAST_%s_MODEL'%inst.instrument.upper()]
#-- Get all integrated fluxes and errors for this particular model,
# no matter the filename of the data (ie no matter the band).
# If no integrated flux available, set chi2_inttot[this_id] to 0
all_mints = self.getRatios(this_id=this_id,sel_type='dint_bands',\
data_type='mint_bands',filename=filename)
sel = np.isfinite(all_mints)*np.isfinite(all_dints)
if not all_mints[sel].size:
self.chi2_inttot[this_id] = 0
else:
chi2 = bs.calcChiSquared(data=all_dints,model=all_mints,\
noise=all_derrs*all_dints,\
mode=chi2_method,ndf=ndf)
self.chi2_inttot[this_id] = chi2
#-- Calculate chi2 based on the convolved model with respect to the
# continuum-subtracted data.
all_dflux = []
all_mflux = []
all_dstd = []
if not filename:
fns = inst.data_filenames
fluxes = inst.data_flux_list
else:
fns = filename is None and inst.data_filenames or [filename]
fluxes = [inst.data_flux_list[inst.data_filenames.index(fn)]
for fn in fns]
for fn,dflux in zip(fns,fluxes):
dstd = self.data_stats[fn]['std']
#-- Cannot return empty list as the selection of existing
# convolutions is done in Statistics.setModels()
mflux = inst.getSphinxConvolution(star,fn)[1]
#-- Make sure all points are positive. Some data points may be
# <0 due to continuum subtraction.
mflux = mflux[dflux>0]
dflux = dflux[dflux>0]
all_dflux.extend(dflux[mflux>0])
all_mflux.extend(mflux[mflux>0])
all_dstd.extend([dstd]*len(mflux[mflux>0]))
self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,\
all_mflux,all_dstd,\
mode=chi2_method,ndf=ndf)
def __setIntRatios(self, ifn, fn, chi2_type='normal'):
'''
Calculate ratios of integrated intensities, if requested.
Assumes there is a linefit filename available in the Instrument()
object. Only done for those line present in this file, based on
Doppler shifted wavelength.
@param ifn: Index of the data band in self.instrument lists.
@type ifn: int
@param fn: The filename of the data set. Needed for book keeping.
@type fn: string
@keyword chi2_type: The type of chi-squared calculated for integrated
fluxes. 'normal' for the usual kind, 'log' for chi2
of the log of the integrated fluxes and noise.
(default: normal)
@type chi2_type: string
'''
#-- Get some data properties, and extract data wavelength and flux
inst = self.instrument
self.int_ratios[fn] = dict()
self.int_ratios_err[fn] = dict()
self.chi2_intsi[fn] = dict()
#-- Comparing integrated intensities between PACS and models.
# Comparisons only made per filename!
inst.intIntMatch(trans_list=self.sample_trans[fn], ifn=ifn)
for star in self.star_grid:
#-- From here on, we start extracting the model specific int ints.
this_id = star['LAST_%s_MODEL' % inst.instrument.upper()]
mtrans = array(
[star.getTransition(t) for t in self.sample_trans[fn]])
these_ratios = []
these_errs = []
self.chi2_intsi[fn][this_id] = []
for mt, st in zip(mtrans, self.sample_trans[fn]):
# 4) No trans == sample_trans found for this model, or sample
# trans does not contain a PACS integrated intensity.
if mt is None or st.getIntIntUnresolved(fn)[0] is None or \
st.getIntIntUnresolved(fn)[0] == 'inblend':
these_ratios.append(None)
these_errs.append(None)
# 5) Match found with a wave_fit value. Get int ratio
# m/d. If dintint is negative, it is a blend due to large
# FWHM! If blends is not None, multiple sample trans have
# been found in the wavelength resolution bin of the
# fitted line and also indicates a blend.
else:
dintint, dintinterr, blends = st.getIntIntUnresolved(fn)
if blends is None:
mintint = mt.getIntIntIntSphinx()
else:
#-- blends is a list of sample transitions that refers
# to the transitions involved in the blend, so get
# these from the model grid, add them up and make
# sure the ratio will be negative to indicate a blend
blendlines = [
star.getTransition(t) for t in blends
if star.getTransition(t) <> None
]
mintint = sum(
[t.getIntIntIntSphinx() for t in blendlines])
dintint = -1. * abs(dintint)
if dintint > 0 and not mt.sphinx.nans_present:
if chi2_type == 'log':
ichi2 = bs.calcChiSquared(log10(dintint),\
log10(mintint),\
log10(dintint*dintinterr))
else:
ichi2 = bs.calcChiSquared(dintint,\
mintint,\
dintint*dintinterr)
#ichi2 = bs.calcLoglikelihood(dintint,\
# mintint,\
# dintint*dintinterr)
self.chi2_intsi[fn][this_id].append(ichi2)
this_ratio = mintint / dintint
these_ratios.append(this_ratio)
these_errs.append(abs(this_ratio) * dintinterr)
self.int_ratios[fn][this_id] = these_ratios
self.int_ratios_err[fn][this_id] = these_errs
def calcChiSquared(self, chi2_method='diff', ndf=0, filename=None):
'''
Calculate the chi_squared value for given models and data.
If integrated fluxes are available, they are used without the line
blends. LINES FLAGGED AS A BLEND ARE EXCLUDED FROM THE CHI2 CALCULATION.
If not, the modeled, convolved spectrum is compared directly with the
data, where the model is not just 0 flux.
@keyword chi2_method: The type of chi-squared calculated for integrated
fluxes. 'diff' for standard chi^2 kind, 'log' for
redistributing the data/model ratios on an
absolute logarithmic scale before calculating the
chi2
(default: 'diff')
@type chi2_method: string
@keyword ndf: Number of degrees of freedom. Default in case of calculating
for one single model. Typically the number of variable grid
parameters in a grid calculation.
(default: 0)
@type ndf: int
@keyword filename: the filename for which you want to return the list,
if None, all filenames are used and the lists are
merged into one
(default: None)
@type filename: string
'''
if chi2_method not in ['diff', 'log']:
chi2_method = 'diff'
print "WARNING: STAT_CHI2 not recognized. Using default 'diff'."
#-- For now excluding blends
inst = self.instrument
all_dints = self.getRatios(sel_type='dint_bands',data_type='dint_bands',\
filename=filename)
all_derrs = self.getRatios(sel_type='dint_bands',data_type='derr_bands',\
filename=filename)
for istar, star in enumerate(self.star_grid):
this_id = star['LAST_%s_MODEL' % inst.instrument.upper()]
#-- Get all integrated fluxes and errors for this particular model,
# no matter the filename of the data (ie no matter the band).
# If no integrated flux available, set chi2_inttot[this_id] to 0
all_mints = self.getRatios(this_id=this_id,sel_type='dint_bands',\
data_type='mint_bands',filename=filename)
sel = np.isfinite(all_mints) * np.isfinite(all_dints)
if not all_mints[sel].size:
self.chi2_inttot[this_id] = 0
else:
chi2 = bs.calcChiSquared(data=all_dints,model=all_mints,\
noise=all_derrs*all_dints,\
mode=chi2_method,ndf=ndf)
self.chi2_inttot[this_id] = chi2
#-- Calculate chi2 based on the convolved model with respect to the
# continuum-subtracted data.
all_dflux = []
all_mflux = []
all_dstd = []
if not filename:
fns = inst.data_filenames
fluxes = inst.data_flux_list
else:
fns = filename is None and inst.data_filenames or [filename]
fluxes = [
inst.data_flux_list[inst.data_filenames.index(fn)]
for fn in fns
]
for fn, dflux in zip(fns, fluxes):
dstd = self.data_stats[fn]['std']
#-- Cannot return empty list as the selection of existing
# convolutions is done in Statistics.setModels()
mflux = inst.getSphinxConvolution(star, fn)[1]
#-- Make sure all points are positive. Some data points may be
# <0 due to continuum subtraction.
mflux = mflux[dflux > 0]
dflux = dflux[dflux > 0]
all_dflux.extend(dflux[mflux > 0])
all_mflux.extend(mflux[mflux > 0])
all_dstd.extend([dstd] * len(mflux[mflux > 0]))
self.chi2_con[this_id] = bs.calcChiSquared(all_dflux,\
all_mflux,all_dstd,\
mode=chi2_method,ndf=ndf)