def compute_model(options):
import numpy
import astropy.io.fits as fits
import JLA_library as JLA
from astropy.table import Table
from astropy.cosmology import FlatwCDM
from scipy.interpolate import interp1d
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
# ----------- Read in the SN ordering ------------------------
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '').replace('_smp', '')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
print 'There are %d SNe' % (nSNe)
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
redshift = SNe['zcmb']
replace=(redshift < 0)
# For SNe that do not have the CMB redshift
redshift[replace]=SNe[replace]['zhel']
print len(redshift)
if options.raw:
# Data from the bottom left hand figure of Mosher et al. 2014.
# This is option ii) that is descibed above
offsets=Table.read(JLA.get_full_path(params['modelOffset']),format='ascii.csv')
Delta_M=interp1d(offsets['z'], offsets['offset'], kind='linear',bounds_error=False,fill_value='extrapolate')(redshift)
else:
Om_0=0.303 # JLA value in the wCDM model
cosmo1 = FlatwCDM(name='SNLS3+WMAP7', H0=70.0, Om0=Om_0, w0=-1.0)
cosmo2 = FlatwCDM(name='SNLS3+WMAP7', H0=70.0, Om0=Om_0, w0=-1.024)
Delta_M=5*numpy.log10(cosmo1.luminosity_distance(redshift)/cosmo2.luminosity_distance(redshift))
# Build the covariance matrix. Note that only magnitudes are affected
Zero=numpy.zeros(nSNe)
H=numpy.concatenate((Delta_M,Zero,Zero)).reshape(3,nSNe).ravel(order='F')
C_model=numpy.matrix(H).T * numpy.matrix(H)
date = JLA.get_date()
fits.writeto('C_model_%s.fits' % (date),numpy.array(C_model),clobber=True)
return None
def compute_Cstat(options):
"""Python program to compute C_stat
"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table
import JLA_library as JLA
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
# ----------- Read in the SN ordering ------------------------
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# ----------- Read in the data --------------------------
print 'There are %d SNe in the sample' % (nSNe)
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe=SNe[indices]
C_stat=numpy.zeros(9*nSNe*nSNe).reshape(3*nSNe,3*nSNe)
for i,SN in enumerate(SNe):
cov=numpy.zeros(9).reshape(3,3)
cov[0,0]=SN['dmb']**2.
cov[1,1]=SN['dx1']**2.
cov[2,2]=SN['dcolor']**2.
cov[0,1]=SN['cov_m_s']
cov[0,2]=SN['cov_m_c']
cov[1,2]=SN['cov_s_c']
# symmetrise
cov=cov+cov.T-numpy.diag(cov.diagonal())
C_stat[i*3:i*3+3,i*3:i*3+3]=cov
# ----------- Read in the base matrix computed using salt2_stat.cc ------------
if options.base!=None:
C_stat+=fits.getdata(options.base)
date = JLA.get_date()
fits.writeto('C_stat_%s.fits' % date,C_stat,clobber=True)
return
"-l",
"--lcfits",
dest="lcfits",
default="lightCurveFits",
help="Key in config file pointing to lightcurve fit parameters")
(options, args) = parser.parse_args()
params = JLA.build_dictionary(options.config)
lcfile = JLA.get_full_path(params[options.lcfits])
SN_data = Table.read(lcfile, format='fits')
SN_list_long = np.genfromtxt(options.SNlist, usecols=(0), dtype='S30')
SN_list = [
name.replace('lc-', '').replace('.list', '') for name in SN_list_long
]
SN_indices = JLA.reindex_SNe(SN_list, SN_data)
SN_data = SN_data[SN_indices]
velfile = JLA.get_full_path(params['velocityField'])
vel_correction = VelocityCorrection(velfile)
#z_correction = vel_correction.apply(SN_data)
C_pecvel = vel_correction.covmat_pecvel(SN_data)
date = JLA.get_date()
fits.writeto('C_pecvel_%s.fits' % date, np.array(C_pecvel), clobber=True)
def compute_rel_size(options):
import numpy
import astropy.io.fits as fits
from astropy.table import Table
import JLA_library as JLA
from astropy.cosmology import FlatwCDM
import os
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
# ---------- Read in the SNe list -------------------------
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-',
'').replace('.list', '')
# ----------- Read in the data JLA --------------------------
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
nSNe = len(SNe)
print 'There are %d SNe in this sample' % (nSNe)
# sort it to match the listing in options.SNlist
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
# ---------- Compute the Jacobian ----------------------
# The Jacobian is an m by 4 matrix, where m is the number of SNe
# The columns are ordered in terms of Om, w, alpha and beta
J = []
JLA_result = {
'Om': 0.303,
'w': -1.00,
'alpha': 0.141,
'beta': 3.102,
'M_B': -19.05
}
offset = {'Om': 0.01, 'w': 0.01, 'alpha': 0.01, 'beta': 0.01, 'M_B': 0.01}
nFit = 4
cosmo1 = FlatwCDM(name='SNLS3+WMAP7',
H0=70.0,
Om0=JLA_result['Om'],
w0=JLA_result['w'])
# Varying Om
cosmo2 = FlatwCDM(name='SNLS3+WMAP7',
H0=70.0,
Om0=JLA_result['Om'] + offset['Om'],
w0=JLA_result['w'])
J.append(5 * numpy.log10((cosmo1.luminosity_distance(SNe['zcmb']) /
cosmo2.luminosity_distance(SNe['zcmb']))[:, 0]))
# varying alpha
J.append(1.0 * offset['alpha'] * SNe['x1'][:, 0])
# varying beta
J.append(-1.0 * offset['beta'] * SNe['color'][:, 0])
# varying M_B
J.append(offset['M_B'] * numpy.ones(nSNe))
J = numpy.matrix(
numpy.concatenate((J)).reshape(nSNe, nFit, order='F') * 100.)
# Set up the covariance matrices
systematic_terms = [
'bias', 'cal', 'host', 'dust', 'model', 'nonia', 'pecvel', 'stat'
]
covmatrices = {
'bias': params['bias'],
'cal': params['cal'],
'host': params['host'],
'dust': params['dust'],
'model': params['model'],
'nonia': params['nonia'],
'pecvel': params['pecvel'],
'stat': params['stat']
}
if options.type in systematic_terms:
print "Using %s for the %s term" % (options.name, options.type)
covmatrices[options.type] = options.name
# Combine the matrices to compute the full covariance matrix, and compute its inverse
if options.all:
#read in the user provided matrix, otherwise compute it, and write it out
C = fits.getdata(JLA.get_full_path(params['all']))
else:
C = add_covar_matrices(covmatrices, params['diag'])
date = JLA.get_date()
fits.writeto('C_total_%s.fits' % (date), C, clobber=True)
Cinv = numpy.matrix(C).I
# Construct eta, a 3n vector
eta = numpy.zeros(3 * nSNe)
for i, SN in enumerate(SNe):
eta[3 * i] = SN['mb']
eta[3 * i + 1] = SN['x1']
eta[3 * i + 2] = SN['color']
# Construct A, a n x 3n matrix
A = numpy.zeros(nSNe * 3 * nSNe).reshape(nSNe, 3 * nSNe)
for i in range(nSNe):
A[i, 3 * i] = 1.0
A[i, 3 * i + 1] = JLA_result['alpha']
A[i, 3 * i + 2] = -JLA_result['beta']
# ---------- Compute W ----------------------
# W has shape m * 3n, where m is the number of fit paramaters.
W = (J.T * Cinv * J).I * J.T * Cinv * numpy.matrix(A)
# Note that (J.T * Cinv * J) is a m x m matrix, where m is the number of fit parameters
# ----------- Compute V_x, where x represents the systematic uncertainty
result = []
for term in systematic_terms:
cov = numpy.matrix(fits.getdata(JLA.get_full_path(covmatrices[term])))
if 'C_stat' in covmatrices[term]:
# Add diagonal term from Eq. 13 to the magnitude
sigma = numpy.genfromtxt(
JLA.get_full_path(params['diag']),
comments='#',
usecols=(0, 1, 2),
dtype='f8,f8,f8',
names=['sigma_coh', 'sigma_lens', 'sigma_pecvel'])
for i in range(nSNe):
cov[3 * i, 3 * i] += sigma['sigma_coh'][i]**2 + sigma[
'sigma_lens'][i]**2 + sigma['sigma_pecvel'][i]**2
V = W * cov * W.T
result.append(V[0, 0])
print '%20s\t%5s\t%5s\t%s' % ('Term', 'sigma', 'var', 'Percentage')
for i, term in enumerate(systematic_terms):
if options.type != None and term == options.type:
print '* %18s\t%5.4f\t%5.4f\t%4.1f' % (term, numpy.sqrt(
result[i]), result[i], result[i] / numpy.sum(result) * 100.)
else:
print '%20s\t%5.4f\t%5.4f\t%4.1f' % (term, numpy.sqrt(
result[i]), result[i], result[i] / numpy.sum(result) * 100.)
print '%20s\t%5.4f' % ('Total', numpy.sqrt(numpy.sum(result)))
return
def compute_Ccal(options):
"""Python program to compute Ccal
"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table
import multiprocessing as mp
import matplotlib.pyplot as plt
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
try:
salt_prefix = params['saltPrefix']
except KeyError:
salt_prefix = ''
# ---------- Read in the SNe list -------------------------
SNeList = Table(numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S100',
names=['id', 'lc']))
for i,SN in enumerate(SNeList):
SNeList['id'][i]=SNeList['id'][i].replace('lc-', '').replace('.list', '').replace('_smp', '')
# ---------- Read in the SN light curve fits ------------
# This is used to get the SN redshifts which are used in smoothing the Jacbian
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# Make sure that the order is correct
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
if len(indices) != len(SNeList['id']):
print "We are missing SNe"
exit()
# ----------- Set up the structures to handle the different salt models -------
# The first model is the unperturbed salt model
SALTpath=JLA.get_full_path(params['saltPath'])
SALTmodels=JLA.SALTmodels(SALTpath+'/saltModels.list')
nSALTmodels=len(SALTmodels)-1
print SALTmodels, nSALTmodels
nSNe=len(SNeList)
print 'There are %d SNe in the sample' % (nSNe)
print 'There are %d SALT models' % (nSALTmodels)
# Add a survey column, which we use with the smoothing, and the redshift
SNeList['survey'] = numpy.zeros(nSNe,'a10')
SNeList['z'] = SNe['zhel']
# Identify the SNLS, SDSS, HST and low-z SNe. We use this when smoothing the Jacobian
# There is rather inelegant
# We still need to allow for Vanina's naming convention when doing this for the photometric sample
for i,SN in enumerate(SNeList):
if SN['id'][0:4]=='SDSS':
SNeList['survey'][i]='SDSS'
elif SN['id'][2:4] in ['D1','D2','D3','D4']:
SNeList['survey'][i]='SNLS'
elif SN['id'][0:3] in ['DES']:
SNeList['survey'][i]='DES'
elif SN['id'][0:2]=='sn':
SNeList['survey'][i]='nearby'
else:
SNeList['survey'][i]='high-z'
# ----------- Read in the calibration matrix -----------------
Cal=fits.getdata(JLA.get_full_path(params['C_kappa']))
# Multiply the ZP submatrix by 100^2, and the two ZP-offset submatrices by 100,
# because the magnitude offsets are 0.01 mag and the units of the covariance matrix are mag
size=Cal.shape[0] / 2
Cal[0:size,0:size]=Cal[0:size,0:size]*10000.
Cal[0:size,size:]*=Cal[0:size,size:]*100.
Cal[size:,0:size]=Cal[size:,0:size]*100.
# ------------- Create an area to work in -----------------------
workArea = JLA.get_full_path(options.workArea)
try:
os.mkdir(workArea)
except:
pass
# ----------- The lightcurve fitting --------------------------
firstSN=True
log=open('log.txt','w')
for i,SN in enumerate(SNeList):
J=[]
try:
os.mkdir(workArea+'/'+SN['id'])
except:
pass
#firstModel=True
print 'Examining SN #%d %s' % (i+1,SN['id'])
# Set up the number of processes
pool = mp.Pool(processes=int(options.processes))
# runSALT is the program that does the lightcurve fitting
results = [pool.apply(runSALT, args=(SALTpath,
SALTmodel,
salt_prefix,
SN['lc'],
SN['id'])) for SALTmodel in SALTmodels]
for result in results[1:]:
# The first model is the unperturbed model
dM,dX,dC=JLA.computeOffsets(results[0],result)
J.extend([dM,dX,dC])
pool.close() # This prevents to many open files
if firstSN:
J_new=numpy.array(J).reshape(nSALTmodels,3).T
firstSN=False
else:
J_new=numpy.concatenate((J_new,numpy.array(J).reshape(nSALTmodels,3).T),axis=0)
log.write('%d rows %d columns\n' % (J_new.shape[0],J_new.shape[1]))
log.close()
# Compute the new covariance matrix J . Cal . J.T produces a 3 * n_SN by 3 * n_SN matrix
# J=jacobian
J_smoothed=numpy.array(J_new)*0.0
J=J_new
# We need to concatenate the different samples ...
if options.Plot:
try:
os.mkdir('figures')
except:
pass
nPoints={'SNLS':11,'SDSS':11,'nearby':11,'high-z':11,'DES':11}
#sampleList=['nearby','DES']
sampleList=params['smoothList'].split(',')
if options.smoothed:
# We smooth the Jacobian
# We roughly follow the method descibed in the footnote of p13 of B14
for sample in sampleList:
selection=(SNeList['survey']==sample)
J_sample=J[numpy.repeat(selection,3)]
for sys in range(nSALTmodels):
# We need to convert to a numpy array
# There is probably a better way
redshifts=numpy.array([z for z in SNeList[selection]['z']])
derivatives_mag=J_sample[0::3][:,sys] # [0::3] = [0,3,6 ...] Every 3rd one
#print redshifts.shape, derivatives_mag.shape, nPoints[sample]
forPlotting_mag,res_mag=JLA.smooth(redshifts,derivatives_mag,nPoints[sample])
derivatives_x1=J_sample[1::3][:,sys]
forPlotting_x1,res_x1=JLA.smooth(redshifts,derivatives_x1,nPoints[sample])
derivatives_c=J_sample[2::3][:,sys]
forPlotting_c,res_c=JLA.smooth(redshifts,derivatives_c,nPoints[sample])
# We need to insert the new results into the smoothed Jacobian matrix in the correct place
# The Jacobian ia a 3 * n_SN by nSATLModels matrix
# The rows are ordered by the mag, stretch and colour of each SNe.
J_smoothed[numpy.repeat(selection,3),sys]=numpy.concatenate([res_mag,res_x1,res_c]).reshape(3,selection.sum()).ravel('F')
# If required, make some plots as a way of checking
if options.Plot:
print 'Creating plot for systematic %d and sample %s' % (sys, sample)
fig=plt.figure()
ax1=fig.add_subplot(311)
ax2=fig.add_subplot(312)
ax3=fig.add_subplot(313)
ax1.plot(redshifts,derivatives_mag,'bo')
ax1.plot(forPlotting_mag[0],forPlotting_mag[1],'r-')
ax1.set_ylabel('mag')
ax2.plot(redshifts,derivatives_x1,'bo')
ax2.plot(forPlotting_x1[0],forPlotting_x1[1],'r-')
ax2.set_ylabel('x1')
ax3.plot(redshifts,derivatives_c,'bo')
ax3.plot(forPlotting_c[0],forPlotting_c[1],'r-')
ax3.set_ylabel('c')
ax3.set_xlabel('z')
plt.savefig('figures/%s_sys_%d.png' % (sample,sys))
plt.close()
date=JLA.get_date()
fits.writeto('J_%s.fits' % (date) ,J,clobber=True)
fits.writeto('J_smoothed_%s.fits' % (date), J_smoothed,clobber=True)
# Some matrix arithmatic
# C_cal is a nSALTmodels by nSALTmodels matrix
# Read in a smoothed Jacobian specified in the options
if options.jacobian != None:
J_smoothed=fits.getdata(options.jacobian)
# else:
# # Replace the NaNs in your smoothed Jacobian with zero
# J_smoothed[numpy.isnan(J_smoothed)]=0
C=numpy.matrix(J_smoothed)*numpy.matrix(Cal)*numpy.matrix(J_smoothed).T
if options.output==None:
fits.writeto('C_cal_%s.fits' % (date), numpy.array(C), clobber=True)
else:
fits.writeto('%s.fits' % (options.output),numpy.array(C),clobber=True)
return
def compute_bias(options):
import numpy
import astropy.io.fits as fits
import JLA_library as JLA
from astropy.table import Table
from astropy.cosmology import FlatwCDM
from scipy.optimize import leastsq
import matplotlib.pyplot as plt
from scipy.stats import t
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
# ----------- Read in the SN ordering ------------------------
SNeList = Table(numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc']))
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '').replace('_smp','')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
print 'There are %d SNe' % (nSNe)
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe=SNe[indices]
# Add a column that records the error in the bias correction
SNe['e_bias'] = numpy.zeros(nSNe,'f8')
# Read in the bias correction (see, for example, Fig.5 in B14)
# Fit a polynomial to the data
# Determine the uncertainties
bias = numpy.genfromtxt(JLA.get_full_path(params['biasPolynomial']),
skip_header=4,
usecols=(0, 1, 2, 3),
dtype='S10,f8,f8,f8',
names=['sample', 'redshift', 'bias', 'e_bias'])
if options.plot:
fig=plt.figure()
ax=fig.add_subplot(111)
colour={'nearby':'b','SNLS':'r','SDSS':'g','DES':'k'}
for sample in numpy.unique(bias['sample']):
selection=(bias['sample']==sample)
guess=[0,0,0]
print bias[selection]
plsq=leastsq(residuals, guess, args=(bias[selection]['bias'],
bias[selection]['redshift'],
bias[selection]['e_bias'],
'poly'), full_output=1)
if plsq[4] in [1,2,3,4]:
print 'Solution for %s found' % (sample)
if options.plot:
ax.errorbar(bias[selection]['redshift'],
bias[selection]['bias'],
yerr=bias[selection]['e_bias'],
ecolor='k',
color=colour[sample],
fmt='o',
label=sample)
z=numpy.arange(numpy.min(bias[selection]['redshift']),numpy.max(bias[selection]['redshift']),0.001)
ax.plot(z,poly(z,plsq[0]),color=colour[sample])
# For each SNe, determine the uncerainty in the correction. We use the approach descibed in
# https://www.astro.rug.nl/software/kapteyn/kmpfittutorial.html
# Compute the chi-sq.
chisq=(((bias[selection]['bias']-poly(bias[selection]['redshift'],plsq[0]))/bias[selection]['e_bias'])**2.).sum()
dof=selection.sum()-len(guess)
print "Reduced chi-square value for sample %s is %5.2e" % (sample, chisq / dof)
alpha=0.315 # Confidence interval is 100 * (1-alpha)
# Compute the upper alpha/2 value for the student t distribution with dof
thresh=t.ppf((1-alpha/2.0), dof)
if options.plot and sample!='nearby':
# The following is only valid for polynomial fitting functions, and we do not compute it for the nearby sample
upper_curve=[]
lower_curve=[]
for x in z:
vect=numpy.matrix([1,x,x**2.])
offset=thresh * numpy.sqrt(chisq / dof * (vect*numpy.matrix(plsq[1])*vect.T)[0,0])
upper_curve.append(poly(x,plsq[0])+offset)
lower_curve.append(poly(x,plsq[0])-offset)
ax.plot(z,lower_curve,'--',color=colour[sample])
ax.plot(z,upper_curve,'--',color=colour[sample])
# Compute the error in the bias
# We increase the absolute value
# In other words, if the bias is negative, we subtract the error to make it even more negative
# This is to get the correct sign in the off diagonal elements
# We assume 100% correlation between SNe
for i,SN in enumerate(SNe):
if SN['zcmb'] > 0:
redshift = SN['zcmb']
else:
redshift = SN['zhel']
if JLA.survey(SN) == sample:
# For the nearby SNe, the uncertainty in the bias correction is the bias correction itself
if sample=='nearby':
SNe['e_bias'][i]=poly(redshift,plsq[0])
#print SN['name'],redshift, SNe['e_bias'][i]
else:
vect = numpy.matrix([1,redshift,redshift**2.])
if poly(redshift,plsq[0]) > 0:
sign = 1
else:
sign = -1
SNe['e_bias'][i] = sign * thresh * numpy.sqrt(chisq / dof * (vect*numpy.matrix(plsq[1])*vect.T)[0,0])
# We are getting some unrealistcally large values
date = JLA.get_date()
if options.plot:
ax.legend()
plt.savefig('C_bias_%s.png' % (date))
plt.close()
# Compute the bias matrix
#
Zero=numpy.zeros(nSNe)
H=numpy.concatenate((SNe['e_bias'],Zero,Zero)).reshape(3,nSNe).ravel(order='F')
C_bias = numpy.matrix(H).T * numpy.matrix(H)
fits.writeto('C_bias_%s.fits' % (date),C_bias,clobber=True)
return None
def compute_Ccal(options):
"""Python program to compute Ccal
"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table
import multiprocessing as mp
import matplotlib.pyplot as plt
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
try:
salt_prefix = params['saltPrefix']
except KeyError:
salt_prefix = ''
# ---------- Read in the SNe list -------------------------
SNeList = Table(
numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S100',
names=['id', 'lc']))
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-',
'').replace('.list', '')
# ---------- Read in the SN light curve fits ------------
# This is mostly used to get the redshifts of the SNe.
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# Make sure that the order is correct
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
# ----------- Set up the structures to handle the different salt models -------
SALTpath = JLA.get_full_path(params['saltPath'])
SALTmodels = JLA.SALTmodels(SALTpath + '/saltModels.list')
nSALTmodels = len(SALTmodels) - 1
#print SALTmodels, nSALTmodels
nSNe = len(SNeList)
print 'There are %d SNe in the sample' % (nSNe)
print 'There are %d SALT models' % (nSALTmodels)
# Add a survey column, which we use with the smoothing, and the redshift
SNeList['survey'] = numpy.zeros(nSNe, 'a10')
SNeList['z'] = SNe['zhel']
# Identify the SNLS, SDSS, HST and low-z SNe. We use this when smoothing the Jacobian
# There is probably a more elegant and efficient way of doing this
# We need to allow for Vanina's naming convention when doing this for the photometric sample
for i, SN in enumerate(SNeList):
if SN['id'][0:4] == 'SDSS':
SNeList['survey'][i] = 'SDSS'
elif SN['id'][2:4] in ['D1', 'D2', 'D3', 'D4']:
SNeList['survey'][i] = 'SNLS'
elif SN['id'][0:2] == 'sn':
SNeList['survey'][i] = 'nearby'
else:
SNeList['survey'][i] = 'high-z'
# ----------- Read in the calibration matrix -----------------
Cal = fits.getdata(JLA.get_full_path(params['C_kappa']))
# Multiply the ZP submatrix by 100^2, and the two ZP-offset matrices by 100,
# because the magnitude offsets are 0.01 mag and the units of the covariance matrix are mag
Cal[0:37, 0:37] = Cal[0:37, 0:37] * 10000.
#
Cal[0:37, 37:] *= Cal[0:37, 37:] * 100.
Cal[37:, 0:37] = Cal[37:, 0:37] * 100.
#print SALTpath
# ------------- Create an area to work in -----------------------
try:
os.mkdir(options.workArea)
except:
pass
# ----------- The lightcurve fitting --------------------------
firstSN = True
log = open('log.txt', 'w')
for i, SN in enumerate(SNeList):
J = []
try:
os.mkdir(options.workArea + '/' + SN['id'])
except:
pass
firstModel = True
print 'Examining SN #%d %s' % (i + 1, SN['id'])
# Set up the number of processes
pool = mp.Pool(processes=int(options.processes))
results = [
pool.apply(runSALT,
args=(SALTpath, SALTmodel, salt_prefix, SN['lc'],
SN['id'])) for SALTmodel in SALTmodels
]
for result in results[1:]:
dM, dX, dC = JLA.computeOffsets(results[0], result)
J.extend([dM, dX, dC])
pool.close() # This prevents to many open files
if firstSN:
J_new = numpy.array(J).reshape(nSALTmodels, 3).T
firstSN = False
else:
J_new = numpy.concatenate(
(J_new, numpy.array(J).reshape(nSALTmodels, 3).T), axis=0)
log.write('%d rows %d columns\n' % (J_new.shape[0], J_new.shape[1]))
log.close()
# Compute the new covariance matrix J . Cal . J.T produces a 3 * n_SN by 3 * n_SN matrix
# J=jacobian
J_smoothed = numpy.array(J_new) * 0.0
J = J_new
# We need to concatenate the different samples ...
if options.Plot:
try:
os.mkdir('figures')
except:
pass
if options.smoothed:
# We smooth the Jacobian
# We roughly follow the method descibed in the footnote of p13 of B14
# Note that HST is smoothed as well.
nPoints = {'SNLS': 11, 'SDSS': 11, 'nearby': 11, 'high-z': 11}
for sample in ['SNLS', 'SDSS', 'nearby']:
selection = (SNeList['survey'] == sample)
J_sample = J[numpy.repeat(selection, 3)]
for sys in range(nSALTmodels):
# We need to convert to a numpy array
# There is probably a better way
redshifts = numpy.array(
[z[0] for z in SNeList[selection]['z']])
derivatives_mag = J_sample[
0::3][:, sys] # [0::3] = [0,3,6 ...] Every 3rd one
#print redshifts.shape, derivatives_mag.shape, nPoints[sample]
forPlotting_mag, res_mag = JLA.smooth(redshifts,
derivatives_mag,
nPoints[sample])
derivatives_x1 = J_sample[1::3][:, sys]
forPlotting_x1, res_x1 = JLA.smooth(redshifts, derivatives_x1,
nPoints[sample])
derivatives_c = J_sample[2::3][:, sys]
forPlotting_c, res_c = JLA.smooth(redshifts, derivatives_c,
nPoints[sample])
# We need to insert the new results into the smoothed Jacobian matrix in the correct place
# The Jacobian ia a 3 * n_SN by nSATLModels matrix
# The rows are ordered by the mag, stretch and colour of each SNe.
J_smoothed[numpy.repeat(selection, 3),
sys] = numpy.concatenate(
[res_mag, res_x1,
res_c]).reshape(3, selection.sum()).ravel('F')
# If required, make some plots as a way of checking
if options.Plot:
print 'Creating plot for systematic %d and sample %s' % (
sys, sample)
fig = plt.figure()
ax1 = fig.add_subplot(311)
ax2 = fig.add_subplot(312)
ax3 = fig.add_subplot(313)
ax1.plot(redshifts, derivatives_mag, 'bo')
ax1.plot(forPlotting_mag[0], forPlotting_mag[1], 'r-')
ax2.plot(redshifts, derivatives_x1, 'bo')
ax2.plot(forPlotting_x1[0], forPlotting_x1[1], 'r-')
ax3.plot(redshifts, derivatives_c, 'bo')
ax3.plot(forPlotting_c[0], forPlotting_c[1], 'r-')
plt.savefig('figures/%s_sys_%d.png' % (sample, sys))
plt.close()
date = JLA.get_date()
fits.writeto('J_%s.fits' % (date), J, clobber=True)
fits.writeto('J_smoothed_%s.fits' % (date), J_smoothed, clobber=True)
# Some matrix arithmatic
# C_cal is a nSALTmodels by nSALTmodels matrix
# Read in a smoothed Jacobian specified in the options
if options.jacobian != None:
J_smoothed = fits.getdata(options.jacobian)
# else:
# # Replace the NaNs in your smoothed Jacobian with zero
# J_smoothed[numpy.isnan(J_smoothed)]=0
C = numpy.matrix(J_smoothed) * numpy.matrix(Cal) * numpy.matrix(
J_smoothed).T
if options.output == None:
fits.writeto('C_cal_%s.fits' % (date), numpy.array(C), clobber=True)
else:
fits.writeto('%s.fits' % (options.output),
numpy.array(C),
clobber=True)
return
parser = OptionParser()
parser.add_option("-c", "--config", dest="config", default="JLA.config",
help="Parameter file containing the location of various JLA parameters")
parser.add_option("-s", "--SNlist", dest="SNlist",
help="List of SNe")
parser.add_option("-l", "--lcfits", dest="lcfits", default="lightCurveFits",
help="Key in config file pointing to lightcurve fit parameters")
parser.add_option("-o", "--output", dest="output",default="sigma_mu.txt",
help="Output")
(options, args) = parser.parse_args()
params = JLA.build_dictionary(options.config)
lcfile = JLA.get_full_path(params[options.lcfits])
SN_data = Table.read(lcfile, format='fits')
SN_list_long = np.genfromtxt(options.SNlist, usecols=(0), dtype='S30')
SN_list = [name.replace('lc-', '').replace('.list', '').replace('_smp','') for name in SN_list_long]
SN_indices = JLA.reindex_SNe(SN_list, SN_data)
SN_data = SN_data[SN_indices]
sigma_diag = compute_diag(SN_data)
np.savetxt(options.output,sigma_diag, header='coh lens pecvel')
def compute_rel_size(options):
import numpy
import astropy.io.fits as fits
from astropy.table import Table
import JLA_library as JLA
from astropy.cosmology import FlatwCDM
import os
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
# ---------- Read in the SNe list -------------------------
SNeList=numpy.genfromtxt(options.SNlist,usecols=(0,2),dtype='S30,S200',names=['id','lc'])
for i,SN in enumerate(SNeList):
SNeList['id'][i]=SNeList['id'][i].replace('lc-','').replace('.list','')
# ----------- Read in the data JLA --------------------------
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
nSNe=len(SNe)
print 'There are %d SNe in this sample' % (nSNe)
# sort it to match the listing in options.SNlist
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe=SNe[indices]
# ---------- Compute the Jacobian ----------------------
# The Jacobian is an m by 4 matrix, where m is the number of SNe
# The columns are ordered in terms of Om, w, alpha and beta
J=[]
JLA_result={'Om':0.303,'w':-1.00,'alpha':0.141,'beta':3.102,'M_B':-19.05}
offset={'Om':0.01,'w':0.01,'alpha':0.01,'beta':0.01,'M_B':0.01}
nFit=4
cosmo1 = FlatwCDM(name='SNLS3+WMAP7', H0=70.0, Om0=JLA_result['Om'], w0=JLA_result['w'])
# Varying Om
cosmo2 = FlatwCDM(name='SNLS3+WMAP7', H0=70.0, Om0=JLA_result['Om']+offset['Om'], w0=JLA_result['w'])
J.append(5*numpy.log10((cosmo1.luminosity_distance(SNe['zcmb'])/cosmo2.luminosity_distance(SNe['zcmb']))[:,0]))
# varying alpha
J.append(1.0*offset['alpha']*SNe['x1'][:,0])
# varying beta
J.append(-1.0*offset['beta']*SNe['color'][:,0])
# varying M_B
J.append(offset['M_B']*numpy.ones(nSNe))
J = numpy.matrix(numpy.concatenate((J)).reshape(nSNe,nFit,order='F') * 100.)
# Set up the covariance matrices
systematic_terms = ['bias', 'cal', 'host', 'dust', 'model', 'nonia', 'pecvel', 'stat']
covmatrices = {'bias':params['bias'],
'cal':params['cal'],
'host':params['host'],
'dust':params['dust'],
'model':params['model'],
'nonia':params['nonia'],
'pecvel':params['pecvel'],
'stat':params['stat']}
if options.type in systematic_terms:
print "Using %s for the %s term" % (options.name,options.type)
covmatrices[options.type]=options.name
# Combine the matrices to compute the full covariance matrix, and compute its inverse
if options.all:
#read in the user provided matrix, otherwise compute it, and write it out
C=fits.getdata(JLA.get_full_path(params['all']))
else:
C=add_covar_matrices(covmatrices,params['diag'])
date=JLA.get_date()
fits.writeto('C_total_%s.fits' % (date), C, clobber=True)
Cinv=numpy.matrix(C).I
# Construct eta, a 3n vector
eta=numpy.zeros(3*nSNe)
for i,SN in enumerate(SNe):
eta[3*i]=SN['mb']
eta[3*i+1]=SN['x1']
eta[3*i+2]=SN['color']
# Construct A, a n x 3n matrix
A=numpy.zeros(nSNe*3*nSNe).reshape(nSNe,3*nSNe)
for i in range(nSNe):
A[i,3*i]=1.0
A[i,3*i+1]=JLA_result['alpha']
A[i,3*i+2]=-JLA_result['beta']
# ---------- Compute W ----------------------
# W has shape m * 3n, where m is the number of fit paramaters.
W=(J.T * Cinv * J).I * J.T* Cinv* numpy.matrix(A)
# Note that (J.T * Cinv * J) is a m x m matrix, where m is the number of fit parameters
# ----------- Compute V_x, where x represents the systematic uncertainty
result=[]
for term in systematic_terms:
cov=numpy.matrix(fits.getdata(JLA.get_full_path(covmatrices[term])))
if 'C_stat' in covmatrices[term]:
# Add diagonal term from Eq. 13 to the magnitude
sigma = numpy.genfromtxt(JLA.get_full_path(params['diag']),comments='#',usecols=(0,1,2),dtype='f8,f8,f8',names=['sigma_coh','sigma_lens','sigma_pecvel'])
for i in range(nSNe):
cov[3*i,3*i] += sigma['sigma_coh'][i] ** 2 + sigma['sigma_lens'][i] ** 2 + sigma['sigma_pecvel'][i] ** 2
V=W * cov * W.T
result.append(V[0,0])
print '%20s\t%5s\t%5s\t%s' % ('Term','sigma','var','Percentage')
for i,term in enumerate(systematic_terms):
if options.type!=None and term==options.type:
print '* %18s\t%5.4f\t%5.4f\t%4.1f' % (term,numpy.sqrt(result[i]),result[i],result[i]/numpy.sum(result)*100.)
else:
print '%20s\t%5.4f\t%5.4f\t%4.1f' % (term,numpy.sqrt(result[i]),result[i],result[i]/numpy.sum(result)*100.)
print '%20s\t%5.4f' % ('Total',numpy.sqrt(numpy.sum(result)))
return
def compute_nonIa(options):
"""Pythom program to compute the systematic unsertainty related to
the contamimation from Ibc SNe"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table, MaskedColumn, vstack
import JLA_library as JLA
# The program computes the covaraince for the spectroscopically confirmed SNe Ia only
# The prgram assumes that the JLA SNe are first in any list
# Taken from C11
# Inputs are the rates of SNe Ia and Ibc, the most likely contaminant
# Ia rate - Perett et al.
# SN Ibc rate - proportional to the star formation rate - Hopkins and Beacom
# SN Ib luminosity distribution. Li et al + bright SN Ibc Richardson
# The bright Ibc population
# d_bc = 0.25 # The offset in magnitude between the Ia and bright Ibc
# s_bc = 0.25 # The magnitude spread
# f_bright = 0.25 # The fraction of Ibc SN that are bright
# Simulate the characteristics of the SNLS survey
# Apply outlier rejection
# All SNe that pass the cuts are included in the sample
# One then has a mixture of SNe Ia and SNe Ibc
# and the average magnitude at each redshift is biased. This
# is called the raw bias. One multiplies the raw bias by the fraction of
# objects classified as SNe Ia*
# The results are presented in 7 redshift bins defined in table 14 of C11
# We use these results to generate the matrix.
# Only the SNLS SNe in the JLA sample are considered.
# For the photometrically selected sample and other surveys, this will probably be different
# JLA compute this for the SNLS sample only
# We assume that the redshift in this table refers to the left hand edge of each bin
z_bin = numpy.array([0.0, 0.1, 0.26, 0.41, 0.57, 0.72, 0.89, 1.04])
raw_bias = numpy.array(
[0.0, 0.015, 0.024, 0.024, 0.024, 0.023, 0.026, 0.025])
f_star = numpy.array([0.0, 0.00, 0.06, 0.14, 0.17, 0.24, 0.50, 0.00])
# The covaraiance between SNe Ia in the same redshift bin is fully correlated
# Otherwise, it is uncorrelated
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-',
'').replace('.list', '')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# Add a bin column and a column that specified of the covariance is non-zero
SNe['bin'] = 0
SNe['eval'] = False
# make order of data (in SNe) match SNeList
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
# Identify the SNLS SNe in the JLA sample
for i, SN in enumerate(SNe):
if SN['source'][0] == 'JLA' and SN['name'][0][2:4] in [
'D1', 'D2', 'D3', 'D4'
]:
SNe['eval'][i] = True
# Work out which redshift bin each SNe belongs to
# In numpy.digitize, the bin number starts at 1, so we subtract 1
SNe['bin'] = numpy.digitize(SNe['zhel'], z_bin) - 1
# Build the covariance matrix
C_nonIa = numpy.zeros(nSNe * 3 * nSNe * 3).reshape(nSNe * 3, nSNe * 3)
# It is only computes the covariance for the spectroscopically confirmed SNLS SNe
# We assume that covariance between redshift bins is uncorrelated
for i in range(nSNe):
bin1 = SNe['bin'][i]
for j in range(nSNe):
bin2 = SNe['bin'][j]
if SNe['eval'][j] and SNe['eval'][i] and bin1 == bin2:
C_nonIa[3 * i, 3 *
j] = (raw_bias[bin1] *
f_star[bin1]) * (raw_bias[bin2] * f_star[bin2])
date = JLA.get_date()
fits.writeto('C_nonIa_%s.fits' % date, numpy.array(C_nonIa), clobber=True)
return
def compute_nonIa(options):
"""Pythom program to compute the systematic unsertainty related to
the contamimation from Ibc SNe"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table, MaskedColumn, vstack
import JLA_library as JLA
# The program computes the covaraince for the spectroscopically confirmed SNe Ia only
# The prgram assumes that the JLA SNe are first in any list
# Taken from C11
# Inputs are the rates of SNe Ia and Ibc, the most likely contaminant
# Ia rate - Perett et al.
# SN Ibc rate - proportional to the star formation rate - Hopkins and Beacom
# SN Ib luminosity distribution. Li et al + bright SN Ibc Richardson
# The bright Ibc population
# d_bc = 0.25 # The offset in magnitude between the Ia and bright Ibc
# s_bc = 0.25 # The magnitude spread
# f_bright = 0.25 # The fraction of Ibc SN that are bright
# Simulate the characteristics of the SNLS survey
# Apply outlier rejection
# All SNe that pass the cuts are included in the sample
# One then has a mixture of SNe Ia and SNe Ibc
# and the average magnitude at each redshift is biased. This
# is called the raw bias. One multiplies the raw bias by the fraction of
# objects classified as SNe Ia*
# The results are presented in 7 redshift bins defined in table 14 of C11
# We use these results to generate the matrix.
# Only the SNLS SNe in the JLA sample are considered.
# For the photometrically selected sample and other surveys, this will probably be different
# JLA compute this for the SNLS sample only
# We assume that the redshift in this table refers to the left hand edge of each bin
z_bin = numpy.array([0.0, 0.1, 0.26, 0.41, 0.57, 0.72, 0.89, 1.04])
raw_bias = numpy.array([0.0, 0.015, 0.024, 0.024, 0.024, 0.023, 0.026, 0.025])
f_star = numpy.array([0.0, 0.00, 0.06, 0.14, 0.17, 0.24, 0.50, 0.00])
# The covaraiance between SNe Ia in the same redshift bin is fully correlated
# Otherwise, it is uncorrelated
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# Add a bin column and a column that specified of the covariance is non-zero
SNe['bin'] = 0
SNe['eval'] = False
# make order of data (in SNe) match SNeList
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
# Identify the SNLS SNe in the JLA sample
for i, SN in enumerate(SNe):
if SN['source'][0] == 'JLA' and SN['name'][0][2:4] in ['D1', 'D2', 'D3', 'D4']:
SNe['eval'][i] = True
# Work out which redshift bin each SNe belongs to
# In numpy.digitize, the bin number starts at 1, so we subtract 1
SNe['bin'] = numpy.digitize(SNe['zhel'], z_bin)-1
# Build the covariance matrix
C_nonIa = numpy.zeros(nSNe*3*nSNe*3).reshape(nSNe*3, nSNe*3)
# It is only computes the covariance for the spectroscopically confirmed SNLS SNe
# We assume that covariance between redshift bins is uncorrelated
for i in range(nSNe):
bin1 = SNe['bin'][i]
for j in range(nSNe):
bin2 = SNe['bin'][j]
if SNe['eval'][j] and SNe['eval'][i] and bin1 == bin2:
C_nonIa[3*i, 3*j] = (raw_bias[bin1] * f_star[bin1])*(raw_bias[bin2] * f_star[bin2])
date = JLA.get_date()
fits.writeto('C_nonIa_%s.fits' % date, numpy.array(C_nonIa), clobber=True)
return
def compute_nonIa(options):
"""Pythom program to compute the systematic unsertainty related to
the contamimation from Ibc SNe"""
import numpy
import astropy.io.fits as fits
from astropy.table import Table, MaskedColumn, vstack
import JLA_library as JLA
# The program computes the covaraince for the spectroscopically confirmed SNe Ia only
# The prgram assumes that the JLA SNe are first in any list
# Taken from C11
# Inputs are the rates of SNe Ia and Ibc, the most likely contaminant
# Ia rate - Perett et al.
# SN Ibc rate - proportional to the star formation rate - Hopkins and Beacom
# SN Ib luminosity distribution. Li et al + bright SN Ibc Richardson
# The bright Ibc population
# d_bc = 0.25 # The offset in magnitude between the Ia and bright Ibc
# s_bc = 0.25 # The magnitude spread
# f_bright = 0.25 # The fraction of Ibc SN that are bright
# Simulate the characteristics of the SNLS survey
# Apply outlier rejection
# All SNe that pass the cuts are included in the sample
# One then has a mixture of SNe Ia and SNe Ibc
# and the average magnitude at each redshift is biased. This
# is called the raw bias. One multiplies the raw bias by the fraction of
# objects classified as SNe Ia*
# The results are presented in 7 redshift bins defined in table 14 of C11
# We use these results to generate the matrix.
# Only the SNLS SNe in the JLA sample are considered.
# For the photometrically selected sample and other surveys, this will probably be different
# JLA compute this for the SNLS sample only
# We assume that the redshift in this table refers to the left hand edge of each bin
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
data=numpy.genfromtxt(JLA.get_full_path(params['classification']),comments="#",usecols=(0,1,2),dtype=['float','float','float'],names=['redshift','raw_bias','fraction'])
z_bin=data['redshift']
raw_bias=data['raw_bias']
f_star=data['fraction']
# The covaraiance between SNe Ia in the same redshift bin is fully correlated
# Otherwise, it is uncorrelated
# ----------- Read in the configuration file ------------
params = JLA.build_dictionary(options.config)
SNeList = numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc'])
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '').replace('_smp','')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
# Add a bin column and a column that specifies if the covariance needs to be computed
SNe['bin'] = 0
SNe['eval'] = False
# make the order of data (in SNe) match SNeList
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe = SNe[indices]
nSNe = len(SNe)
# Identify the SNLS SNe in the JLA sample
# We use the source and the name to decide if we want to add corrections for non-Ia contamination
# Identify the DESS SNe in the DES sample.
for i, SN in enumerate(SNe):
try:
# If the source keyword exists
if (SN['source'] == 'JLA' or SN['source'] == 'SNLS_spec') and SN['name'][2:4] in ['D1', 'D2', 'D3', 'D4']:
SNe['eval'][i] = True
elif (SN['source']== 'SNLS_photo') and (SN['name'][2:4] in ['D1', 'D2', 'D3', 'D4'] or (SN['name'][0:2] in ['D1', 'D2', 'D3', 'D4'])):
SNe['eval'][i] = True
except:
# If the source keyword does not exist
if SN['name'][0:3]=="DES":
SNe['eval'][i] = True
print list(SNe['eval']).count(True)
# Work out which redshift bin each SNe belongs to
# In numpy.digitize, the bin number starts at 1, so we subtract 1 -- need to check...
SNe['bin'] = numpy.digitize(SNe['zhel'], z_bin)-1
# Build the covariance matrix
C_nonIa = numpy.zeros(nSNe*3*nSNe*3).reshape(nSNe*3, nSNe*3)
# It only computes the covariance for the spectroscopically confirmed SNLS SNe
# We assume that covariance between redshift bins is uncorrelated
# Within a redshift bin, we assume 100% covariance between SNe in that bin
for i in range(nSNe):
bin1 = SNe['bin'][i]
if SNe['eval'][i]:
print SNe['zhel'][i], bin1, raw_bias[bin1], f_star[bin1], i
for j in range(nSNe):
bin2 = SNe['bin'][j]
if SNe['eval'][j] and SNe['eval'][i] and bin1 == bin2:
C_nonIa[3*i, 3*j] = (raw_bias[bin1] * f_star[bin1])**2
# print SNe['bin'][:239]
# I am unable to reproduce this JLA covariance matrix
date = JLA.get_date()
fits.writeto('C_nonIa_%s.fits' % date, numpy.array(C_nonIa), clobber=True)
return
def compute_bias(options):
import numpy
import astropy.io.fits as fits
import JLA_library as JLA
from astropy.table import Table
from astropy.cosmology import FlatwCDM
from scipy.optimize import leastsq
import matplotlib.pyplot as plt
from scipy.stats import t
# ----------- Read in the configuration file ------------
params=JLA.build_dictionary(options.config)
# ----------- Read in the SN ordering ------------------------
SNeList = Table(numpy.genfromtxt(options.SNlist,
usecols=(0, 2),
dtype='S30,S200',
names=['id', 'lc']))
nSNe = len(SNeList)
for i, SN in enumerate(SNeList):
SNeList['id'][i] = SNeList['id'][i].replace('lc-', '').replace('.list', '')
lcfile = JLA.get_full_path(params[options.lcfits])
SNe = Table.read(lcfile, format='fits')
print 'There are %d SNe' % (nSNe)
indices = JLA.reindex_SNe(SNeList['id'], SNe)
SNe=SNe[indices]
# Add a column that records the error in the bias
SNe['e_bias'] = numpy.zeros(nSNe,'f8')
# Read in the points from B14 figure
# Fit a polynomial to the data
# Determine the uncertainties
bias = numpy.genfromtxt(JLA.get_full_path(params['biasPolynomial']),
skip_header=3,
usecols=(0, 1, 2, 3),
dtype='S10,f8,f8,f8',
names=['sample', 'redshift', 'bias', 'e_bias'])
if options.plot:
fig=plt.figure()
ax=fig.add_subplot(111)
colour={'nearby':'b','SNLS':'r','SDSS':'g'}
for sample in numpy.unique(bias['sample']):
selection=(bias['sample']==sample)
guess=[0,0,0]
plsq=leastsq(residuals, guess, args=(bias[selection]['bias'],
bias[selection]['redshift'],
bias[selection]['e_bias'],
'poly'), full_output=1)
if plsq[4] in [1,2,3,4]:
print 'Solution for %s found' % (sample)
if options.plot:
ax.errorbar(bias[selection]['redshift'],
bias[selection]['bias'],
yerr=bias[selection]['e_bias'],
ecolor='k',
color=colour[sample],
fmt='o',
label=sample)
z=numpy.arange(numpy.min(bias[selection]['redshift']),numpy.max(bias[selection]['redshift']),0.001)
ax.plot(z,poly(z,plsq[0]),color=colour[sample])
# For each SNe, determine the uncerainty in the correction. We use the covariance martix
# prediction bounds for the fitted curve.
# https://www.astro.rug.nl/software/kapteyn/kmpfittutorial.html
# Compute the chi-sq.
chisq=(((bias[selection]['bias']-poly(bias[selection]['redshift'],plsq[0]))/bias[selection]['e_bias'])**2.).sum()
dof=selection.sum()-len(guess)
print "Reduced chi-square value for sample %s is %5.2e" % (sample, chisq / dof)
alpha=0.315 # Confidence interval is 100 * (1-alpha)
# Compute the upper alpha/2 vallue for the student t distribution with dof
thresh=t.ppf((1-alpha/2.0), dof)
if options.plot:
# The following is only valid for polynomial fitting functions
upper_curve=[]
lower_curve=[]
for x in z:
vect=numpy.matrix([1,x,x**2.])
offset=thresh * numpy.sqrt(chisq / dof * (vect*numpy.matrix(plsq[1])*vect.T)[0,0])
upper_curve.append(poly(x,plsq[0])+offset)
lower_curve.append(poly(x,plsq[0])-offset)
ax.plot(z,lower_curve,'--',color=colour[sample])
ax.plot(z,upper_curve,'--',color=colour[sample])
# Compute the error in the bias
# We increase the absolute vlaue
# In other words, if the bias is negative, we subtract the error to make it even more negative
# We assume 100% correlation between SNe
for i,SN in enumerate(SNe):
if JLA.survey(SN) == sample:
if SN['zcmb'] > 0:
redshift = SN['zcmb']
else:
redshift = SN['zhel']
vect = numpy.matrix([1,redshift,redshift**2.])
if poly(redshift,plsq[0]) > 0:
sign = 1
else:
sign = -1
SNe['e_bias'][i] = sign * thresh * numpy.sqrt(chisq / dof * (vect*numpy.matrix(plsq[1])*vect.T)[0,0])
# We are getting some unrealistcally large values
if options.plot:
ax.legend()
plt.show()
plt.close()
# Compute the bias matrix
#
date = JLA.get_date()
Zero=numpy.zeros(nSNe)
H=numpy.concatenate((SNe['e_bias'],Zero,Zero)).reshape(3,nSNe).ravel(order='F')
C_bias = numpy.matrix(H)
fits.writeto('C_bias_%s.fits' % (date),C_bias.T*C_bias,clobber=True)
return None