def selectRandomSampleOfTimeDelays( nSamples, massCut=0.):
'''
From a given pdf randomly select some time delays
'''
#Real world inerpolator so it neeeds a mass cut of >massCut
#arbitatiry large upper bound to make sure all are included (i.e 13)
realWorldInterpolator = \
hubbleModel.hubbleInterpolator( )
realWorldInterpolator.getTrainingData('pickles/trainingDataWithMass.pkl')
realWorldInterpolator.getTimeDelayModel()
interpolateToTheseTimes= np.linspace(-3, 3, 1e6)
theta = {'H0':0.7, 'OmegaM':0.3, 'OmegaK':0., \
'OmegaL':0.7, 'zLens':0.56, 'densityProfile':-1.75, 'totalMass':11.3}
interpolatedCumSum = \
realWorldInterpolator.predictCDF( interpolateToTheseTimes, \
theta )
interpolatedProb = undoCumSum( interpolatedCumSum)
interpolatedProb[interpolatedProb<0] = 0
interpolatedProb /= np.sum(interpolatedProb)
#Interpolate the p
logTimeDelays = \
np.random.choice(interpolateToTheseTimes, \
p=interpolatedProb, size=np.int(nSamples))
bins = np.linspace(-1,4,1000)
y, x = np.histogram(logTimeDelays, \
bins=bins, density=True)
dX = (x[1] - x[0])
xcentres = (x[1:] + x[:-1])/2.
error = np.sqrt(y*nSamples)/nSamples
cumsumY = np.cumsum( y ) / np.sum(y)
cumsumYError = np.sqrt(np.cumsum(error**2)/(np.arange(len(error))+1))
xcentres += dX/2.
hubbleInterpolaterClass = hubbleModel.hubbleInterpolator( massCut=[0., massCut] )
hubbleInterpolaterClass.getTrainingData('pickles/trainingDataMassCutLower_%0.2f.pkl' % massCut )
hubbleInterpolaterClass.getTimeDelayModel()
pred = hubbleInterpolaterClass.predictCDF(interpolateToTheseTimes, theta)
plt.plot(interpolateToTheseTimes, pred, 'b')
plt.plot(interpolateToTheseTimes, interpolatedCumSum, 'r')
plt.show()
return {'x':xcentres, 'y':cumsumY, 'error':cumsumYError}
def getPredictedConstraints(nIterations=10, nSampleSizes=11):
hubbleInterpolaterClass = \
hubbleModel.hubbleInterpolator( )
hubbleInterpolaterClass.getTrainingData('pickles/trainingDataWithMass.pkl')
hubbleInterpolaterClass.getTimeDelayModel()
sampleSizes = 10**np.linspace(2, 4, nSampleSizes)
estimates = np.zeros(nSampleSizes)
#Loop through each sample size
for i, iSampleSize in enumerate([100]):
print("Sample Size: %i" % (iSampleSize))
samples = \
getMCMCchainForSamplesSize(iSampleSize, nIterations, \
hubbleInterpolaterClass)
median, error = getModeAndError(samples[:, 0])
estimates[i] = error
return sampleSizes, estimates
def plotAllDefaultCosmologies():
hubbleInterpolator = hubbleModel.hubbleInterpolator( regressorNoiseLevel=1e0)
hubbleInterpolator.getTrainingData('exactPDFpickles/noCosmologyWithMass.pkl')
hubbleInterpolator.getTimeDelayModel()
for i in np.arange(hubbleInterpolator.pdfArray.shape[0]):
plt.plot(hubbleInterpolator.timeDelays, \
hubbleInterpolator.pdfArray[i,:], color='grey', alpha=0.5)
inputParams = cp(hubbleInterpolator.fiducialCosmology)
inputParams['H0'] /= 100.
inputParams['zLens'] = hubbleInterpolator.features['zLens'][i]
inputParams['densityProfile'] = \
hubbleInterpolator.features['densityProfile'][i]
inputParams['totalMass'] = \
hubbleInterpolator.features['totalMass'][i]
predict = \
hubbleInterpolator.predictCDF(hubbleInterpolator.timeDelays, inputParams)
plt.plot(hubbleInterpolator.timeDelays, predict, color='red', alpha=0.5)
pklFileName = \
'../output/CDM/selectionFunction/'+\
'allHalosFiducialCosmology.pkl'
finalMergedPDFdict = pkl.load(open(pklFileName,'rb'))
cumsum = np.cumsum(finalMergedPDFdict['y'])/np.sum(finalMergedPDFdict['y'])
plt.plot(finalMergedPDFdict['x'],cumsum, 'b')
plt.plot(hubbleInterpolator.timeDelays,\
np.mean(hubbleInterpolator.pdfArray, axis=0), 'g')
plt.show()
def getPredictedConstraints(nIterations = 2,nSampleSizes = 3, massCut=0.):
#this is what i will fit with, so this will training on things less than
#the mass cut
hubbleInterpolaterClass = hubbleModel.hubbleInterpolator( massCut=[0., massCut] )
hubbleInterpolaterClass.getTrainingData('pickles/trainingDataMassCutLower_%0.2f.pkl' % massCut )
hubbleInterpolaterClass.getTimeDelayModel()
sampleSizes = 10**np.linspace(2,4,nSampleSizes)
estimates = np.zeros(nSampleSizes)
#Loop through each sample size
for i, iSampleSize in enumerate([10000]):
print("Sample Size: %i" % (iSampleSize))
samples = \
getMCMCchainForSamplesSize(iSampleSize, nIterations, \
hubbleInterpolaterClass,\
massCut=massCut)
median, error = getModeAndError( samples[:,0])
estimates[i] = error
return sampleSizes, estimates
def selectRandomSampleOfTimeDelays(nSamples):
'''
From a given pdf randomly select some time delays
'''
#Real world inerpolator
realWorldInterpolator = \
hubbleModel.hubbleInterpolator( )
realWorldInterpolator.getTrainingData(
'pickles/trainingDataForObsWithMass.pkl')
realWorldInterpolator.getTimeDelayModel()
interpolateToTheseTimes = np.linspace(-3, 3, 1e6)
inputParams = {'H0':0.7, 'OmegaM':0.3, 'OmegaK':0., 'OmegaL':0.7, \
'zLens':0.4, 'densityProfile':-1.75, 'totalMass':11.1}
interpolatedCumSumA = \
realWorldInterpolator.predictCDF( interpolateToTheseTimes, inputParams )
interpolatedProbA = undoCumSum(interpolatedCumSumA)
inputParams = {'H0':0.7, 'OmegaM':0.3, 'OmegaK':0., 'OmegaL':0.7, \
'zLens':0.4, 'densityProfile':-1.75, 'totalMass':11.25}
interpolatedCumSumB = \
realWorldInterpolator.predictCDF( interpolateToTheseTimes, inputParams )
interpolatedProbB = undoCumSum(interpolatedCumSumB)
interpolatedProb = interpolatedProbA + interpolatedProbB
interpolatedProb[interpolatedProb < 0] = 0
interpolatedProb /= np.sum(interpolatedProb)
#Interpolate the p
logTimeDelays = \
np.random.choice(interpolateToTheseTimes, \
p=interpolatedProb, size=np.int(nSamples))
bins = np.linspace(-1, 4, 1000)
y, x = np.histogram(logTimeDelays, \
bins=bins, density=True)
dX = (x[1] - x[0])
xcentres = (x[1:] + x[:-1]) / 2.
error = np.sqrt(y * nSamples) / nSamples
cumsumY = np.cumsum(y) / np.sum(y)
cumsumYError = np.sqrt(np.cumsum(error**2) / (np.arange(len(error)) + 1))
xcentres += dX / 2.
return {'x': xcentres, 'y': cumsumY, 'error': cumsumYError}
def getBestParsForGPR():
hubbleInterpolator = \
hubbleModel.hubbleInterpolator( )
hubbleInterpolator.getTrainingData('pickles/trainingDataWithMass.pkl')
hubbleInterpolator.getTimeDelayModel()
defaultLog = hubbleInterpolator.getGaussProcessLogLike()
print(defaultLog)
'''
length_scale = 10**np.linspace(0.,4, 5)
nuList = np.linspace(0.1, 2., 5)
noise_level = 10**np.linspace(0.,4, 5)
alphaList = 10**np.linspace(-4.,2, 6)
logLike = []
params = []
for iL in length_scale:
for nu in nuList:
for noise in noise_level:
for alpha in alphaList:
'''
nLength=10
nWhite = 100
maternWeightList = [1.5] #10**np.linspace(-1,3,nLength)
whiteKernelList = 10**np.linspace(-5,0,nWhite)
logLike = np.zeros([nLength,nWhite])
for i, maternWeight in enumerate(maternWeightList):
for j, whiteKernel in enumerate(whiteKernelList):
hubbleInterpolator.regressorNoiseLevel= whiteKernel
hubbleInterpolator.learnPrincipalComponents( length_scale=maternWeight)
logLike[ i, j] = hubbleInterpolator.getGaussProcessLogLike()
#if logLike[ i, j] > defaultLog:
# print(maternWeight,whiteKernel, logLike[ i, j] - defaultLog)
#pdb.set_trace()
#plt.imshow(logLike, extent=[-5, 0, -1, 3], origin='lower')
plt.figure(figsize=(8,6))
gs = gridspec.GridSpec(2,1)
ax = plt.subplot(gs[0,0])
ax.set_xlabel(r'log(Noise Level)')
ax.set_ylabel(r'log likelihood')
#plt.ylabel(r'log(Matern Length Scale)')
print(whiteKernelList[np.argmax(logLike[0,:])])
ax.plot(whiteKernelList, logLike[0,:], '-')
ax.set_xscale('log')
plt.savefig('../plots/GPRparams.pdf')
plt.show()
pdb.set_trace()
def posteriorGradientForMassAndHubble(nPoints=10):
'''
What is the gradient of the posterior dH0/dlogM?
'''
hubbleInterpolaterClass = \
hubbleModel.hubbleInterpolator( )
hubbleInterpolaterClass.getTrainingData(
'pickles/trainingDataForObsWithMass.pkl')
hubbleInterpolaterClass.getTimeDelayModel()
data = getObservations()
posteriorPoints = [list(np.linspace(0.65, 0.75, nPoints))]
for iParam in hubbleInterpolaterClass.features.dtype.names:
minParam = np.min(hubbleInterpolaterClass.features[iParam])
maxParam = np.max(hubbleInterpolaterClass.features[iParam])
posteriorPoints.append(list(np.linspace(minParam, maxParam, nPoints)))
nParams = len(posteriorPoints)
restOfCosmology = [0.3, 0.7, 0.]
prob = np.array([])
for progress, iParamCombination in enumerate(product(*posteriorPoints)):
allParams = list(iParamCombination) + restOfCosmology
prob = np.append(
prob,
getPosterior(allParams, data['x'], data['y'], None,
hubbleInterpolaterClass))
probArrayShape = tuple(np.array([0 for i in range(nParams)]) + nPoints)
probArray = prob.reshape(probArrayShape)
probArray[np.isfinite(probArray) == False] = np.nan
probArrayJustH0Mass = np.nansum(np.nansum(probArray, axis=1), axis=1)
probArrayJustH0Mass /= np.max(probArrayJustH0Mass)
extent = [0.65,0.75,np.min(hubbleInterpolaterClass.features['totalMass']),\
np.max(hubbleInterpolaterClass.features['totalMass'])]
aspect = (extent[1] - extent[0]) / (extent[3] - extent[2])
plt.imshow(probArrayJustH0Mass,
origin='lower',
extent=extent,
aspect=aspect)
plt.show()
pdb.set_trace()
def getPredictedConstraints(nIterations = 100,\
nSampleSizes = 16,\
minimumTimeDelay=0.):
hubbleInterpolaterClass = \
hubbleModel.hubbleInterpolator(minimumTimeDelay=minimumTimeDelay )
if minimumTimeDelay > 0:
hubbleInterpolaterClass.getTrainingData(pklFile='picklesMinimumDelay/trainingDataWithMass.pkl')
else:
hubbleInterpolaterClass.getTrainingData('pickles/trainingDataWithMass.pkl')
hubbleInterpolaterClass.getTimeDelayModel()
sampleSizes = 10**np.linspace(1,4,nSampleSizes)[4:-1]
estimates = None
#Loop through each sample size
for i, iSampleSize in enumerate(sampleSizes):
print("Sample Size: %i" % (iSampleSize))
samples = \
getMCMCchainForSamplesSize(iSampleSize, nIterations, \
hubbleInterpolaterClass,\
minimumTimeDelay=minimumTimeDelay)
if estimates is None:
estimates = np.zeros((samples.shape[1], 2, len(sampleSizes)))
for iPar in range(samples.shape[1]):
median, error = getModeAndError( samples[:,iPar])
estimates[iPar, 0, i] = median
estimates[iPar, 1, i] = error
return sampleSizes, estimates
def getDiffArraysForNumPrincpalComponents( nComponents, nSamples=100 ):
'''
New method, test the GPR for the default cosmology
and then lineraly interpolate this to a new cosmology
'''
#hubble interpolator over a small number o
hubbleInterpolator = \
hubbleModel.hubbleInterpolator( nPrincipalComponents=nComponents )
hubbleInterpolator.getTrainingData('exactPDFpickles/trainingDataWithMass.pkl')
hubbleInterpolator.getTimeDelayModel()
allDistributions = \
pkl.load(open(hubbleInterpolator.allDistributionsPklFile,'rb'))
#set up an array
#This one is truth - predictedCDF
diffArray = np.zeros((nSamples, len(hubbleInterpolator.timeDelays)))
#this one is truth - true pca with shifted cosmology
diffPCA= np.zeros((nSamples, len(hubbleInterpolator.timeDelays)))
#Cosmology labels
cosmoKeys = hubbleInterpolator.fiducialCosmology.keys()
doneInts = []
for i in np.arange(nSamples):
#Cant do all of them so randomly select one
randInt = np.random.randint(0, len(allDistributions))
#Makes sure i dont re-do one
if randInt in doneInts:
continue
doneInts.append(randInt)
#Get the raw distriubtion
iDist = allDistributions[randInt]
#and determine the cdf
truth = iDist['y'][ iDist['x'] > \
hubbleInterpolator.logMinimumTimeDelay]
truthCumSum = np.cumsum(truth)/np.sum(truth)
#Get the params of this distribution
params = iDist['cosmology']
fileName = iDist['fileNames'][0]
zLensStr = fileName.split('/')[-2]
zLens = np.float(zLensStr.split('_')[1])
densityProfile = \
getDensity.getDensityProfileIndex(fileName)[0]
totalMassForHalo = getMass.getTotalMass( fileName )
defaultDistributionIndex = \
( hubbleInterpolator.features['densityProfile'] == densityProfile ) &\
( hubbleInterpolator.features['zLens'] == zLens ) & \
( hubbleInterpolator.features['totalMass'] == totalMassForHalo )
params['zLens'] = zLens
params['densityProfile'] = densityProfile
params['totalMass'] = totalMassForHalo
params['H0'] /= 100.
#and get the principal components that describe this
truePrincpalComponents = \
hubbleInterpolator.principalComponents[defaultDistributionIndex,:]
#and then the distriubtion described by the PCA in the default cosmology
pcaCDFinDefaultCosmology = hubbleInterpolator.pca.inverse_transform( truePrincpalComponents[0] )
#Get the cosmological interpolated shift and shift it
interpolateThisCosmology = \
np.array([ iDist['cosmology'][i] for i in \
hubbleInterpolator.fiducialCosmology.keys()])
cosmoShift = \
hubbleInterpolator.interpolatorFunction.predict(interpolateThisCosmology.reshape(1,-1))
spline = CubicSpline( hubbleInterpolator.timeDelays+cosmoShift, pcaCDFinDefaultCosmology)
#So this is the PDF of the true components, interpolated to the new cosmology
pcaPDFinShiftedCosmology = spline( hubbleInterpolator.timeDelays)
diffPCA[i,:] = pcaPDFinShiftedCosmology - truthCumSum
#This is the predicted CDF from GPR interpolated to the new cosmology
predictCDF = hubbleInterpolator.predictCDF( hubbleInterpolator.timeDelays, params)
diffArray[i,:] = predictCDF - truthCumSum
return {'x': hubbleInterpolator.timeDelays, 'diffPredict': diffArray, 'diffPCA':diffPCA}
def selectRandomSampleOfTimeDelays( nSamples, minimumTimeDelay=0.):
'''
From a given pdf randomly select some time delays
'''
#Real world inerpolator
dataSelectionFunction = '../output/CDM/selectionFunction/SF_data.pkl'
realWorldInterpolator = \
hubbleModel.hubbleInterpolator()
#realWorldInterpolator = \
# hubbleModel.hubbleInterpolator( )
realWorldInterpolator.getTrainingData('pickles/trainingDataWithMass.pkl')
#realWorldInterpolator.getTrainingData('pickles/trainingDataWithMassForDataTrainingSet.pkl')
realWorldInterpolator.getTimeDelayModel()
if minimumTimeDelay == 0:
minimumTimeDelay = 1e-3
logMinimumTimeDelay = np.log10(minimumTimeDelay)
interpolateToTheseTimes= np.linspace(-3, 3, 1e6)
inputParams = {'H0':0.7, 'OmegaM':0.3, 'OmegaK':0., 'OmegaL':0.7, \
'zLens':0.40, 'densityProfile':-1.75, 'totalMass':11.05}
interpolatedCumSum = \
realWorldInterpolator.predictCDF( interpolateToTheseTimes, inputParams )
interpolatedProb = undoCumSum( interpolatedCumSum)
interpolatedProb[interpolatedProb<0] = 0
interpolatedProb /= np.sum(interpolatedProb)
#Interpolate the p
logTimeDelays = \
np.random.choice(interpolateToTheseTimes, \
p=interpolatedProb, size=np.int(nSamples))
logTimeDelays = logTimeDelays[ logTimeDelays > logMinimumTimeDelay ]
bins = np.linspace(-1,4,1000)
y, x = np.histogram(logTimeDelays, bins=bins, density=True)
dX = (x[1] - x[0])
xcentres = (x[1:] + x[:-1])/2.
error = np.sqrt(y*nSamples)/nSamples
cumsumY = np.cumsum( y ) / np.sum(y)
cumsumYError = np.sqrt(np.cumsum(error**2)/(np.arange(len(error))+1))
xcentres += dX/2.
return {'x':xcentres, 'y':cumsumY, 'error':cumsumYError}