def _buildModel(self): # Lower and upper parallax limits are fixed @deterministic(plot=False) def minParallax(): """Lower limit on true parallax values [mas].""" return self.lowParallax @deterministic(plot=False) def maxParallax(): """Upper limit on true parallax values [mas].""" return self.upParallax # Calculate initial guesses for the true parallaxes and absolute magnitudes. clippedObservedParallaxes=self.simulatedSurvey.observedParallaxes.clip(minParallax.value, maxParallax.value) initialAbsMagGuesses = (self.simulatedSurvey.observedMagnitudes + 5.0*np.log10(clippedObservedParallaxes) - 10.0) meanAbsoluteMagnitudeGuess=initialAbsMagGuesses.mean() # Prior on mean absolute magnitude expMagInit=(self.maxMeanAbsoluteMagnitude-self.minMeanAbsoluteMagnitude)/2.0 meanAbsoluteMagnitude = Uniform('meanAbsoluteMagnitude', lower=self.minMeanAbsoluteMagnitude, upper=self.maxMeanAbsoluteMagnitude, value=expMagInit) # Prior on absolute magnitude variance. Use f(tau)=1/tau, i.e., f(sigma^2)=1/sigma^2 (tau=1/sigma^2). # This non-informative prior is appropriate for a scale parameter. expTauInit=(self.maxTau-self.minTau)/(np.log(self.maxTau)-np.log(self.minTau)) tauAbsoluteMagnitude = OneOverX('tauAbsoluteMagnitude', lower=self.minTau, upper=self.maxTau, value=expTauInit) # Prior on parallax. Uniform distribution of stars in space around the Sun. priorParallaxes=OneOverXFourth('priorParallaxes', lower=minParallax, upper=maxParallax, size=self.simulatedSurvey.numberOfStarsInSurvey) # Prior on absolute magnitude priorAbsoluteMagnitudes=Normal('priorAbsoluteMagnitudes', mu=meanAbsoluteMagnitude, tau=tauAbsoluteMagnitude, size=self.simulatedSurvey.numberOfStarsInSurvey) # Apparent magnitudes depend on the parallax and absolute magnitude. @deterministic(plot=False) def apparentMagnitudes(parallaxes=priorParallaxes, absoluteMagnitudes=priorAbsoluteMagnitudes): return absoluteMagnitudes-5.0*np.log10(parallaxes)+10.0 # The likelihood of the data predictedParallaxes=Normal('predictedParallaxes',mu=priorParallaxes, tau=inverseVariance(self.simulatedSurvey.parallaxErrors), value=self.simulatedSurvey.observedParallaxes, observed=True) predictedApparentMagnitudes=Normal('predictedApparentMagnitudes',mu=apparentMagnitudes, tau=inverseVariance(self.simulatedSurvey.magnitudeErrors), value=self.simulatedSurvey.observedMagnitudes, observed=True) return locals()
def runMCMCmodel(args): """ Simulate the survey data and run the MCMC luminosity calibration model. Parameters ---------- args - Command line arguments """ mcmcParams = args['mcmcString'] surveyParams = args['surveyString'] priorParams = args['priorsString'] maxIter = int(mcmcParams[0]) burnIter = int(mcmcParams[1]) thinFactor = int(mcmcParams[2]) walkerFactor = int(mcmcParams[3]) minParallax = float(surveyParams[1]) maxParallax = float(surveyParams[2]) meanAbsoluteMagnitude = float(surveyParams[3]) varianceAbsoluteMagnitude = float(surveyParams[4]) if surveyParams[5] == 'Inf': magLim = np.Inf else: magLim = float(surveyParams[5]) simulatedSurvey = U.UniformDistributionSingleLuminosity( int(surveyParams[0]), float(surveyParams[1]), float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]), surveyLimit=magLim) #simulatedSurvey.setRandomNumberSeed(53949896) simulatedSurvey.generateObservations() numberOfStarsInSurvey = simulatedSurvey.numberOfStarsInSurvey # Calculate initial guesses for the true parallaxes and absolute magnitudes of the stars. clippedObservedParallaxes = simulatedSurvey.observedParallaxes.clip( minParallax, maxParallax) initialAbsMagGuesses = simulatedSurvey.observedMagnitudes + 5.0 * np.log10( clippedObservedParallaxes) - 10.0 meanAbsoluteMagnitudeGuess = initialAbsMagGuesses.mean() # Initial guesses for hyper parameters (mean absolute magnitude and sigma^2) # # Mean absolute magnitude uniform on (meanAbsMagLow, meanAbsMagHigh) meanAbsMagLow = float(priorParams[0]) meanAbsMagHigh = float(priorParams[1]) # Variance has 1/x distribution with lower and upper limit as prior varianceLow = float(priorParams[2]) varianceHigh = float(priorParams[3]) varianceInit = (varianceHigh - varianceLow) / (np.log(varianceHigh) - np.log(varianceLow)) initialParameters = np.concatenate( (np.array([meanAbsoluteMagnitudeGuess, varianceInit]), clippedObservedParallaxes, initialAbsMagGuesses)) # Parameters for emcee ln-posterior function posteriorDict = { 'minParallax': minParallax, 'maxParallax': maxParallax, 'muLow': meanAbsMagLow, 'muHigh': meanAbsMagHigh, 'varLow': varianceLow, 'varHigh': varianceHigh } observations = np.concatenate((simulatedSurvey.observedParallaxes, simulatedSurvey.observedMagnitudes)) observationalErrors = inverseVariance( np.concatenate( (simulatedSurvey.parallaxErrors, simulatedSurvey.magnitudeErrors))) # MCMC sampler parameters ndim = 2 * numberOfStarsInSurvey + 2 nwalkers = walkerFactor * ndim # Generate initial positions for each walker initialPositions = [np.empty((ndim)) for i in xrange(nwalkers)] initialPositions[0] = initialParameters for i in xrange(nwalkers - 1): ranMeanAbsMag = np.random.rand() * (meanAbsMagHigh - meanAbsMagLow) + meanAbsMagLow ranVariance = random_oneOverX(varianceLow, varianceHigh, 1) ranParallaxes = np.zeros_like(clippedObservedParallaxes) for j in xrange(numberOfStarsInSurvey): #if (i<nwalkers/2): ranParallaxes[j] = clippedObservedParallaxes[ j] + simulatedSurvey.parallaxErrors[j] * np.random.randn() #else: # ranParallaxes[j]=random_oneOverXFourth(minParallax,maxParallax,1) ranAbsMag = np.sqrt(ranVariance) * np.random.randn( numberOfStarsInSurvey) + ranMeanAbsMag initialPositions[i + 1] = np.concatenate( (np.array([ranMeanAbsMag, ranVariance]), ranParallaxes.clip(minParallax, maxParallax), ranAbsMag)) print '** Building sampler **' sampler = emcee.EnsembleSampler( nwalkers, ndim, UniformSpaceDensityGaussianLFBookemcee, threads=4, args=[posteriorDict, observations, observationalErrors]) # burn-in print '** Burn in **' start = now() pos, prob, state = sampler.run_mcmc(initialPositions, burnIter) print '** Finished burning in **' print ' Time (s): ', now() - start print 'Median acceptance fraction: ', np.median( sampler.acceptance_fraction) print( 'Acceptance fraction IQR: {0}'.format( np.percentile(sampler.acceptance_fraction, 25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction, 75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median( correlationTimes[2:numberOfStarsInSurvey + 2]) print ' Median for magnitudes: ', np.median( correlationTimes[numberOfStarsInSurvey + 2:]) print # final chain sampler.reset() start = now() print '** Starting sampling **' sampler.run_mcmc(pos, maxIter, rstate0=state, thin=thinFactor) print '** Finished sampling **' print ' Time (s): ', now() - start print 'Median acceptance fraction: ', np.median( sampler.acceptance_fraction) print( 'Acceptance fraction IQR: {0}'.format( np.percentile(sampler.acceptance_fraction, 25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction, 75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median( correlationTimes[2:numberOfStarsInSurvey + 2]) print ' Median for magnitudes: ', np.median( correlationTimes[numberOfStarsInSurvey + 2:]) # Extract the samples of the posterior distribution chain = sampler.flatchain # Point estimates of mean Absolute Magnitude and its standard deviation. meanAbsoluteMagnitudeSamples = chain[:, 0].flatten() varAbsoluteMagnitudeSamples = chain[:, 1].flatten() estimatedAbsMag = meanAbsoluteMagnitudeSamples.mean() errorEstimatedAbsMag = meanAbsoluteMagnitudeSamples.std() estimatedVarMag = varAbsoluteMagnitudeSamples.mean() errorEstimatedVarMag = varAbsoluteMagnitudeSamples.std() print "emcee estimates" print "mu_M={:4.2f}".format(estimatedAbsMag) + " +/- {:4.2f}".format( errorEstimatedAbsMag) print "sigma^2_M={:4.2f}".format(estimatedVarMag) + " +/- {:4.2f}".format( errorEstimatedVarMag) # Plot results # MAP estimates muDensity = gaussian_kde(meanAbsoluteMagnitudeSamples) mapValueMu = scipy.optimize.fmin(lambda x: -1.0 * muDensity(x), np.median(meanAbsoluteMagnitudeSamples), maxiter=1000, ftol=0.0001) varDensity = gaussian_kde(varAbsoluteMagnitudeSamples) mapValueVar = scipy.optimize.fmin(lambda x: -1.0 * varDensity(x), np.median(varAbsoluteMagnitudeSamples), maxiter=1000, ftol=0.0001) fig = plt.figure(figsize=(12, 8.5)) fig.add_subplot(2, 2, 1) x = np.linspace(meanAbsoluteMagnitudeSamples.min(), meanAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, muDensity(x), 'k-') plt.axvline(meanAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\mu_M$") plt.ylabel("$P(\\mu_M)$") fig.add_subplot(2, 2, 2) x = np.linspace(varAbsoluteMagnitudeSamples.min(), varAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, varDensity(x), 'k-') plt.axvline(varianceAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\sigma^2_M$") plt.ylabel("$P(\\sigma^2_M)$") fig.add_subplot(2, 2, 3) plt.hexbin(meanAbsoluteMagnitudeSamples, varAbsoluteMagnitudeSamples, C=None, bins='log', cmap=cm.gray_r) plt.xlabel("$\\mu_M$") plt.ylabel("$\\sigma^2_M$") plt.figtext(0.55, 0.4, "$\\widetilde{\\mu_M}=" + "{:4.2f}".format(estimatedAbsMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedAbsMag), ha='left') plt.figtext( 0.75, 0.4, "$\\mathrm{MAP}(\\widetilde{\\mu_M})=" + "{:4.2f}".format(mapValueMu[0]) + "$") plt.figtext(0.55, 0.35, "$\\widetilde{\\sigma^2_M}=" + "{:4.2f}".format(estimatedVarMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedVarMag), ha='left') plt.figtext( 0.75, 0.35, "$\\mathrm{MAP}(\\widetilde{\\sigma^2_M})=" + "{:4.2f}".format(mapValueVar[0]) + "$") titelA = ("$N_\\mathrm{stars}" + "={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude) + "$") titelB = ("Iterations = {0}".format(maxIter) + ", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA + "\\quad\\quad " + titelB) titelA = ("$N_\\mathrm{stars}" + "={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude) + "$") titelB = ("Iterations = {0}".format(maxIter) + ", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA + "\\quad\\quad " + titelB) titelC = [] titelC.append("MCMC sampling with emcee") titelC.append("$N_\\mathrm{walkers}" + "={0}".format(nwalkers) + "$, $N_\\mathrm{dim}" + "={0}".format(ndim) + "$") plt.figtext(0.55, 0.15, titelC[0], horizontalalignment='left') plt.figtext(0.60, 0.10, titelC[1], horizontalalignment='left') priorInfo = [] priorInfo.append( "Prior on $\\mu_M$: flat $\\quad{0}".format(meanAbsMagLow) + "<\\mu_M<{0}".format(meanAbsMagHigh) + "$") priorInfo.append( "Prior on $\\sigma^2_M$: $1/\\sigma^2_M\\quad{0}".format(varianceLow) + "<\\sigma^2_M<{0}".format(varianceHigh) + "$") plt.figtext(0.55, 0.25, priorInfo[0], horizontalalignment='left') plt.figtext(0.55, 0.20, priorInfo[1], horizontalalignment='left') if (args['pdfOutput']): plt.savefig('luminosityCalibrationResultsEmcee.pdf') elif (args['pngOutput']): plt.savefig('luminosityCalibrationResultsEmcee.png') elif (args['epsOutput']): plt.savefig('luminosityCalibrationResultsEmcee.eps') else: plt.show()
def runMCMCmodel(args): """ Simulate the survey data and run the MCMC luminosity calibration model. Parameters ---------- args - Command line arguments """ mcmcParams = args['mcmcString'] surveyParams = args['surveyString'] priorParams = args['priorsString'] maxIter, burnIter, thinFactor, walkerFactor = [ int(par) for par in mcmcParams ] minParallax, maxParallax, meanAbsoluteMagnitude, varianceAbsoluteMagnitude = [ float(par) for par in surveyParams[1:5] ] meanAbsMagLow, meanAbsMagHigh, varianceShape, varianceScale = [ float(par) for par in priorParams ] if surveyParams[5] == 'Inf': magLim = np.Inf else: magLim = float(surveyParams[5]) simulatedSurvey = U.UniformDistributionSingleLuminosity( int(surveyParams[0]), float(surveyParams[1]), float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]), surveyLimit=magLim) #simulatedSurvey.setRandomNumberSeed(53949896) simulatedSurvey.generateObservations() numberOfStarsInSurvey = simulatedSurvey.numberOfStarsInSurvey # Calculate initial guesses for the true parallaxes and absolute magnitudes of the stars. clippedObservedParallaxes = simulatedSurvey.observedParallaxes.clip( minParallax, maxParallax) initialAbsMagGuesses = simulatedSurvey.observedMagnitudes + 5.0 * np.log10( clippedObservedParallaxes) - 10.0 # Initial guesses for hyper parameters (mean absolute magnitude and sigma^2) meanAbsoluteMagnitudeGuess = initialAbsMagGuesses.mean() varianceInit = varianceScale * (varianceShape - 1) initialParameters = np.concatenate( (np.array([meanAbsoluteMagnitudeGuess, varianceInit]), clippedObservedParallaxes, initialAbsMagGuesses)) # Parameters for emcee ln-posterior function posteriorDict = { 'minParallax': minParallax, 'maxParallax': maxParallax, 'muLow': meanAbsMagLow, 'muHigh': meanAbsMagHigh, 'varShape': varianceShape, 'varScale': varianceScale } observations = np.concatenate((simulatedSurvey.observedParallaxes, simulatedSurvey.observedMagnitudes)) observationalErrors = inverseVariance( np.concatenate( (simulatedSurvey.parallaxErrors, simulatedSurvey.magnitudeErrors))) # MCMC sampler parameters ndim = 2 * numberOfStarsInSurvey + 2 nwalkers = walkerFactor * ndim # Generate initial positions for each walker initialPositions = [np.empty((ndim)) for i in xrange(nwalkers)] initialPositions[0] = initialParameters for i in xrange(nwalkers - 1): ranMeanAbsMag = np.random.rand() * (meanAbsMagHigh - meanAbsMagLow) + meanAbsMagLow ranVariance = gamma.rvs(varianceShape, scale=varianceScale) ranParallaxes = np.zeros_like(clippedObservedParallaxes) ranParallaxes = clippedObservedParallaxes + simulatedSurvey.parallaxErrors * np.random.randn( numberOfStarsInSurvey) ranAbsMag = np.sqrt(ranVariance) * np.random.randn( numberOfStarsInSurvey) + ranMeanAbsMag initialPositions[i + 1] = np.concatenate( (np.array([ranMeanAbsMag, ranVariance]), ranParallaxes.clip(minParallax, maxParallax), ranAbsMag)) print '{:*^30}'.format('Building sampler') sampler = emcee.EnsembleSampler( nwalkers, ndim, UniformSpaceDensityGaussianLFemcee, threads=4, args=[posteriorDict, observations, observationalErrors]) # burn-in print '{:*^30}'.format('Burn in') start = now() pos, prob, state = sampler.run_mcmc(initialPositions, burnIter) print '{:*^30}'.format('Finished burning') printSamplingStats(now() - start, sampler, numberOfStarsInSurvey) print # final chain sampler.reset() print '{:*^30}'.format('Start sampling') start = now() sampler.run_mcmc(pos, maxIter, rstate0=state, thin=thinFactor) print '{:*^30}'.format('Finished sampling') printSamplingStats(now() - start, sampler, numberOfStarsInSurvey) # Extract the samples of the posterior distribution chain = sampler.flatchain # Point estimates of mean Absolute Magnitude and its standard deviation. meanAbsoluteMagnitudeSamples = chain[:, 0].flatten() varAbsoluteMagnitudeSamples = chain[:, 1].flatten() estimatedAbsMag = meanAbsoluteMagnitudeSamples.mean() errorEstimatedAbsMag = meanAbsoluteMagnitudeSamples.std() estimatedVarMag = varAbsoluteMagnitudeSamples.mean() errorEstimatedVarMag = varAbsoluteMagnitudeSamples.std() print "emcee estimates" print " mu_M = {:4.2f} +/- {:4.2f}".format(estimatedAbsMag, errorEstimatedAbsMag) print "sigma^2_M = {:4.2f} +/- {:4.2f}".format(estimatedVarMag, errorEstimatedVarMag) # Plot results # Kernel density estimate of posterior distributions of mu_M and sigma^2_M, also obtain maximum a # posteriori estimate for these quantitities. muDensity, mapValueMu = kdeAndMap(meanAbsoluteMagnitudeSamples) varDensity, mapValueVar = kdeAndMap(varAbsoluteMagnitudeSamples) fig = plt.figure(figsize=(12, 8.5)) fig.add_subplot(2, 2, 1) x = np.linspace(meanAbsoluteMagnitudeSamples.min(), meanAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, muDensity(x), 'k-') plt.axvline(meanAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\mu_M$") plt.ylabel("$P(\\mu_M)$") fig.add_subplot(2, 2, 2) x = np.linspace(varAbsoluteMagnitudeSamples.min(), varAbsoluteMagnitudeSamples.max(), 500) plt.plot(x, varDensity(x), 'k-') plt.axvline(varianceAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\sigma^2_M$") plt.ylabel("$P(\\sigma^2_M)$") ax = fig.add_subplot(2, 2, 3) plt.hexbin(meanAbsoluteMagnitudeSamples, varAbsoluteMagnitudeSamples, C=None, bins='log', cmap=cm.gray_r) ax.plot(meanAbsoluteMagnitude, varianceAbsoluteMagnitude, 'or', mec='r', markersize=8, scalex=False, scaley=False) plt.xlabel("$\\mu_M$") plt.ylabel("$\\sigma^2_M$") plt.figtext(0.55, 0.4, "$\\widetilde{{\\mu_M}}={:4.2f}\\pm{:4.2f}$".format( estimatedAbsMag, errorEstimatedAbsMag), ha='left') plt.figtext( 0.75, 0.4, "$\\mathrm{{MAP}}(\\widetilde{{\\mu_M}})={:4.2f}$".format( mapValueMu[0])) plt.figtext(0.55, 0.35, "$\\widetilde{{\\sigma^2_M}}={:4.2f}\\pm{:4.2f}$".format( estimatedVarMag, errorEstimatedVarMag), ha='left') plt.figtext( 0.75, 0.35, "$\\mathrm{{MAP}}(\\widetilde{{\\sigma^2_M}})={:4.2f}$".format( mapValueVar[0])) titelA = ( "$N_\\mathrm{{stars}}={}$, True values: $\\mu_M={}$, $\\sigma^2_M={}$". format(numberOfStarsInSurvey, meanAbsoluteMagnitude, varianceAbsoluteMagnitude)) titelB = ("Iterations = {}, Burn = {}, Thin = {}".format( maxIter, burnIter, thinFactor)) plt.suptitle(titelA + "\\quad\\quad " + titelB) titelC = [] titelC.append("MCMC sampling with emcee") titelC.append("$N_\\mathrm{{walkers}}={}$, $N_\\mathrm{{dim}}={}$".format( nwalkers, ndim)) plt.figtext(0.55, 0.15, titelC[0], horizontalalignment='left') plt.figtext(0.60, 0.10, titelC[1], horizontalalignment='left') priorInfo = [] priorInfo.append("Prior on $\\mu_M$: flat $\\quad{}<\\mu_M<{}$".format( meanAbsMagLow, meanAbsMagHigh)) priorInfo.append( "Prior on $\\sigma^2_M$: $\\Gamma(\\sigma^2_M | k={},\\theta={})$". format(varianceShape, varianceScale)) plt.figtext(0.55, 0.25, priorInfo[0], horizontalalignment='left') plt.figtext(0.55, 0.20, priorInfo[1], horizontalalignment='left') if (args['pdfOutput']): plt.savefig('luminosityCalibrationResultsEmcee.pdf') elif (args['pngOutput']): plt.savefig('luminosityCalibrationResultsEmcee.png') elif (args['epsOutput']): plt.savefig('luminosityCalibrationResultsEmcee.eps') else: plt.show()
def _buildModel(self): # Lower and upper parallax limits are fixed @deterministic(plot=False) def minParallax(): """Lower limit on true parallax values [mas].""" return self.lowParallax @deterministic(plot=False) def maxParallax(): """Upper limit on true parallax values [mas].""" return self.upParallax # Calculate initial guesses for the true parallaxes and absolute magnitudes. clippedObservedParallaxes = self.simulatedSurvey.observedParallaxes.clip( minParallax.value, maxParallax.value) initialAbsMagGuesses = (self.simulatedSurvey.observedMagnitudes + 5.0 * np.log10(clippedObservedParallaxes) - 10.0) meanAbsoluteMagnitudeGuess = initialAbsMagGuesses.mean() # Prior on mean absolute magnitude expMagInit = (self.maxMeanAbsoluteMagnitude - self.minMeanAbsoluteMagnitude) / 2.0 meanAbsoluteMagnitude = Uniform('meanAbsoluteMagnitude', lower=self.minMeanAbsoluteMagnitude, upper=self.maxMeanAbsoluteMagnitude, value=expMagInit) # Prior on absolute magnitude variance. Use f(tau)=1/tau, i.e., f(sigma^2)=1/sigma^2 (tau=1/sigma^2). # This non-informative prior is appropriate for a scale parameter. expTauInit = (self.maxTau - self.minTau) / (np.log(self.maxTau) - np.log(self.minTau)) tauAbsoluteMagnitude = OneOverX('tauAbsoluteMagnitude', lower=self.minTau, upper=self.maxTau, value=expTauInit) # Prior on parallax. Uniform distribution of stars in space around the Sun. priorParallaxes = OneOverXFourth( 'priorParallaxes', lower=minParallax, upper=maxParallax, size=self.simulatedSurvey.numberOfStarsInSurvey) # Prior on absolute magnitude priorAbsoluteMagnitudes = Normal( 'priorAbsoluteMagnitudes', mu=meanAbsoluteMagnitude, tau=tauAbsoluteMagnitude, size=self.simulatedSurvey.numberOfStarsInSurvey) # Apparent magnitudes depend on the parallax and absolute magnitude. @deterministic(plot=False) def apparentMagnitudes(parallaxes=priorParallaxes, absoluteMagnitudes=priorAbsoluteMagnitudes): return absoluteMagnitudes - 5.0 * np.log10(parallaxes) + 10.0 # The likelihood of the data predictedParallaxes = Normal( 'predictedParallaxes', mu=priorParallaxes, tau=inverseVariance(self.simulatedSurvey.parallaxErrors), value=self.simulatedSurvey.observedParallaxes, observed=True) predictedApparentMagnitudes = Normal( 'predictedApparentMagnitudes', mu=apparentMagnitudes, tau=inverseVariance(self.simulatedSurvey.magnitudeErrors), value=self.simulatedSurvey.observedMagnitudes, observed=True) return locals()
def runMCMCmodel(args): """ Simulate the survey data and run the MCMC luminosity calibration model. Parameters ---------- args - Command line arguments """ mcmcParams=args['mcmcString'] surveyParams=args['surveyString'] priorParams=args['priorsString'] maxIter=int(mcmcParams[0]) burnIter=int(mcmcParams[1]) thinFactor=int(mcmcParams[2]) walkerFactor=int(mcmcParams[3]) minParallax=float(surveyParams[1]) maxParallax=float(surveyParams[2]) meanAbsoluteMagnitude=float(surveyParams[3]) varianceAbsoluteMagnitude=float(surveyParams[4]) if surveyParams[5] == 'Inf': magLim = np.Inf else: magLim = float(surveyParams[5]) simulatedSurvey=U.UniformDistributionSingleLuminosity(int(surveyParams[0]), float(surveyParams[1]), float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]), surveyLimit=magLim) #simulatedSurvey.setRandomNumberSeed(53949896) simulatedSurvey.generateObservations() numberOfStarsInSurvey=simulatedSurvey.numberOfStarsInSurvey # Calculate initial guesses for the true parallaxes and absolute magnitudes of the stars. clippedObservedParallaxes=simulatedSurvey.observedParallaxes.clip(minParallax, maxParallax) initialAbsMagGuesses=simulatedSurvey.observedMagnitudes+5.0*np.log10(clippedObservedParallaxes)-10.0 meanAbsoluteMagnitudeGuess=initialAbsMagGuesses.mean() # Initial guesses for hyper parameters (mean absolute magnitude and sigma^2) # # Mean absolute magnitude uniform on (meanAbsMagLow, meanAbsMagHigh) meanAbsMagLow=float(priorParams[0]) meanAbsMagHigh=float(priorParams[1]) # Variance has 1/x distribution with lower and upper limit as prior varianceLow=float(priorParams[2]) varianceHigh=float(priorParams[3]) varianceInit=(varianceHigh-varianceLow)/(np.log(varianceHigh)-np.log(varianceLow)) initialParameters = np.concatenate((np.array([meanAbsoluteMagnitudeGuess, varianceInit]), clippedObservedParallaxes, initialAbsMagGuesses)) # Parameters for emcee ln-posterior function posteriorDict = {'minParallax':minParallax, 'maxParallax':maxParallax, 'muLow':meanAbsMagLow, 'muHigh':meanAbsMagHigh, 'varLow':varianceLow, 'varHigh':varianceHigh} observations = np.concatenate((simulatedSurvey.observedParallaxes, simulatedSurvey.observedMagnitudes)) observationalErrors=inverseVariance(np.concatenate((simulatedSurvey.parallaxErrors, simulatedSurvey.magnitudeErrors))) # MCMC sampler parameters ndim = 2*numberOfStarsInSurvey+2 nwalkers = walkerFactor*ndim # Generate initial positions for each walker initialPositions=[np.empty((ndim)) for i in xrange(nwalkers)] initialPositions[0]=initialParameters for i in xrange(nwalkers-1): ranMeanAbsMag=np.random.rand()*(meanAbsMagHigh-meanAbsMagLow)+meanAbsMagLow ranVariance=random_oneOverX(varianceLow,varianceHigh,1) ranParallaxes=np.zeros_like(clippedObservedParallaxes) for j in xrange(numberOfStarsInSurvey): #if (i<nwalkers/2): ranParallaxes[j]=clippedObservedParallaxes[j]+simulatedSurvey.parallaxErrors[j]*np.random.randn() #else: # ranParallaxes[j]=random_oneOverXFourth(minParallax,maxParallax,1) ranAbsMag=np.sqrt(ranVariance)*np.random.randn(numberOfStarsInSurvey)+ranMeanAbsMag initialPositions[i+1]=np.concatenate((np.array([ranMeanAbsMag, ranVariance]), ranParallaxes.clip(minParallax, maxParallax), ranAbsMag)) print '** Building sampler **' sampler = emcee.EnsembleSampler(nwalkers, ndim, UniformSpaceDensityGaussianLFBookemcee, threads=4, args=[posteriorDict, observations, observationalErrors]) # burn-in print '** Burn in **' start = now() pos,prob,state = sampler.run_mcmc(initialPositions, burnIter) print '** Finished burning in **' print ' Time (s): ',now()-start print 'Median acceptance fraction: ',np.median(sampler.acceptance_fraction) print ('Acceptance fraction IQR: {0}'.format(np.percentile(sampler.acceptance_fraction,25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction,75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median(correlationTimes[2:numberOfStarsInSurvey+2]) print ' Median for magnitudes: ', np.median(correlationTimes[numberOfStarsInSurvey+2:]) print # final chain sampler.reset() start = now() print '** Starting sampling **' sampler.run_mcmc(pos, maxIter, rstate0=state, thin=thinFactor) print '** Finished sampling **' print ' Time (s): ',now()-start print 'Median acceptance fraction: ',np.median(sampler.acceptance_fraction) print ('Acceptance fraction IQR: {0}'.format(np.percentile(sampler.acceptance_fraction,25)) + ' -- {0}'.format(np.percentile(sampler.acceptance_fraction,75))) correlationTimes = sampler.acor print 'Autocorrelation times: ' print ' Mean absolute magnitude: ', correlationTimes[0] print ' Variance absolute magnitude: ', correlationTimes[1] print ' Median for parallaxes: ', np.median(correlationTimes[2:numberOfStarsInSurvey+2]) print ' Median for magnitudes: ', np.median(correlationTimes[numberOfStarsInSurvey+2:]) # Extract the samples of the posterior distribution chain = sampler.flatchain # Point estimates of mean Absolute Magnitude and its standard deviation. meanAbsoluteMagnitudeSamples = chain[:,0].flatten() varAbsoluteMagnitudeSamples = chain[:,1].flatten() estimatedAbsMag=meanAbsoluteMagnitudeSamples.mean() errorEstimatedAbsMag=meanAbsoluteMagnitudeSamples.std() estimatedVarMag=varAbsoluteMagnitudeSamples.mean() errorEstimatedVarMag=varAbsoluteMagnitudeSamples.std() print "emcee estimates" print "mu_M={:4.2f}".format(estimatedAbsMag)+" +/- {:4.2f}".format(errorEstimatedAbsMag) print "sigma^2_M={:4.2f}".format(estimatedVarMag)+" +/- {:4.2f}".format(errorEstimatedVarMag) # Plot results # MAP estimates muDensity = gaussian_kde(meanAbsoluteMagnitudeSamples) mapValueMu = scipy.optimize.fmin(lambda x: -1.0*muDensity(x),np.median(meanAbsoluteMagnitudeSamples),maxiter=1000,ftol=0.0001) varDensity = gaussian_kde(varAbsoluteMagnitudeSamples) mapValueVar = scipy.optimize.fmin(lambda x: -1.0*varDensity(x),np.median(varAbsoluteMagnitudeSamples),maxiter=1000,ftol=0.0001) fig=plt.figure(figsize=(12,8.5)) fig.add_subplot(2,2,1) x = np.linspace(meanAbsoluteMagnitudeSamples.min(), meanAbsoluteMagnitudeSamples.max(), 500) plt.plot(x,muDensity(x),'k-') plt.axvline(meanAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\mu_M$") plt.ylabel("$P(\\mu_M)$") fig.add_subplot(2,2,2) x = np.linspace(varAbsoluteMagnitudeSamples.min(), varAbsoluteMagnitudeSamples.max(), 500) plt.plot(x,varDensity(x),'k-') plt.axvline(varianceAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\sigma^2_M$") plt.ylabel("$P(\\sigma^2_M)$") fig.add_subplot(2,2,3) plt.hexbin(meanAbsoluteMagnitudeSamples,varAbsoluteMagnitudeSamples, C=None, bins='log', cmap=cm.gray_r) plt.xlabel("$\\mu_M$") plt.ylabel("$\\sigma^2_M$") plt.figtext(0.55,0.4,"$\\widetilde{\\mu_M}="+"{:4.2f}".format(estimatedAbsMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedAbsMag),ha='left') plt.figtext(0.75,0.4,"$\\mathrm{MAP}(\\widetilde{\\mu_M})="+"{:4.2f}".format(mapValueMu[0])+"$") plt.figtext(0.55,0.35,"$\\widetilde{\\sigma^2_M}="+"{:4.2f}".format(estimatedVarMag) + "$ $\\pm$ ${:4.2f}$".format(errorEstimatedVarMag), ha='left') plt.figtext(0.75,0.35,"$\\mathrm{MAP}(\\widetilde{\\sigma^2_M})="+"{:4.2f}".format(mapValueVar[0])+"$") titelA=("$N_\\mathrm{stars}"+"={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude)+"$") titelB=("Iterations = {0}".format(maxIter)+", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA+"\\quad\\quad "+titelB) titelA=("$N_\\mathrm{stars}"+"={0}".format(numberOfStarsInSurvey) + "$, True values: $\\mu_M={0}".format(meanAbsoluteMagnitude) + "$, $\\sigma^2_M={0}".format(varianceAbsoluteMagnitude)+"$") titelB=("Iterations = {0}".format(maxIter)+", Burn = {0}".format(burnIter) + ", Thin = {0}".format(thinFactor)) plt.suptitle(titelA+"\\quad\\quad "+titelB) titelC=[] titelC.append("MCMC sampling with emcee") titelC.append("$N_\\mathrm{walkers}" + "={0}".format(nwalkers)+"$, $N_\\mathrm{dim}"+"={0}".format(ndim)+"$") plt.figtext(0.55,0.15,titelC[0],horizontalalignment='left') plt.figtext(0.60,0.10,titelC[1],horizontalalignment='left') priorInfo=[] priorInfo.append("Prior on $\\mu_M$: flat $\\quad{0}".format(meanAbsMagLow) + "<\\mu_M<{0}".format(meanAbsMagHigh)+"$") priorInfo.append("Prior on $\\sigma^2_M$: $1/\\sigma^2_M\\quad{0}".format(varianceLow) + "<\\sigma^2_M<{0}".format(varianceHigh)+"$") plt.figtext(0.55,0.25,priorInfo[0],horizontalalignment='left') plt.figtext(0.55,0.20,priorInfo[1],horizontalalignment='left') if (args['pdfOutput']): plt.savefig('luminosityCalibrationResultsEmcee.pdf') elif (args['pngOutput']): plt.savefig('luminosityCalibrationResultsEmcee.png') elif (args['epsOutput']): plt.savefig('luminosityCalibrationResultsEmcee.eps') else: plt.show()
def runMCMCmodel(args): """ Simulate the survey data and run the MCMC luminosity calibration model. Parameters ---------- args - Command line arguments """ mcmcParams=args['mcmcString'] surveyParams=args['surveyString'] priorParams=args['priorsString'] maxIter, burnIter, thinFactor, walkerFactor = [int(par) for par in mcmcParams] minParallax, maxParallax, meanAbsoluteMagnitude, varianceAbsoluteMagnitude =[float(par) for par in surveyParams[1:5]] meanAbsMagLow, meanAbsMagHigh, varianceShape, varianceScale = [float(par) for par in priorParams] if surveyParams[5] == 'Inf': magLim = np.Inf else: magLim = float(surveyParams[5]) simulatedSurvey=U.UniformDistributionSingleLuminosity(int(surveyParams[0]), float(surveyParams[1]), float(surveyParams[2]), float(surveyParams[3]), float(surveyParams[4]), surveyLimit=magLim) #simulatedSurvey.setRandomNumberSeed(53949896) simulatedSurvey.generateObservations() numberOfStarsInSurvey=simulatedSurvey.numberOfStarsInSurvey # Calculate initial guesses for the true parallaxes and absolute magnitudes of the stars. clippedObservedParallaxes=simulatedSurvey.observedParallaxes.clip(minParallax, maxParallax) initialAbsMagGuesses=simulatedSurvey.observedMagnitudes+5.0*np.log10(clippedObservedParallaxes)-10.0 # Initial guesses for hyper parameters (mean absolute magnitude and sigma^2) meanAbsoluteMagnitudeGuess=initialAbsMagGuesses.mean() varianceInit=varianceScale*(varianceShape-1) initialParameters = np.concatenate((np.array([meanAbsoluteMagnitudeGuess, varianceInit]), clippedObservedParallaxes, initialAbsMagGuesses)) # Parameters for emcee ln-posterior function posteriorDict = {'minParallax':minParallax, 'maxParallax':maxParallax, 'muLow':meanAbsMagLow, 'muHigh':meanAbsMagHigh, 'varShape':varianceShape, 'varScale':varianceScale} observations = np.concatenate((simulatedSurvey.observedParallaxes, simulatedSurvey.observedMagnitudes)) observationalErrors=inverseVariance(np.concatenate((simulatedSurvey.parallaxErrors, simulatedSurvey.magnitudeErrors))) # MCMC sampler parameters ndim = 2*numberOfStarsInSurvey+2 nwalkers = walkerFactor*ndim # Generate initial positions for each walker initialPositions=[np.empty((ndim)) for i in xrange(nwalkers)] initialPositions[0]=initialParameters for i in xrange(nwalkers-1): ranMeanAbsMag=np.random.rand()*(meanAbsMagHigh-meanAbsMagLow)+meanAbsMagLow ranVariance=gamma.rvs(varianceShape,scale=varianceScale) ranParallaxes=np.zeros_like(clippedObservedParallaxes) ranParallaxes=clippedObservedParallaxes+simulatedSurvey.parallaxErrors*np.random.randn(numberOfStarsInSurvey) ranAbsMag=np.sqrt(ranVariance)*np.random.randn(numberOfStarsInSurvey)+ranMeanAbsMag initialPositions[i+1]=np.concatenate((np.array([ranMeanAbsMag, ranVariance]), ranParallaxes.clip(minParallax, maxParallax), ranAbsMag)) print '{:*^30}'.format('Building sampler') sampler = emcee.EnsembleSampler(nwalkers, ndim, UniformSpaceDensityGaussianLFemcee, threads=4, args=[posteriorDict, observations, observationalErrors]) # burn-in print '{:*^30}'.format('Burn in') start = now() pos,prob,state = sampler.run_mcmc(initialPositions, burnIter) print '{:*^30}'.format('Finished burning') printSamplingStats(now()-start, sampler, numberOfStarsInSurvey) print # final chain sampler.reset() print '{:*^30}'.format('Start sampling') start = now() sampler.run_mcmc(pos, maxIter, rstate0=state, thin=thinFactor) print '{:*^30}'.format('Finished sampling') printSamplingStats(now()-start, sampler, numberOfStarsInSurvey) # Extract the samples of the posterior distribution chain = sampler.flatchain # Point estimates of mean Absolute Magnitude and its standard deviation. meanAbsoluteMagnitudeSamples = chain[:,0].flatten() varAbsoluteMagnitudeSamples = chain[:,1].flatten() estimatedAbsMag=meanAbsoluteMagnitudeSamples.mean() errorEstimatedAbsMag=meanAbsoluteMagnitudeSamples.std() estimatedVarMag=varAbsoluteMagnitudeSamples.mean() errorEstimatedVarMag=varAbsoluteMagnitudeSamples.std() print "emcee estimates" print " mu_M = {:4.2f} +/- {:4.2f}".format(estimatedAbsMag, errorEstimatedAbsMag) print "sigma^2_M = {:4.2f} +/- {:4.2f}".format(estimatedVarMag, errorEstimatedVarMag) # Plot results # Kernel density estimate of posterior distributions of mu_M and sigma^2_M, also obtain maximum a # posteriori estimate for these quantitities. muDensity, mapValueMu = kdeAndMap(meanAbsoluteMagnitudeSamples) varDensity, mapValueVar = kdeAndMap(varAbsoluteMagnitudeSamples) fig=plt.figure(figsize=(12,8.5)) fig.add_subplot(2,2,1) x = np.linspace(meanAbsoluteMagnitudeSamples.min(), meanAbsoluteMagnitudeSamples.max(), 500) plt.plot(x,muDensity(x),'k-') plt.axvline(meanAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\mu_M$") plt.ylabel("$P(\\mu_M)$") fig.add_subplot(2,2,2) x = np.linspace(varAbsoluteMagnitudeSamples.min(), varAbsoluteMagnitudeSamples.max(), 500) plt.plot(x,varDensity(x),'k-') plt.axvline(varianceAbsoluteMagnitude, linewidth=2, color="red") plt.xlabel("$\\sigma^2_M$") plt.ylabel("$P(\\sigma^2_M)$") fig.add_subplot(2,2,3) plt.hexbin(meanAbsoluteMagnitudeSamples,varAbsoluteMagnitudeSamples, C=None, bins='log', cmap=cm.gray_r) plt.xlabel("$\\mu_M$") plt.ylabel("$\\sigma^2_M$") plt.figtext(0.55,0.4,"$\\widetilde{{\\mu_M}}={:4.2f}\\pm{:4.2f}$".format(estimatedAbsMag, errorEstimatedAbsMag),ha='left') plt.figtext(0.75,0.4,"$\\mathrm{{MAP}}(\\widetilde{{\\mu_M}})={:4.2f}$".format(mapValueMu[0])) plt.figtext(0.55,0.35,"$\\widetilde{{\\sigma^2_M}}={:4.2f}\\pm{:4.2f}$".format(estimatedVarMag, errorEstimatedVarMag), ha='left') plt.figtext(0.75,0.35,"$\\mathrm{{MAP}}(\\widetilde{{\\sigma^2_M}})={:4.2f}$".format(mapValueVar[0])) titelA=("$N_\\mathrm{{stars}}={}$, True values: $\\mu_M={}$, $\\sigma^2_M={}$".format(numberOfStarsInSurvey, meanAbsoluteMagnitude, varianceAbsoluteMagnitude)) titelB=("Iterations = {}, Burn = {}, Thin = {}".format(maxIter, burnIter, thinFactor)) plt.suptitle(titelA+"\\quad\\quad "+titelB) titelC=[] titelC.append("MCMC sampling with emcee") titelC.append("$N_\\mathrm{{walkers}}={}$, $N_\\mathrm{{dim}}={}".format(nwalkers, ndim)) plt.figtext(0.55,0.15,titelC[0],horizontalalignment='left') plt.figtext(0.60,0.10,titelC[1],horizontalalignment='left') priorInfo=[] priorInfo.append("Prior on $\\mu_M$: flat $\\quad{}<\\mu_M<{}$".format(meanAbsMagLow, meanAbsMagHigh)) priorInfo.append("Prior on $\\sigma^2_M$: $\\Gamma(\\sigma^2_M | k={},\\theta={})$".format(varianceShape, varianceScale)) plt.figtext(0.55,0.25,priorInfo[0],horizontalalignment='left') plt.figtext(0.55,0.20,priorInfo[1],horizontalalignment='left') if (args['pdfOutput']): plt.savefig('luminosityCalibrationResultsEmcee.pdf') elif (args['pngOutput']): plt.savefig('luminosityCalibrationResultsEmcee.png') elif (args['epsOutput']): plt.savefig('luminosityCalibrationResultsEmcee.eps') else: plt.show()