def goodFits(bestfitloc='posteriorpdf.fits', Nfits=12, Ngood=5000, cleanup=True, interactive=True, showOptical=False): """ Read posterior PDF and draw Nfits realizations from the final Ngood models at random. Plot the model from each realization and compare to the data. Also plot the residuals obtained after subtracting the model from the data and compare to the data. By default: Nfits = 12, Ngood=5000. """ import modifypdf import numpy # read the posterior PDFs print("Found posterior PDF file: {:s}".format(bestfitloc)) fitresults = fits.getdata(bestfitloc) fitresults = fitresults[-Ngood:] fitresults = modifypdf.prune(fitresults) # get keys from astropy.table import Table fitKeys = Table.read(bestfitloc).keys() # select the random realizations model Nunprune = len(fitresults) realids = numpy.floor(numpy.random.uniform(0, Nunprune, Nfits)) for ifit in range(Nfits): realid = numpy.int(realids[ifit]) fitresult = fitresults[realid] tag = 'goodfit' + str(realid).zfill(4) printFitParam(fitresult, fitKeys) visualutil.plotFit(config, fitresult, tag=tag, showOptical=showOptical, cleanup=cleanup, interactive=interactive)
def plotPDF(fitresults, tag, limits='', Ngood=5000, axes='auto'): """ Plot the PDF of each parameter of the model. """ import numpy import matplotlib.pyplot as plt from pylab import savefig from matplotlib import rc import modifypdf # plotting parameters rc( 'font', **{ 'family': 'sans-serif', 'sans-serif': ['Arial Narrow'], 'size': '12' }) # grab the last Ngood fits fitresults = fitresults[-Ngood:] #lnprobstring = "prior to pruning <Ln Prob>: {:f}" #print(lnprobstring.format(fitresults['lnprob'].mean())) # identify the good fits fitresultsgood = modifypdf.prune(fitresults) # determine dimensions of PDF plots nparams = len(fitresultsgood[0]) ncol = 4 nrow = nparams / ncol + 1 j = 1 plt.figure(figsize=(12.0, 1.5 * nrow)) # set up the plotting window plt.subplots_adjust(left=0.08, bottom=0.15, right=0.95, top=0.95, wspace=0.4, hspace=0.65) pnames = fitresultsgood.names counter = 0 for pname in pnames: # Position of primary lens frg = fitresultsgood[pname] rmsval = numpy.std(frg) if rmsval > 1e-6: avgval = numpy.mean(frg) print(pname + ' = ' + str(avgval) + ' +/- ' + str(rmsval)) totalwidth = frg.max() - frg.min() nbins = totalwidth / rmsval * 5 ax = plt.subplot(nrow, ncol, j) j += 1 plt.hist(frg, nbins, edgecolor='blue') plt.ylabel('N') plt.xlabel(pname) if axes == 'auto': start, end = ax.get_xlim() nticks = 5 stepsize = (end - start) / nticks ax.xaxis.set_ticks( numpy.arange(start, end + 0.99 * stepsize, stepsize)) elif axes == 'initial': oldaxis = plt.axis() if pname[0:6] == 'lnprob': xmin = frg.min() xmax = frg.max() elif pname[0:2] == 'mu': xmin = 0 xmax = 30 else: p_l = limits[0] p_u = limits[1] xmin = p_l[counter] xmax = p_u[counter] counter += 1 ymin = oldaxis[2] ymax = oldaxis[3] plt.axis([xmin, xmax, ymin, ymax]) savefile = tag + 'PDFs.png' savefig(savefile)
def covariance(bestfitloc='posteriorpdf.fits'): """ Plot the covariance matrix for the parameters of the model. """ import matplotlib.pyplot as plt import numpy from pylab import savefig import modifypdf from astropy.table import Table from matplotlib import rc # plotting parameters rc('font', **{ 'family': 'sans-serif', 'sans-serif': ['Arial Narrow'], 'size': '6' }) posteriorpdf = Table.read(bestfitloc) posteriorpdf = posteriorpdf[-5000:] # remove columns where the values are not changing posteriorpdfclean = modifypdf.cleanColumns(posteriorpdf) posteriorpdfgood = modifypdf.prune(posteriorpdfclean) headers = posteriorpdf.colnames ncol = len(headers) k = 0 xsize = ncol * 2 ysize = ncol * 1.5 fig = plt.figure(figsize=(xsize, ysize)) plt.subplots_adjust(left=0.020, bottom=0.02, right=0.99, top=0.97, wspace=0.5, hspace=0.5) #for i in numpy.arange(ncol): # ax = plt.subplot(ncol, ncol, i + 1) # namex = 'mu_aper' # namey = headers[i] # datax = mupdfgood[namex] # datay = posteriorpdfgood[namey] # if namex == 'lnprob': # datax = datax.max() - datax # if namey == 'lnprob': # datay = datay.max() - datay # lnprob = posteriorpdfgood['lnprob'].max() - posteriorpdfgood['lnprob'] # plt.hexbin(datax, datay, C = lnprob) # plt.xlabel(namex) # plt.ylabel(namey) for i in numpy.arange(ncol): for j in numpy.arange(ncol - i - 1) + i + 1: plotspot = ncol * i + j ax = plt.subplot(ncol, ncol, plotspot) namex = headers[i] namey = headers[j] #datax = posteriorpdforig[namex] #datay = posteriorpdforig[namey] #lnprob = posteriorpdforig['lnprob'] #plt.hexbin(datax, datay, C = lnprob, color='black') datax = posteriorpdfgood[namex] datay = posteriorpdfgood[namey] if namex == 'lnprob': datax = datax.max() - datax if namey == 'lnprob': datay = datay.max() - datay lnprob = posteriorpdfgood['lnprob'].max( ) - posteriorpdfgood['lnprob'] plt.hexbin(datax, datay, C=lnprob) plt.xlabel(namex) plt.ylabel(namey) print(i, j, plotspot, namex, namey) k += 1 #plt.suptitle(iau_address, x=0.5, y=0.987, fontsize='xx-large') savefig('covariance.pdf') plt.clf()
# extract object name from directory objname = dataname[itarg] iauname = dataphot['iauname'][itarg] goodfit = goodfitdat['intrinsic'][itarg] #goodfit = goodfitdat['observed'][itarg] fitdir = anloc + dataname[itarg] + '/' + goodfit + '/' os.chdir(fitdir) # ------------------------------------------------------------------------ # Read posterior probability distributions fitloc = 'posteriorpdf.fits' fitresults = Table.read(fitloc) fitresults = fitresults[-5000:] fitresultsgood = modifypdf.cleanColumns(fitresults) fitresultsgood = modifypdf.bigScoop(fitresultsgood) fitresultsgood = modifypdf.prune(fitresultsgood, quiet=True) nfits = len(fitresultsgood) configfile = open('config.yaml') config = yaml.load(configfile) paramData = setuputil.loadParams(config) if objname != objname_previous: radeg_centroid = config['Region0']['RACentroid'] decdeg_centroid = config['Region0']['DecCentroid'] c = SkyCoord(ra=radeg_centroid*u.degree, dec=decdeg_centroid*u.degree) fullRA = c.ra.to_string(unit='hour', pad=True, sep=':', precision=3) fullDec = c.dec.to_string('deg', pad=True, alwayssign=True, sep=':', precision=2) msg = '${0:9}$ ${1:12}$ ${2:12}$' msgfmt = msg.format(objname, fullRA, fullDec)
def all(goodfit, update=False, mu_estimate=False): # Check to see whether we've already loaded the data if not update: ffile = 'flux_total_observed.dat' if os.path.exists(ffile): ffile = 'flux_total_observed.dat' flux_total_observed = Table.read(ffile, format='ascii') ffile = 'flux_indiv_observed.dat' flux_indiv_observed = Table.read(ffile, format='ascii') ffile = 'flux_indiv_intrinsic.dat' flux_indiv_intrinsic = Table.read(ffile, format='ascii') else: update = True if update: cwd = os.getcwd() # read in the sample targets anloc = '../../ModelFits/' dataphotloc = 'targetlist.dat' dataphot = Table.read(dataphotloc, format='ascii') dataname = dataphot['dataname'] ntarg = dataname.size dataflag = numpy.zeros(ntarg) for i in range(ntarg): datanamei = dataname[i] dataloc = anloc + datanamei + '/' + goodfit if os.path.exists(dataloc): dataflag[i] = 1 yesmodel = dataflag == 1 dataphot = dataphot[yesmodel] dataname = dataphot['dataname'] ntarg = len(dataphot) dataname_indiv = [] flux_total = [] e_flux_total = [] flux_indiv = [] e_flux_indiv = [] flux_indiv_int = [] e_flux_indiv_int = [] for itarg in range(0, ntarg): # extract object name from directory objname = dataname[itarg] iauname = dataphot['iauname'][itarg] fitdir = anloc + dataname[itarg] + '/' + goodfit + '/' os.chdir(fitdir) # ------------------------------------------------------------------------ # Read posterior probability distributions fitloc = 'posteriorpdf.fits' fitresults = Table.read(fitloc) fitresults = fitresults[-5000:] fitresultsgood = modifypdf.cleanColumns(fitresults) fitresultsgood = modifypdf.bigScoop(fitresultsgood) fitresultsgood = modifypdf.prune(fitresultsgood, quiet=True) configfile = open('config.yaml') config = yaml.load(configfile) paramData = setuputil.loadParams(config) nregions = paramData['nregions'] flux_target = 0 os.chdir(cwd) for iregion in range(nregions): sr = str(iregion) nsource = paramData['nsource_regions'][iregion] for isource in range(nsource): ss = str(isource) datanamesource = dataname[itarg] + '.Source' + ss tag1 = 'IntrinsicFlux_Source' + ss + '_Region' + sr if tag1 not in fitresults.keys(): tag1 = 'Region' + sr + ' Source' + ss + ' IntrinsicFlux' fluxdist = fitresultsgood[tag1] tag2 = 'mu_tot.Source' + ss + '.Region' + sr # get the by-eye magnification estimates if mu_estimate: datamu = Table.read('mu_estimate.dat', format='ascii') muindx = datamu['target'] == datanamesource muerr = datamu['muerr'][muindx][0] if muerr == 0.25: mu_by_eye = datamu['mu'][muindx][0] else: mu_by_eye = 1. else: mu_by_eye = 1. flux_indiv_int.append(fluxdist.mean() / mu_by_eye) e_flux_indiv_int.append(fluxdist.std() / mu_by_eye) if tag2 in fitresults.keys(): if fitresultsgood[tag2].mean() == 100: continue observedflux = fluxdist * fitresultsgood[tag2] print(datanamesource, fitresultsgood[tag2].mean(), fitresultsgood[tag2].std()) else: observedflux = fluxdist print(datanamesource, 1.0) flux_target += observedflux flux_indiv.append(observedflux.mean()) e_flux_indiv.append(observedflux.std()) dataname_indiv.append(datanamesource) flux_measure = flux_target.mean() flux_total.append(flux_measure) flux_error = flux_target.std() e_flux_total.append(flux_error) #import matplotlib.pyplot as plt #plt.hist(flux_target) #plt.show() #import pdb; pdb.set_trace() #print(objname, flux_measure, ' +/- ', flux_error) flux_total_observed = {'targets': dataname, 'fnu': flux_total, 'e_fnu': e_flux_total} t1 = Table(flux_total_observed) t1.write('flux_total_observed.dat', format='ascii') flux_indiv_observed = {'targets': dataname_indiv, 'fnu': flux_indiv, 'e_fnu': e_flux_indiv} t1 = Table(flux_indiv_observed) t1.write('flux_indiv_observed.dat', format='ascii') flux_indiv_intrinsic = {'targets': dataname_indiv, 'fnu': flux_indiv_int, 'e_fnu': e_flux_indiv_int} t1 = Table(flux_indiv_intrinsic) t1.write('flux_indiv_intrinsic.dat', format='ascii') #import matplotlib.pyplot as plt #m1 = 0 #m2 = 1 + int(numpy.array(flux_total + flux_indiv + flux_indiv_int).max()) #binwidth = 2 #bins = numpy.arange(m1, m2 + binwidth, binwidth) #plt.hist(flux_total, bins = bins, histtype='stepfilled') #binwidth = 1.8 #bins = numpy.arange(m1, m2 + binwidth, binwidth) #plt.hist(flux_indiv, bins = bins, alpha=0.5, histtype='stepfilled') #binwidth = 1.6 #bins = numpy.arange(m1, m2 + binwidth, binwidth) #plt.hist(flux_indiv_int, bins = bins, alpha=0.2, histtype='stepfilled') #plt.show() return flux_total_observed, flux_indiv_observed, flux_indiv_intrinsic
def covariance(bestfitloc='posteriorpdf.fits'): """ Plot the covariance matrix for the parameters of the model. """ import matplotlib.pyplot as plt import numpy from pylab import savefig import modifypdf from astropy.table import Table from matplotlib import rc # plotting parameters rc('font',**{'family':'sans-serif', 'sans-serif':['Arial Narrow'], 'size':'6'}) posteriorpdf = Table.read(bestfitloc) posteriorpdf = posteriorpdf[-5000:] # remove columns where the values are not changing posteriorpdfclean = modifypdf.cleanColumns(posteriorpdf) posteriorpdfgood = modifypdf.prune(posteriorpdfclean) headers = posteriorpdf.colnames ncol = len(headers) k = 0 xsize = ncol * 2 ysize = ncol * 1.5 fig = plt.figure(figsize=(xsize, ysize)) plt.subplots_adjust(left=0.020, bottom=0.02, right=0.99, top=0.97, wspace=0.5, hspace=0.5) #for i in numpy.arange(ncol): # ax = plt.subplot(ncol, ncol, i + 1) # namex = 'mu_aper' # namey = headers[i] # datax = mupdfgood[namex] # datay = posteriorpdfgood[namey] # if namex == 'lnprob': # datax = datax.max() - datax # if namey == 'lnprob': # datay = datay.max() - datay # lnprob = posteriorpdfgood['lnprob'].max() - posteriorpdfgood['lnprob'] # plt.hexbin(datax, datay, C = lnprob) # plt.xlabel(namex) # plt.ylabel(namey) for i in numpy.arange(ncol): for j in numpy.arange(ncol - i - 1) + i + 1: plotspot = ncol * i + j ax = plt.subplot(ncol, ncol, plotspot) namex = headers[i] namey = headers[j] #datax = posteriorpdforig[namex] #datay = posteriorpdforig[namey] #lnprob = posteriorpdforig['lnprob'] #plt.hexbin(datax, datay, C = lnprob, color='black') datax = posteriorpdfgood[namex] datay = posteriorpdfgood[namey] if namex == 'lnprob': datax = datax.max() - datax if namey == 'lnprob': datay = datay.max() - datay lnprob = posteriorpdfgood['lnprob'].max() - posteriorpdfgood['lnprob'] plt.hexbin(datax, datay, C = lnprob) plt.xlabel(namex) plt.ylabel(namey) print(i, j, plotspot, namex, namey) k += 1 #plt.suptitle(iau_address, x=0.5, y=0.987, fontsize='xx-large') savefig('covariance.pdf') plt.clf()
nticks = 5 #deltachi2 = 100 posteriorloc = 'posteriorpdf.hdf5' # read posterior PDF print "Reading output from emcee" fitresults = hdf5.read_table_hdf5(posteriorloc) fitresults = fitresults[-5000:] print 'prior to pruning: ', fitresults['lnprob'].mean() # identify the good fits fitresultsgood = modifypdf.prune(fitresults) # determine dimensions of PDF plots nparams = len(fitresultsgood[0]) ncol = 4 nrow = nparams / ncol + 1 j = 1 fig = mpl.figure(figsize=(8.0, 1.0 * nrow)) # set up the plotting window mpl.subplots_adjust(left=0.08, bottom=0.1, right=0.95, top=0.95, wspace=0.4, hspace=0.65) pnames = fitresultsgood.keys()
def plotPDF(fitresults, tag, limits='', Ngood=5000, axes='auto'): """ Plot the PDF of each parameter of the model. """ import numpy import matplotlib.pyplot as plt from pylab import savefig from matplotlib import rc import modifypdf # plotting parameters rc('font',**{'family':'sans-serif', 'sans-serif':['Arial Narrow'], 'size':'12'}) # grab the last Ngood fits fitresults = fitresults[-Ngood:] #lnprobstring = "prior to pruning <Ln Prob>: {:f}" #print(lnprobstring.format(fitresults['lnprob'].mean())) # identify the good fits fitresultsgood = modifypdf.prune(fitresults) # determine dimensions of PDF plots nparams = len(fitresultsgood[0]) ncol = 4 nrow = nparams / ncol + 1 j = 1 plt.figure(figsize=(12.0, 1.5 * nrow)) # set up the plotting window plt.subplots_adjust(left=0.08, bottom=0.15, right=0.95, top=0.95, wspace=0.4, hspace=0.65) pnames = fitresultsgood.names counter = 0 for pname in pnames: # Position of primary lens frg = fitresultsgood[pname] rmsval = numpy.std(frg) if rmsval > 1e-6: avgval = numpy.mean(frg) print(pname + ' = ' + str(avgval) + ' +/- ' + str(rmsval)) totalwidth = frg.max() - frg.min() nbins = totalwidth / rmsval * 5 ax = plt.subplot(nrow, ncol, j) j += 1 plt.hist(frg, nbins, edgecolor='blue') plt.ylabel('N') plt.xlabel(pname) if axes == 'auto': start, end = ax.get_xlim() nticks = 5 stepsize = (end - start) / nticks ax.xaxis.set_ticks(numpy.arange(start, end + 0.99*stepsize, stepsize)) elif axes == 'initial': oldaxis = plt.axis() if pname[0:6] == 'lnprob': xmin = frg.min() xmax = frg.max() elif pname[0:2] == 'mu': xmin = 0 xmax = 30 else: p_l = limits[0] p_u = limits[1] xmin = p_l[counter] xmax = p_u[counter] counter += 1 ymin = oldaxis[2] ymax = oldaxis[3] plt.axis([xmin, xmax, ymin, ymax]) savefile = tag + 'PDFs.png' savefig(savefile)
def plotPDF(fitresults, tag, limits='', Ngood=5000, axes='auto'): """ Plot the PDF of each parameter of the model. Returns ------- avg_dic: dict key = names of the model parameters, value = average value from the last Ngood samples """ import numpy import matplotlib.pyplot as plt from pylab import savefig from matplotlib import rc import modifypdf # plotting parameters rc('font',**{'family':'sans-serif', 'sans-serif':['Arial Narrow'], 'size':'12'}) # grab the last Ngood fits fitresults = fitresults[-Ngood:] #lnprobstring = "prior to pruning <Ln Prob>: {:f}" #print(lnprobstring.format(fitresults['lnprob'].mean())) # identify the good fits fitresultsgood = modifypdf.prune(fitresults) # determine dimensions of PDF plots nparams = len(fitresultsgood[0]) ncol = 4 nrow = (nparams/ncol + 1) if nparams % ncol != 0 else nparams/ncol plt.figure(figsize=(18.0, 2.0 * nrow)) # set up the plotting window plt.subplots_adjust(left=0.08, bottom=0.15, right=0.95, top=0.95, wspace=0.4, hspace=0.65) pnames = fitresultsgood.names width = 1e-6 # intialize a dictionary w/ keywords = pnames to hold the average value of each paramter in the chain avg_dic = dict.fromkeys(pnames) for i, pname in enumerate(pnames): ax = plt.subplot(nrow, ncol, i+1) # nparams/ncol frg = fitresultsgood[pname] rmsval = numpy.std(frg) if rmsval > width: avgval = numpy.mean(frg) print("{:s} = {:.4f} +/- {:.4f}").format(pname, avgval, rmsval) avg_dic[pname] = avgval totalwidth = frg.max() - frg.min() nbins = totalwidth / rmsval * 5 plt.hist(frg, int(nbins), edgecolor='blue') plt.ylabel('N') plt.xlabel(pname) if axes == 'auto': start, end = ax.get_xlim() nticks = 5 stepsize = (end - start) / nticks ax.xaxis.set_ticks(numpy.arange(start, end + 0.99*stepsize, stepsize)) elif axes == 'initial': oldaxis = plt.axis() if pname[0:6] == 'lnprob': xmin = frg.min() xmax = frg.max() elif pname[0:2] == 'mu': xmin = 0 xmax = 30 else: p_l = limits[0] p_u = limits[1] xmin = p_l[i] xmax = p_u[i] ymin = oldaxis[2] ymax = oldaxis[3] plt.axis([xmin, xmax, ymin, ymax]) else: print('*** Distribution narrorwer than {} ***').format(width) print('*** This is likely a fixed parameter: {}. ***').format(pname) print('*** Check config.yaml *** ') savefile = tag + 'PDFs.png' savefig(savefile) return avg_dic
""" import matplotlib.pyplot as plt from astropy.io.misc import hdf5 import numpy from pylab import savefig import modifypdf # examine each inputs_lens.dat file to determine total number of lenses posteriorpdfloc = 'posteriorpdf.hdf5' posteriorpdf = hdf5.read_table_hdf5(posteriorpdfloc) posteriorpdf = posteriorpdf[-5000:] posteriorpdfgood = modifypdf.prune(posteriorpdf) headers = posteriorpdf.colnames() ncol = len(headers) k = 0 xsize = ncol * 3 ysize = ncol * 2.5 fig = plt.figure(figsize=(xsize, ysize)) plt.subplots_adjust(left=0.020, bottom=0.02, right=0.99, top=0.97, wspace=0.5, hspace=0.5) #for i in numpy.arange(ncol): # ax = plt.subplot(ncol, ncol, i + 1) # namex = 'mu_aper' # namey = headers[i] # datax = mupdfgood[namex]
cwd = os.getcwd() anloc = '../../ModelFits/' dataphotloc = 'targetlist.dat' dataphot = Table.read(dataphotloc, format='ascii') dataname = dataphot['dataname'] ntarg = dataname.size dataflag = numpy.zeros(ntarg) for itarg in range(0, ntarg): dirloc = anloc + dataname[itarg] + '/' + fitid print dirloc #if dataname[itarg] == 'XMM01': # continue os.chdir(dirloc) burninfiles = glob.glob(burninfile) burninlast = burninfiles[-1] fitresults = Table.read(burninlast, format='ascii', data_start=-5000) prunedpdf = modifypdf.prune(fitresults) import matplotlib.pyplot as plt plt.clf() maxprob = fitresults['lnprob'].max() + 0.1 hist1 = numpy.log10(maxprob - fitresults['lnprob']) plt.hist(hist1, bins=15, log=True) hist1 = numpy.log10(maxprob - prunedpdf['lnprob']) plt.hist(hist1, bins=15, log=True) plt.title(dataname[itarg]) from pylab import savefig savefig('prunecheck') os.chdir(cwd)