def cropMCMC_old(mcmcfile,outfile,cropperc):
""" Removes the "burn-in" phase of the MCMC and writes a
shortened MCMC data file
"""
hdrkeys = readMCMChdr(mcmcfile)
Nparams = getNparams(hdrkeys)
outfileObject = open(outfile,'w')
mcmcfileobj = open(mcmcfile,'r')
mcmcfileobj = mcmcfileobj.readlines()
for line in mcmcfileobj:
if line.startswith('#'):
print >> outfileObject, line.strip('\n')
if not line.startswith('#'):
data_line = ReadMCMCline(line,hdrkeys)
if Nparams == 1:
if data_line['acr'] > 0.44-cropperc and data_line['acr'] < 0.44+cropperc:
print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
print >> outfileObject, line.strip('\n')
if Nparams > 1:
if data_line['acr'] > 0.23-cropperc and data_line['acr'] < 0.23+cropperc:
print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
print >> outfileObject, line.strip('\n')
outfileObject.close()
def cropMCMC(mcmcfile,outfile,cropperc,NumCropLine,**kwargs):
""" Removes the "burn-in" phase of the MCMC and writes a
shortened MCMC data file
"""
silent = True
for key in kwargs:
if key.lower().startswith('sil'):
silent = kwargs[key]
hdrkeys = readMCMChdr(mcmcfile)
Nparams = getNparams(hdrkeys)
outfileObject = open(outfile,'w')
mcmcfileobj = open(mcmcfile,'r')
mcmcfileobj = mcmcfileobj.readlines()
NLine = 0
for line in mcmcfileobj:
NLine += 1
if line.startswith('#'):
print >> outfileObject, line.strip('\n')
if NLine > NumCropLine:
if not line.startswith('#'):
data_line = ReadMCMCline(line,hdrkeys)
if Nparams == 1:
if data_line['acr'] > 0.44-cropperc and data_line['acr'] < 0.44+cropperc:
if not silent: print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
print >> outfileObject, line.strip('\n')
if Nparams > 1:
if data_line['acr'] > 0.23-cropperc and data_line['acr'] < 0.23+cropperc:
if not silent: print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
print >> outfileObject, line.strip('\n')
outfileObject.close()
def makeStartFromExplore(ListOfChains,StablePerc,SampleParamFile,OutputParamFile):
"""
Reads a list of single parameter step-exoploration chains
and prints a startparam file based on this data.
This routine also prints whether a chain has stabilized
the correct single-parameter acceptance rate.
"""
ListFiles = open(ListOfChains,'r')
ListFiles = ListFiles.readlines()
ModelParams = ReadStartParams(SampleParamFile)
step = {}
for file in ListFiles:
data = readMCMC(file.strip('\n'))
if data['acr'][-1] > 0.44-StablePerc and data['acr'][-1] < 0.44+StablePerc:
for par in data.keys():
if not isNonParam(par):
medfrac = np.median(data['frac'][-1000:])
print par+' has stabilized, acr = '+format(data['acr'][-1],'.2f')+' frac = '+str(medfrac)
step[par] = {'frac':medfrac}
else:
for par in data.keys():
if not isNonParam(par):
print par+' has NOT stabilized, acr = '+format(data['acr'][-1],'.2f')
for par in step.keys():
for key in ModelParams.keys():
if key == par:
ModelParams[par]['step'] = ModelParams[par]['step']*step[par]['frac']
ModelParams[par]['open'] = True
PrintModelParams(ModelParams,OutputParamFile)
def TableEntry(errEntry,par,fStr):
Entry = 'Wrong'
if par == 'Period':
Val = format(errEntry[par]['value'],fStr)
elif par.startswith('T0'):
Val = format(errEntry[par]['value'],fStr)
else:
Val = format(errEntry[par]['value'],fStr)
if np.isnan(errEntry[par]['lower']) or \
np.isnan(errEntry[par]['upper']):
Entry = '('+format(errEntry[par]['value'],fStr)+')'
elif (errEntry[par]['lower'] != errEntry[par]['upper']) and \
not errEntry[par]['useSingle']:
Upp = format(errEntry[par]['upper'],fStr)
Low = format(errEntry[par]['lower'],fStr)
if float(Upp) == 0e0 or float(Low) == 0e0:
maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
errPart = '$\pm$'+format(maxerr,fStr)
Entry = Val+errPart
elif float(Upp) == float(Low):
maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
errPart = '$\pm$'+format(maxerr,fStr)
Entry = Val+errPart
else:
errPart = '$^{+'+Upp+'}_{-'+Low+'}$'
Entry = Val+errPart
else:
maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
errPart = '$\pm$'+format(maxerr,fStr)
Entry = Val+errPart
return Entry
def WriteLightCurveFile(OutFileName,file,IngressTime,EgressTime):
""" Writes a Lightcurve file (usable by MCMC), normalizing the out-of-transit
lightcurve to 1.
Inputs
OutFileName - String with the name of the file to write the lightcurve
file - String with the name of the file with Input data (IDL data file)
IngressTime - Float with an approximate guess of the onset of Ingress
EgressTime - Float with an approximate guess of the end of Egress
Result - Lightcurve with out-of-eclipse data normalized to 1 is written to OutFileName
"""
DataDictionary, HeaderData = ReadData(file)
FileObject = open(OutFileName,'w')
Nlen = len(DataDictionary['(BJD)'])
outdata = {'time':[],'flux_ratio':[],'err_fluxratio':[]}
for i in range(Nlen):
outdata['time'].append(DataDictionary['JDprefix']+DataDictionary['(BJD)'][i])
if DataDictionary['f1'][i] == 0.0 or DataDictionary['f2'][i] == 0.0 or \
DataDictionary['f1'][i] < 0.0 or DataDictionary['f2'][i] < 0.0:
# if there is no flux data write 'nans'
outdata['flux_ratio'].append(float('nan'))
outdata['err_fluxratio'].append(float('nan'))
else:
# compute flux ratios
outdata['flux_ratio'].append(DataDictionary['f1'][i]/DataDictionary['f2'][i])
outdata['err_fluxratio'].append((1e0/DataDictionary['f2'][i])*\
np.sqrt(DataDictionary['erf1sq'][i] + DataDictionary['erf2sq'][i]*\
(DataDictionary['f1'][i]/DataDictionary['f2'][i])**2))
flat_flux = []
for i in range(len(outdata['time'])):
if outdata['time'][i] < IngressTime or outdata['time'][i] > EgressTime:
flat_flux.append(outdata['flux_ratio'][i])
# normalize lightcurve
normalizing_factor = np.median(np.array(flat_flux))
print normalizing_factor
if np.isnan(normalizing_factor):
normalizing_factor = 1e0
print >> FileObject, '# TDB | flux_ratio | err_fluxratio'
for i in range(len(outdata['time'])):
time = format(outdata['time'][i],'.9f')
fratio = format(outdata['flux_ratio'][i]/normalizing_factor,'.7f')
err_fratio = format(outdata['err_fluxratio'][i]/normalizing_factor,'.7f')
print >> FileObject, time+' | '+fratio+' | '+err_fratio
FileObject.close()
def printMCMCline(param_tagorder,ModelParams,istep,frac,acr,chi1,chi2):
""" Given ModelParams, the parameter order and other simulation parameters writes lines for the MCMC file. """
line = ''
Nparams = len(param_tagorder.keys())
# print param_tagorder
for el in range(Nparams):
if el == 0:
line =\
str(format(ModelParams[param_tagorder[el]]['value'],\
ModelParams[param_tagorder[el]]['printformat']))
if el > 0:
line = line+'|'+format(ModelParams[param_tagorder[el]]['value'],\
ModelParams[param_tagorder[el]]['printformat'])
line = line+'|'+str(format(istep,'.0f'))+'|'+str(frac)+'|'+str(acr)+\
'|'+str(format(chi1,'.4f'))+'|'+str(format(chi2,'.4f'))+'|:'
return line
def PrintModelParams(ModelParams,OutFile):
""" Given the dictionary of Model parameters, this routine write a file in the format of the
starting parameter files
"""
OutFileObject = open(OutFile,'w')
print >> OutFileObject, '# Parname | value | step | open | printformat '
for param in ModelParams.keys():
if param == 'RefFilt':
print >> OutFileObject, param+' | '+str(ModelParams[param]['value'])+' | '\
+str(ModelParams[param]['step'])+' | '\
+str(ModelParams[param]['open'])+' | '+str(ModelParams[param]['printformat'])
else:
print >> OutFileObject, param+' | '+str(format(ModelParams[param]['value'],ModelParams[param]['printformat']))+' | '\
+str(format(ModelParams[param]['step'],ModelParams[param]['printformat']))+' | '\
+str(ModelParams[param]['open'])+' | '+str(ModelParams[param]['printformat'])
OutFileObject.close()
return
def MinuitPar2Err(ParFile,ErrFile):
""" Convert a Minuit par file to an error file """
ModelParams = ReadStartParams(ParFile)
OutFile = open(ErrFile,'w')
header = '# parname | value | lower | upper | useSingle ?'
print >> OutFile, header
for par in ModelParams.keys():
if par.lower() != 'reffilt':
if ModelParams[par]['open']:
ValString = format(ModelParams[par]['value'],ModelParams[par]['printformat'])
ErrString = format(ModelParams[par]['step'],ModelParams[par]['printformat'])
print >> OutFile, par+'|'+ValString+'|'+ErrString+'|'+ErrString+'| True '
else:
ValString = format(ModelParams[par]['value'],ModelParams[par]['printformat'])
ErrString = 'nan'
print >> OutFile, par+'|'+ValString+'|'+ErrString+'|'+ErrString+'| False '
OutFile.close()
def makeStatLabels(Stats,DataFile):
""" """
parList = getPars(DataFile)
Npar = len(parList)
iplot = 1
StatLabels = {}
for iy in range(Npar):
for ix in range(Npar):
if not ix >= iy:
parName1 = parList[ix]
parName2 = parList[iy]
cov = r'$|\sigma_{(x,y)}|$='+\
format(abs(Stats['cov'][parName1][parName2]['value']),'0.2f')
spe = r'$|\rho|$='+\
format(abs(Stats['spear'][parName1][parName2]['value']),'0.2f')
pea = r'$|r|$='+\
format(abs(Stats['pear'][parName1][parName2]['value']),'0.2f')
StatLabels[iplot] = {'cov':cov,'spe':spe,'pea':pea}
def WriteLowestChisq(file,ModelParams,OutFileName,ShowOutput):
""" Finds the lowest chisq point in MCMC and prints to a file
of format similar to the start paramfile.
Inputs
file - the MCMC file
ModelPars
OutFileName
Output
a file with OutFileName
"""
ModelParamsCopy = {}
if checkFileExists(file):
FileObject = open(file,'r')
FileObjectLines = FileObject.readlines()
minchi = 1e308
for line in FileObjectLines:
if line.startswith('#'):
ParNames = ReadHeaderMCMC(line)
else:
if line.endswith("|:\n"):
par0 = ReadMCMCline(line,ParNames)
if par0['chi1'] < minchi:
minchi = par0['chi1']
if ShowOutput: print 'Lowest chisq = ',format(minchi,'.2f'),\
' @ step = ',format(par0['istep'],'n')
for key in ModelParams.keys():
if ModelParams[key]['open']:
ModelParamsCopy[key] = {'value':par0[key],'step':ModelParams[key]['step'],\
'printformat':ModelParams[key]['printformat'],'open':ModelParams[key]['open']}
else:
ModelParamsCopy[key] = {'value':ModelParams[key]['value'],'step':\
ModelParams[key]['step'],'printformat':ModelParams[key]\
['printformat'],'open':ModelParams[key]['open']}
else:
pass
PrintModelParams(ModelParamsCopy,OutFileName)
else:
print file, ' does not exist.'
def getEntryString(value,lower,upper,useSingle,parformat):
""" """
if useSingle:
errVal = np.max([lower,upper])
errStr = format(errVal,parformat)
ValStr = format(value,parformat)
EntryStr = r'%s $\pm$ %s' % (ValStr,errStr)
else:
if lower == upper:
errVal = np.max([lower,upper])
errStr = format(errVal,parformat)
ValStr = format(value,parformat)
EntryStr = r'%s $\pm$ %s' % (ValStr,errStr)
else:
errUpStr = format(upper,parformat)
errLowStr = format(lower,parformat)
ValStr = format(value,parformat)
EntryStr = r'%s$^{+%s}_{-%s}$' % (ValStr,errUpStr,errLowStr)
return EntryStr
def printErrors(MCMCfile,BestfitFile,OutputFile):
""" Gets data out from the MCMC data file and the BESTFIT parameters file and prints uncertainties """
mcmcData = readMCMC(MCMCfile)
BestFitParams = ReadStartParams(BestfitFile)
erf15 = ((1e0 - scipy.special.erf(1e0/np.sqrt(2e0)))/2e0)
erf84 = (1e0 - erf15)
OutFileObject = open(OutputFile,'w')
print >> OutFileObject, '# Parameter | low15 | upp84 | use Single ?'
for param in BestFitParams.keys():
if BestFitParams[param]['open']:
sort_list = sorted(mcmcData[param])
Npoints = len(mcmcData[param])
id84 = long(round(erf84*(Npoints-1)))
id15 = long(round(erf15*(Npoints-1)))
low15 = abs(BestFitParams[param]['value']-sort_list[id15])
upp84 = abs(BestFitParams[param]['value']-sort_list[id84])
if low15 >= upp84:
greater = low15
lower = upp84
else:
greater = upp84
lower = low15
useSingle = False
if 1.1*lower >= 0.9*greater:
useSingle = True
print >> OutFileObject, param+'|'+\
format(BestFitParams[param]['value'],BestFitParams[param]['printformat'])+'|'+\
format(low15,BestFitParams[param]['printformat'])+'|'+\
format(upp84,BestFitParams[param]['printformat'])+'|'+\
str(useSingle)+'|:'
else:
if param.lower() != 'reffilt':
print >> OutFileObject, param+'|'+\
format(BestFitParams[param]['value'],BestFitParams[param]['printformat'])+'|'+\
format(float('nan'),BestFitParams[param]['printformat'])+'|'+\
format(float('nan'),BestFitParams[param]['printformat'])+'|'+\
str(True)+'|:'
OutFileObject.close()
def WriteLCNUSoutlierRejection(OutFileTag,file,IngressTime,EgressTime):
""" Writes a LC and Nuisance Data file (usable by tmcmc).
Inputs
OutFileName - String with the name of the file to write the nuisance data
file - String with the name of the file with Input data (IDL data file)
Results
Nuisance data is written to OutFileName
"""
namesplit = map(str,file.split('.'))
LCFileObject = open('LIGHTCURVE.'+OutFileTag+'.data','w')
NusFileObject = open('NUISANCE.'+OutFileTag+'.data','w')
DataDictionary, HeaderData = ReadData(file)
outdata = {'time':[],'flux_ratio':[],'err_fluxratio':[]}
Nlen = len(DataDictionary['(BJD)'])
NheaderLen = len(HeaderData.keys())
LineList = {}
for i in range(Nlen):
Line = ''
outdata['time'].append(DataDictionary['JDprefix']+DataDictionary['(BJD)'][i])
if DataDictionary['f1'][i] == 0.0 or DataDictionary['f2'][i] == 0.0 or \
DataDictionary['f1'][i] < 0.0 or DataDictionary['f2'][i] < 0.0:
# if there is no flux data write 'nans'
outdata['flux_ratio'].append(float('+99'))
outdata['err_fluxratio'].append(float('+99'))
else:
# compute flux ratios
outdata['flux_ratio'].append(DataDictionary['f1'][i]/DataDictionary['f2'][i])
outdata['err_fluxratio'].append((1e0/DataDictionary['f2'][i])*\
np.sqrt(DataDictionary['erf1sq'][i] + DataDictionary['erf2sq'][i]*\
(DataDictionary['f1'][i]/DataDictionary['f2'][i])**2))
for j in range(NheaderLen):
if HeaderData[j] != '(BJD)' and HeaderData[j] != 'f1' \
and HeaderData[j] != 'f2' and HeaderData[j] != 'erf1sq' \
and HeaderData[j] != 'erf2sq' and HeaderData[j] != 'ootfg':
Line = Line+str(DataDictionary[HeaderData[j]][i])+' | '
time = str(i)
dist = np.sqrt((DataDictionary['x1'][i]-DataDictionary['x2'][i])**2 + (DataDictionary['y1'][i]-DataDictionary['y2'][i])**2)
diff_sky = DataDictionary['msky1'][i]-DataDictionary['msky2'][i]
sky_ratio1 = DataDictionary['msky1'][i]/DataDictionary['gsky'][i]
sky_ratio2 = DataDictionary['msky2'][i]/DataDictionary['gsky'][i]
Line = Line+time+' | '+str(diff_sky)+' | '+str(dist)+' | '+str(sky_ratio1)+' | '+str(sky_ratio2)
LineList[i] = Line
# normalize lightcurve
dd = np.array(outdata['flux_ratio'])/np.median(outdata['flux_ratio'])
#print np.isnan(dd)
mm, sdv, ngood, goodindex, badindex = binning.MedianMeanOutlierRejection(dd,5.0,'median')
#print mm, TT, file, np.shape(dd), np.shape(goodindex)
import pylab
t = np.array(outdata['time'])
x = np.array(outdata['flux_ratio'])
pylab.plot(t[goodindex],\
x[goodindex], 'bo')
pylab.plot(t[badindex],\
x[badindex], 'ro')
pylab.show()
normalizing_factor = mm
if np.isnan(normalizing_factor):
normalizing_factor = 1e0
for goodi in goodindex:
time = format(outdata['time'][goodi],'.9f')
fratio = format(outdata['flux_ratio'][goodi]/normalizing_factor,'.7f')
err_fratio = format(outdata['err_fluxratio'][goodi]/normalizing_factor,'.7f')
print >> LCFileObject,time+' | '+fratio+' | '+err_fratio
print >> NusFileObject,LineList[goodi]
LCFileObject.close()
NusFileObject.close()
def autocorMCMC_old(File, lowtol, jmax, OutStatFile, mkPlotsFlag, **keywords):
""" Compute the auto-correlation of parameters in a chain """
#ftag = ''
ftag = 'mcmc'
silent = False
for keyw in keywords:
if keyw == 'ftag':
ftag = keywords[keyw]
if keyw == 'Silent':
silent = keywords[keyw]
chain_stats = {}
OutFileObject = open(OutStatFile,'w')
hdrKeys = readMCMChdr(File)
for i in hdrKeys.keys():
key = hdrKeys[i]
# empty list for autocorrelation data
x = []
# skip the non-parameter data
if isNonParam(key):
pass
else:
data = read1parMCMC(File,key)
#ChainLength = len(data['istep'])
#print np.shape(data['istep'])
if not silent: print 'Par '+key
#x = np.array(data[key])
#for i in range(ChainLength):
#x.append(data[key][i])
median_x = np.median(x)
# removing median value
x1 = np.array(data[key])-median_x
cj = [] # auto-correlation
jarr = [] # lag
# starting auto-correlation, set to be much higher than tolerance
cval = 1e0
j = 0
szx1 = len(x1)
while cval > lowtol and j < jmax:
sli0 = 0
sli1 = szx1-j-1
slj0 = j
slj1 = szx1-1
next2 = x1[sli0:sli1]
x1i_x1ipj = x1[sli0:sli1]*x1[slj0:slj1]
x1i_sq = (x1[sli0:sli1]*x1[sli0:sli1])
x1i = (x1[sli0:sli1])
val = (np.mean(x1i_x1ipj) - (np.mean(x1i))**2)/\
(np.mean(x1i_sq) - (np.mean(x1i))**2)
cj.append(val)
jarr.append(j)
cval = val
j += 1
cj = np.array(cj)
jarr = np.array(jarr)
corlen_index = np.where( np.abs(cj-0.5e0) == np.min( np.abs(cj-0.5e0)) )[0]
corlen = jarr[corlen_index[0]]
efflen = long(float(ChainLength)/float(corlen))
if mkPlotsFlag:
print 'plotting acor', ftag+'.ACOR.par_'+key+'.png'
plt.plot(jarr,cj,'b.')
plt.plot([corlen,corlen],[0,1],'k--')
plt.plot([0,max(jarr)],[0.5,0.5],'k--')
plt.xlabel('j (Lag)')
plt.ylabel('C (Auto-Correlation)')
plt.title('Auto-Correlation for "'+key+'"')
plt.savefig(ftag+'.ACOR.par_'+key+'.png')
plt.clf()
print >> OutFileObject, '##'+key+'## Corr Length = '+format(corlen,'d')
print >> OutFileObject, '##'+key+'## Eff Length = '+format(efflen,'d')
chain_stats[key] = {'corlength':corlen,'efflength':efflen}
clen = []
elen = []
for key in chain_stats.keys():
clen.append(chain_stats[key]['corlength'])
elen.append(chain_stats[key]['efflength'])
if not silent:
print '## ALL ## Chain Length = '+format(ChainLength,'d')
print '## ALL ## Corr Length = '+format(max(clen),'d')
print '## ALL ## Eff Length = '+format(min(elen),'d')
print >> OutFileObject, '## ALL ## Chain Length = '+format(ChainLength,'d')
print >> OutFileObject, '## ALL ## Corr Length = '+format(max(clen),'d')
print >> OutFileObject, '## ALL ## Eff Length = '+format(min(elen),'d')
OutFileObject.close()
def autocorMCMC(File, lowtol, jmax, OutStatFile, mkPlotsFlag, **keywords):
""" Compute the auto-correlation of parameters in a chain """
#ftag = ''
ftag = 'mcmc'
silent = False
Nres = 1
Jstep = Nres
SetDynamic = False
for keyw in keywords:
if keyw == 'ftag':
ftag = keywords[keyw]
if keyw.lower() == 'silent':
silent = keywords[keyw]
if keyw.lower().startswith('res'):
Nres = keywords[keyw]
if keyw.lower().startswith('dyn'):
SetDynamic = keywords[keyw]
chain_stats = {}
OutFileObject = open(OutStatFile,'w')
hdrKeys = readMCMChdr(File)
ParList = []
for i in hdrKeys.keys():
key = hdrKeys[i]
if not isNonParam(key):
ParList.append(key)
for key in ParList:
data = read1parMCMC(File,key)
if not silent: print 'Par '+key
x1 = np.array(data[key])-np.median(data[key])
cj = [] # auto-correlation
jarr = [] # lag
# starting auto-correlation, set to be much higher than tolerance
val = 1e0
j = 0
szx1 = len(x1)
sli0 = 0
slj1 = szx1-1
while val > lowtol and j < jmax:
sli1 = szx1-j-1
slj0 = j
#t0 = time.time()
x1i_x1ipj = x1[sli0:sli1]*x1[slj0:slj1]
x1i_sq = (x1[sli0:sli1]*x1[sli0:sli1])
x1i = (x1[sli0:sli1])
mean_x1i_x1ipj = np.mean(x1i_x1ipj)
mean_x1i = np.mean(x1i)
mean_x1i_sq = np.mean(x1i_sq)
#mean_x1i_x1ipj = quickMean(x1i_x1ipj)
#mean_x1i = quickMean(x1i)
#mean_x1i_sq = quickMean(x1i_sq)
val = (mean_x1i_x1ipj - (mean_x1i)**2)/\
(mean_x1i_sq - (mean_x1i)**2)
#t1 = time.time()
if j > 0 and SetDynamic:
dc_dj = np.abs( (cj[-1]-val)/(jarr[-1]-j))
if (val > 0.45) and (val < 0.55):
scale = 0.005
else:
scale = 0.01
Jstep = long(round(scale/dc_dj))
#print t1-t0,j,val, dc_dj, 0.01/(dc_dj), Jstep
if Jstep < 1:
Jstep = 1
elif Jstep > 100:
Jstep = 100
cj.append(val)
jarr.append(j)
if SetDynamic:
j += Jstep
else:
j += Nres
cj = np.array(cj)
jarr = np.array(jarr)
ChainLength = szx1
corlen_index = np.where( np.abs(cj-0.5e0) == np.min( np.abs(cj-0.5e0)) )[0]
corlen = jarr[corlen_index[0]]
efflen = long(float(ChainLength)/float(corlen))
if mkPlotsFlag:
print 'plotting acor', ftag+'.ACOR.par_'+key+'.png'
plt.plot(jarr,cj,'b.')
plt.plot([corlen,corlen],[0,1],'k--')
plt.plot([0,max(jarr)],[0.5,0.5],'k--')
plt.xlabel('j (Lag)')
plt.ylabel('C (Auto-Correlation)')
plt.title('Auto-Correlation for "'+key+'"')
plt.savefig(ftag+'.ACOR.par_'+key+'.png')
plt.clf()
print >> OutFileObject, '##'+key+'## Corr Length = '+format(corlen,'d')
print >> OutFileObject, '##'+key+'## Eff Length = '+format(efflen,'d')
chain_stats[key] = {'corlength':corlen,'efflength':efflen}
clen = []
elen = []
for key in chain_stats.keys():
clen.append(chain_stats[key]['corlength'])
elen.append(chain_stats[key]['efflength'])
if not silent:
print '## ALL ## Chain Length = '+format(ChainLength,'d')
print '## ALL ## Corr Length = '+format(max(clen),'d')
print '## ALL ## Eff Length = '+format(min(elen),'d')
print >> OutFileObject, '## ALL ## Chain Length = '+format(ChainLength,'d')
print >> OutFileObject, '## ALL ## Corr Length = '+format(max(clen),'d')
print >> OutFileObject, '## ALL ## Eff Length = '+format(min(elen),'d')
OutFileObject.close()
def covcorStats(File, FileTag, **kwargs):
""" Given MCMC parameters, this function computes
the covariance between parameters, the pearson's correlation
coefficient and spearman's rank correlation.
"""
DFlag = False
for key in kwargs:
if key.lower().startswith('derive'):
DFlag = kwargs[key]
stats = {}
OutFileObject_COV = open(FileTag+'_COV.data','w')
OutFileObject_PR = open(FileTag+'_PEARSONSR.data','w')
OutFileObject_SR = open(FileTag+'_SPEARMANSR.data','w')
#print >> OutFileObject, '# par1 | par2 | cov | pearson\'s r | spearman\'s r '
topline = '#'
passCount = 0
hdrKeys = readMCMChdr(File)
parList1 = []
parList2 = []
for i in hdrKeys.keys():
key = hdrKeys[i]
if not isNonParam(key):
parList1.append(key)
parList2.append(key)
ComboList = []
ComboDict = {}
for par1 in parList1:
covline = ''
pcorline = ''
scorline = ''
for par2 in parList2:
t0 = time.time()
#x1 = np.arange(10)
Combo = (par1,par2)
if not Combo[::-1] in ComboList:
if par1 == par2:
d1 = read1parMCMC(File,par1,derived=DFlag)
x1 = np.array(d1[par1])
x2 = x1
else:
d1 = read1parMCMC(File,par1,derived=DFlag)
x1 = np.array(d1[par1])
d2 = read1parMCMC(File,par2,derived=DFlag)
x2 = np.array(d2[par2])
ComboList.append((par1,par2))
cov = np.cov(x1,x2)
pcor = scipy.stats.pearsonr(x1,x2)
scor = scipy.stats.spearmanr(x1,x2)
ComboDict[Combo] = {'cov':cov,'pcor':pcor,'scor':scor}
else:
cov = ComboDict[Combo[::-1]]['cov']
pcor = ComboDict[Combo[::-1]]['pcor']
scor = ComboDict[Combo[::-1]]['scor']
#x2 = np.arange(10)
t1 = time.time()
print par1, par2, t1-t0
covline = covline+' '+format(cov[0][1],'+.2e')
pcorline = pcorline+' '+format(pcor[0],'+.2e')
scorline = scorline+' '+format(scor[0],'+.2e')
if passCount == 0: topline = topline+5*' '+par2
if passCount == 0:
print >> OutFileObject_COV, topline
print >> OutFileObject_COV, '#'+88*'-'
print >> OutFileObject_PR, topline
print >> OutFileObject_PR, '#'+88*'-'
print >> OutFileObject_SR, topline
print >> OutFileObject_SR, '#'+88*'-'
lenkey = len(par1)
if lenkey < 10:
fac = 10-lenkey
else:
fac = 10
print >> OutFileObject_COV, par1+fac*' '+' | '+covline
print >> OutFileObject_PR, par1+fac*' '+' | '+pcorline
print >> OutFileObject_SR, par1+fac*' '+' | '+scorline
passCount += 1
OutFileObject_COV.close()
OutFileObject_PR.close()
OutFileObject_SR.close()
def RunMinuit(FunctionName,ObservedData,ModelParams,NuisanceData,BoundParams,tolnum,OutFile):
""" Designed to run Minuit on tmcmc format Data dictionaries """
OpenParNames = []
for key in ModelParams.keys():
if ModelParams[key]['open']:
OpenParNames.append(key)
startchi2 = f_chisquared(FunctionName,ObservedData,ModelParams,NuisanceData,BoundParams)
dumpfile1 = open('dump1','wb')
dumpfile2 = open('dump2','wb')
dumpfile3 = open('dump3','wb')
dumpfile4 = open('dump4','wb')
dumpfile5 = open('dump5','wb')
dumpfile6 = open('dump6','wb')
pickle.dump(FunctionName,dumpfile1,-1)
pickle.dump(ObservedData,dumpfile2,-1)
pickle.dump(ModelParams,dumpfile3,-1)
pickle.dump(NuisanceData,dumpfile4,-1)
pickle.dump(BoundParams,dumpfile5,-1)
pickle.dump(OpenParNames,dumpfile6,-1)
dumpfile1.close()
dumpfile2.close()
dumpfile3.close()
dumpfile4.close()
dumpfile5.close()
dumpfile6.close()
# Begin running Minuit
converge = False
CountA1 = 0
CountFail = 0
AbsoluteFail = False
while not converge:
if CountFail > 20:
print 'Failed'
AbsoluteFail = True
break
try:
m = minuit2.Minuit2(transit_chisquared(OpenParNames))
m.tol = tolnum #55.4720096 #*1e8
m.up = 1
m.strategy = 2
for key in m.values.keys():
m.values[key] = ModelParams[key]['value']
m.errors[key] = ModelParams[key]['step']
m.migrad()
#print 'edm = ',m.edm, '(edm < 1e-3 signifies convergence, edm > 1e2 is probably too high)'
#print m.edm, converge, tolnum, 1e-3*m.tol*m.up, m.ncalls
tolStatus = m.edm/(1e-3*m.tol*m.up)
if tolStatus <= 1e0:
if tolStatus == 1e0:
CountA1 = 100
CountA1 += 1
if CountA1 <= 4:
converge = False
else:
converge = True
tolnum *= tolStatus
#CloseDict['a1'] = tolnum
w = 'a1 Good Convergence'
else:
converge = False
tolnum *= tolStatus
w = 'a2 Poor Convergence'
if np.isnan(tolStatus):
CountFail += 1
print tolStatus, m.edm, m.tol, tolnum, converge, w, CountFail
except:
if m.edm != None:
edm = m.edm
else:
edm = 1e-2*m.tol*m.up
if CountFail < 11:
tolStatus = (edm/(1e-3*m.tol*m.up))
elif CountFail > 11:
tolStatus = (edm/(1e-3*m.tol*m.up))**(-1)
else:
tolStatus = 10e0/tolnum
if tolStatus == 1.0:
tolStatus = 10e0
tolnum *= tolStatus
converge = False
CountFail += 1
print tolStatus, m.edm, m.tol, tolnum, converge, 'b Failed Convergence', CountFail
#raise
for key in m.values.keys():
ModelParams[key]['value'] = m.values[key]
ModelParams[key]['step'] = m.errors[key]
if not AbsoluteFail:
DOF = len(ObservedData['all']['y']) - len(OpenParNames)
PrintModelParams(ModelParams,OutFile)
OutFileObjectAppend = open(OutFile,'a')
print >> OutFileObjectAppend, \
'# Best-fit ChiSQ = '+format(m.fval,'.2f')+' | Starting ChiSQ = ',format(startchi2,'.2f')
print >> OutFileObjectAppend, '# DOF = '+format(DOF)
print >> OutFileObjectAppend, \
'# Best-fit reduced ChiSQ = '+format(m.fval/DOF,'.2f')+' | Starting reduced ChiSQ = '+format(startchi2/DOF,'.2f')
OutFileObjectAppend.close()
def statsTable(self,Stats,stype, **kwargs):
width, height = matplotlib.rcParams['figure.figsize']
size = max([width,height])
# make a square figure
fig = plt.figure(figsize=(size,size))
# default font sizes
xfsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
yfsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
fsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
yshift = 0
xshift = 0
PlotFile = False
TitleString = ''
#print xfsize,yfsize,fsize
if fsize > 24.0:
fsize = 24.0
if xfsize > 18.0:
xfsize = 18.0
if yfsize > 18.0:
yfsize = 18.0
for key in kwargs:
if key.lower().startswith('plotfile'):
PlotFile = True
FileName = kwargs[key]
elif key.lower().startswith('numfsize'):
fsize = kwargs[key]
elif key.lower().startswith('fsizex'):
xfsize = kwargs[key]
elif key.lower().startswith('fsizey'):
yfsize = kwargs[key]
elif key.lower().startswith('xshift'):
xshift = kwargs[key]
elif key.lower().startswith('yshift'):
yshift = kwargs[key]
elif key.lower().startswith('title'):
TitleString = kwargs[key]
else:
pass
for grid in self.GridDict.keys():
plotID = subID(grid,self.GridNX,self.GridNY)
plt.subplot(self.GridNY,self.GridNX,plotID)
Stat = Stats[stype]\
[self.GridDict[grid][0]]\
[self.GridDict[grid][1]]\
['value']
absStat = abs(Stat)
cross = np.linspace(0,1,3)
plt.plot(cross,cross,marker='o',\
markerfacecolor='w',\
markeredgecolor='w',\
linestyle='None')
#numsplit = map(str,text.split('e'))
#OutText = r'%s$\times 10^{%s}$' % (numsplit[0],numsplit[1])
if absStat < 0.01:
text = r'$< 0.01$'
OutText = r'%s' % text
plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize)
elif absStat >= 0.5:
text = format(abs(Stat),'0.2f')
plt.subplot(self.GridNY,self.GridNX,plotID, axisbg='gray')
OutText = r'%s' % text
plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize,color='w')
else:
text = format(abs(Stat),'0.2f')
plt.subplot(self.GridNY,self.GridNX,plotID)
OutText = r'%s' % text
plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize)
plt.setp(plt.gca(),yticklabels=[],yticks=[])
plt.setp(plt.gca(),xticklabels=[],xticks=[])
if grid[1] == 0:
#print self.Label[grid]['x'], grid
plt.xlabel(self.Label[grid]['x'],fontsize=xfsize)
if grid[0] == 0:
#print self.Label[grid]['y'], grid
plt.ylabel(self.Label[grid]['y'],fontsize=yfsize)
plt.subplots_adjust(hspace=0)
plt.subplots_adjust(wspace=0)
plt.suptitle(TitleString,fontsize=24,y=0.95)
if PlotFile:
plt.savefig(FileName,pad_inches=0.5)
else:
plt.show()
def returnDerivedLine_MTQ2011(ModelParams,istep,keyList0):
""" print a file with Derived parameters from the MCMC ensemble. """
derived = {}
Period = computePeriod(ModelParams)
derived['Period'] = {'value':Period,'printformat':'.9f'}
RefFilt = ModelParams['RefFilt']['printformat']
# get Filter and Transit time tags
Tags = getTags(ModelParams)
# compute parameters used to compute lightcurve using the reference filter
Dref, v1ref, v2ref = MTQ_2011.MTQ_FilterParams(RefFilt,Tags,ModelParams)
u1ref, u2ref = LDC_v2u(v1ref,v2ref)
tT = ModelParams['tT']['value']
tG = ModelParams['tG']['value']
TransitRef = MTQ_2011.MTQ_getDerivedParams(Dref,tT,tG,u1ref,u2ref,Period)
#print TransitRef
derived['inc'] = {'value':TransitRef['inc']*180e0/np.pi,'printformat':'.4f'}
derived['b'] = {'value':TransitRef['b'],'printformat':'.6f'}
derived['aRs'] = {'value':TransitRef['aRs'],'printformat':'.6f'}
derived['velRs'] = {'value':TransitRef['velRs'],'printformat':'.6f'}
derived['rho_star'] = {'value':TransitRef['rho_star'],'printformat':'.6f'}
derived['RpRs'+'.'+ModelParams['RefFilt']['printformat']] = \
{'value':TransitRef['RpRs'],'printformat':'.9f'}
for key in ModelParams.keys():
if key.startswith('T0'):
split_TT = map(str,key.split('.'))
transit_tag = split_TT[1].strip()
# compute D, v1, v2 and then u1 and u2 for a given transit tag
D, v1, v2 = MTQ_2011.MTQ_FilterParams(transit_tag,Tags,ModelParams)
u1, u2 = LDC_v2u(v1,v2)
RpRs = computeRpRs(u1,u2,tT,tG,D)
filterD = filterMatchD(transit_tag,Tags,ModelParams)
#print filterD
derived['RpRs'+'.'+filterD] = {'value':RpRs,'printformat':'.9f'}
DerivedLine = ''
if istep == 0:
keylist = derived.keys()
else:
keylist = keyList0
Nkeys = len(keylist)
if istep == 0:
keyorder = {}
for i in range(len(keylist)):
keyorder[i] = keylist[i]
for i in range(Nkeys):
if i == 0:
DerivedLine =\
str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
DerivedLine =\
DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
for i in range(Nkeys):
if i == 0:
DerivedLine =\
str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
DerivedLine = DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
DerivedLine = DerivedLine+'|'+str(format(istep,'.0f'))+'|:'
return DerivedLine, keylist
def printDerived_MTQ_2011(STARTFILE,MCMCfile,DerivedFile):
""" print a file with Derived parameters from the MCMC ensemble. """
hdrKeys = readMCMChdr(MCMCfile)
ModelParams = ReadStartParams(STARTFILE)
mcmcFile = open(MCMCfile,'r')
mcmcFile = mcmcFile.readlines()
OutFileObject = open(DerivedFile,'w')
for line in mcmcFile:
derived = {}
if not line.startswith('#'):
data_line = ReadMCMCline(line,hdrKeys)
print 'step/line = ',format(long(data_line['istep']),'n')
for key in ModelParams.keys():
if ModelParams[key]['open']:
ModelParams[key]['value'] = data_line[key]
Period = computePeriod(ModelParams)
derived['Period'] = {'value':Period,'printformat':'.9f'}
RefFilt = ModelParams['RefFilt']['printformat']
# get Filter and Transit time tags
Tags = getTags(ModelParams)
# compute parameters used to compute lightcurve using the reference filter
Dref, v1ref, v2ref = MTQ_2011.MTQ_FilterParams(RefFilt,Tags,ModelParams)
u1ref, u2ref = LDC_v2u(v1ref,v2ref)
tT = ModelParams['tT']['value']
tG = ModelParams['tG']['value']
TransitRef = MTQ_2011.MTQ_getDerivedParams(Dref,tT,tG,u1ref,u2ref,Period)
#print TransitRef
derived['inc'] = {'value':TransitRef['inc']*180e0/np.pi,'printformat':'.4f'}
derived['b'] = {'value':TransitRef['b'],'printformat':'.6f'}
derived['aRs'] = {'value':TransitRef['aRs'],'printformat':'.6f'}
derived['velRs'] = {'value':TransitRef['velRs'],'printformat':'.6f'}
derived['rho_star'] = {'value':TransitRef['rho_star'],'printformat':'.6f'}
derived['RpRs'+'.'+ModelParams['RefFilt']['printformat']] = \
{'value':TransitRef['RpRs'],'printformat':'.9f'}
for key in ModelParams.keys():
if key.startswith('T0'):
split_TT = map(str,key.split('.'))
transit_tag = split_TT[1].strip()
# compute D, v1, v2 and then u1 and u2 for a given transit tag
D, v1, v2 = MTQ_2011.MTQ_FilterParams(transit_tag,Tags,ModelParams)
u1, u2 = LDC_v2u(v1,v2)
RpRs = computeRpRs(u1,u2,tT,tG,D)
filterD = filterMatchD(transit_tag,Tags,ModelParams)
#print filterD
derived['RpRs'+'.'+filterD] = {'value':RpRs,'printformat':'.9f'}
DerivedLine = ''
keylist = derived.keys()
Nkeys = len(keylist)
if data_line['istep'] == 0:
keyorder = {}
for i in range(len(keylist)):
keyorder[i] = keylist[i]
for i in range(Nkeys):
if i == 0:
DerivedLine =\
str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
DerivedLine =\
DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
for i in range(Nkeys):
if i == 0:
DerivedLine =\
str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
else:
DerivedLine = DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
derived[keylist[i]]['printformat']))
DerivedLine = DerivedLine+'|'+str(format(data_line['istep'],'.0f'))+'|:'
print >> OutFileObject, DerivedLine
keyline = ''
for i in range(Nkeys):
if i == 0:
keyline = '#'+keylist[i]
else:
keyline = keyline+'|'+keylist[i]
keyline = keyline+'|istep|:'
OutFileObject.close()
#print >> OutFileObject, keyline
opencopy = open(DerivedFile,'r')
opencopy = opencopy.readlines()
os.system('rm -v %s' % (DerivedFile))
rearranged = open(DerivedFile,'w')
print >> rearranged, keyline
for iline in range(len(opencopy)):
print >> rearranged, opencopy[iline].strip('\n')
rearranged.close()