Example #1
0
def cropMCMC_old(mcmcfile,outfile,cropperc):
    """ Removes the "burn-in" phase of the MCMC and writes a 
    shortened MCMC data file 
    """
    
    hdrkeys = readMCMChdr(mcmcfile)
    Nparams = getNparams(hdrkeys)

    outfileObject = open(outfile,'w')
    mcmcfileobj = open(mcmcfile,'r')
    mcmcfileobj = mcmcfileobj.readlines()
    for line in mcmcfileobj:
        if line.startswith('#'):
            print >> outfileObject, line.strip('\n')
        if not line.startswith('#'):
            data_line = ReadMCMCline(line,hdrkeys)
            if Nparams == 1:
                if data_line['acr'] > 0.44-cropperc and data_line['acr'] < 0.44+cropperc:
                    print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
                    print >> outfileObject, line.strip('\n')
            if Nparams > 1:
                if data_line['acr'] > 0.23-cropperc and data_line['acr'] < 0.23+cropperc:
                    print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
                    print >> outfileObject, line.strip('\n')

    outfileObject.close()
Example #2
0
def cropMCMC(mcmcfile,outfile,cropperc,NumCropLine,**kwargs):
    """ Removes the "burn-in" phase of the MCMC and writes a 
    shortened MCMC data file 
    """
    silent = True
    for key in kwargs:
        if key.lower().startswith('sil'):
            silent = kwargs[key]
    
    hdrkeys = readMCMChdr(mcmcfile)
    Nparams = getNparams(hdrkeys)

    outfileObject = open(outfile,'w')
    mcmcfileobj = open(mcmcfile,'r')
    mcmcfileobj = mcmcfileobj.readlines()
    
    NLine = 0
    for line in mcmcfileobj:
        NLine += 1
        if line.startswith('#'):
            print >> outfileObject, line.strip('\n')
        
        if NLine > NumCropLine:
            if not line.startswith('#'):
                data_line = ReadMCMCline(line,hdrkeys)
                if Nparams == 1:
                    if data_line['acr'] > 0.44-cropperc and data_line['acr'] < 0.44+cropperc:
                        if not silent: print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
                        print >> outfileObject, line.strip('\n')
                if Nparams > 1:
                    if data_line['acr'] > 0.23-cropperc and data_line['acr'] < 0.23+cropperc:
                        if not silent: print 'printing ',format(data_line['istep'],'n'),' acr ',data_line['acr']
                        print >> outfileObject, line.strip('\n')

    outfileObject.close()
Example #3
0
def makeStartFromExplore(ListOfChains,StablePerc,SampleParamFile,OutputParamFile):
    """
        Reads a list of single parameter step-exoploration chains 
        and prints a startparam file based on this data.
        This routine also prints whether a chain has stabilized
        the correct single-parameter acceptance rate.
    """
    
    ListFiles = open(ListOfChains,'r')
    ListFiles = ListFiles.readlines()
    
    ModelParams = ReadStartParams(SampleParamFile)
    step = {}
    for file in ListFiles:
        data = readMCMC(file.strip('\n'))
        if data['acr'][-1] > 0.44-StablePerc and data['acr'][-1] < 0.44+StablePerc:
            for par in data.keys():
                if not isNonParam(par):
                    medfrac = np.median(data['frac'][-1000:])
                    print par+' has stabilized, acr = '+format(data['acr'][-1],'.2f')+' frac = '+str(medfrac)
                    step[par] = {'frac':medfrac}
        else:
            for par in data.keys():
                if not isNonParam(par):
                    print par+' has NOT stabilized, acr = '+format(data['acr'][-1],'.2f')

    for par in step.keys():
        for key in ModelParams.keys():
            if key == par:
                ModelParams[par]['step'] = ModelParams[par]['step']*step[par]['frac']
                ModelParams[par]['open'] = True

    PrintModelParams(ModelParams,OutputParamFile)
Example #4
0
def TableEntry(errEntry,par,fStr):

    Entry = 'Wrong'

    if par == 'Period':
        Val = format(errEntry[par]['value'],fStr)
    elif par.startswith('T0'):
        Val = format(errEntry[par]['value'],fStr)
    else:
        Val = format(errEntry[par]['value'],fStr)

    if np.isnan(errEntry[par]['lower']) or \
       np.isnan(errEntry[par]['upper']):
        Entry = '('+format(errEntry[par]['value'],fStr)+')'
    elif (errEntry[par]['lower'] != errEntry[par]['upper']) and \
        not errEntry[par]['useSingle']:
        Upp = format(errEntry[par]['upper'],fStr)
        Low = format(errEntry[par]['lower'],fStr)
        if float(Upp) == 0e0 or float(Low) == 0e0:
            maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
            errPart = '$\pm$'+format(maxerr,fStr) 
            Entry = Val+errPart
        elif float(Upp) == float(Low):
            maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
            errPart = '$\pm$'+format(maxerr,fStr) 
            Entry = Val+errPart
        else:
            errPart = '$^{+'+Upp+'}_{-'+Low+'}$'
            Entry = Val+errPart
    else:
        maxerr = max([errEntry[par]['upper'],errEntry[par]['lower']])
        errPart = '$\pm$'+format(maxerr,fStr)
        Entry = Val+errPart

    return Entry
Example #5
0
def WriteLightCurveFile(OutFileName,file,IngressTime,EgressTime):
    """ Writes a Lightcurve file (usable by MCMC), normalizing the out-of-transit 
    lightcurve to 1.
    Inputs
        OutFileName - String with the name of the file to write the lightcurve
        file - String with the name of the file with Input data (IDL data file)
        IngressTime - Float with an approximate guess of the onset of Ingress
        EgressTime - Float with an approximate guess of the end of Egress
    Result - Lightcurve with out-of-eclipse data normalized to 1 is written to OutFileName
    """

    DataDictionary, HeaderData = ReadData(file)
    FileObject = open(OutFileName,'w')
    
    Nlen = len(DataDictionary['(BJD)']) 
    outdata = {'time':[],'flux_ratio':[],'err_fluxratio':[]}
    
    for i in range(Nlen):
        outdata['time'].append(DataDictionary['JDprefix']+DataDictionary['(BJD)'][i])
        if DataDictionary['f1'][i] == 0.0 or DataDictionary['f2'][i] == 0.0 or \
        DataDictionary['f1'][i] < 0.0 or DataDictionary['f2'][i] < 0.0:
            # if there is no flux data write 'nans'
            outdata['flux_ratio'].append(float('nan'))
            outdata['err_fluxratio'].append(float('nan'))
        else:
            # compute flux ratios
            outdata['flux_ratio'].append(DataDictionary['f1'][i]/DataDictionary['f2'][i])
            outdata['err_fluxratio'].append((1e0/DataDictionary['f2'][i])*\
            np.sqrt(DataDictionary['erf1sq'][i] + DataDictionary['erf2sq'][i]*\
            (DataDictionary['f1'][i]/DataDictionary['f2'][i])**2))

    flat_flux = []
    for i in range(len(outdata['time'])):
        if outdata['time'][i] < IngressTime or outdata['time'][i] > EgressTime:
            flat_flux.append(outdata['flux_ratio'][i])

    # normalize lightcurve
    normalizing_factor = np.median(np.array(flat_flux))
    print normalizing_factor
    if np.isnan(normalizing_factor):
        normalizing_factor = 1e0

    print >> FileObject, '# TDB | flux_ratio | err_fluxratio'
    for i in range(len(outdata['time'])):
        time = format(outdata['time'][i],'.9f')
        fratio = format(outdata['flux_ratio'][i]/normalizing_factor,'.7f')
        err_fratio = format(outdata['err_fluxratio'][i]/normalizing_factor,'.7f')
        print >> FileObject, time+' | '+fratio+' | '+err_fratio
    
    FileObject.close()
Example #6
0
def printMCMCline(param_tagorder,ModelParams,istep,frac,acr,chi1,chi2):
    """ Given ModelParams, the parameter order and other simulation parameters writes lines for the MCMC file. """

    line = ''
        
    Nparams = len(param_tagorder.keys())
    # print param_tagorder
    for el in range(Nparams):
        if el == 0:
            line =\
            str(format(ModelParams[param_tagorder[el]]['value'],\
            ModelParams[param_tagorder[el]]['printformat']))
        if el > 0:
            line = line+'|'+format(ModelParams[param_tagorder[el]]['value'],\
            ModelParams[param_tagorder[el]]['printformat'])
    line = line+'|'+str(format(istep,'.0f'))+'|'+str(frac)+'|'+str(acr)+\
    '|'+str(format(chi1,'.4f'))+'|'+str(format(chi2,'.4f'))+'|:'

    return line
Example #7
0
def PrintModelParams(ModelParams,OutFile):
    """ Given the dictionary of Model parameters, this routine write a file in the format of the 
        starting parameter files
    """

    OutFileObject = open(OutFile,'w')

    print >> OutFileObject, '# Parname | value | step | open | printformat '
    for param in ModelParams.keys():
        if param == 'RefFilt':
            print >> OutFileObject, param+'  |  '+str(ModelParams[param]['value'])+' | '\
            +str(ModelParams[param]['step'])+' | '\
            +str(ModelParams[param]['open'])+' | '+str(ModelParams[param]['printformat'])
        else:
            print >> OutFileObject, param+'  |  '+str(format(ModelParams[param]['value'],ModelParams[param]['printformat']))+' | '\
            +str(format(ModelParams[param]['step'],ModelParams[param]['printformat']))+' | '\
            +str(ModelParams[param]['open'])+' | '+str(ModelParams[param]['printformat'])

    OutFileObject.close()
    return
Example #8
0
def MinuitPar2Err(ParFile,ErrFile):
    """ Convert a Minuit par file to an error file """
    
    ModelParams = ReadStartParams(ParFile)
    OutFile = open(ErrFile,'w')
    header = '# parname     |    value   | lower   | upper    | useSingle ?'
    print >> OutFile, header

    for par in ModelParams.keys():
        if par.lower() != 'reffilt':
            if ModelParams[par]['open']:
                ValString = format(ModelParams[par]['value'],ModelParams[par]['printformat'])
                ErrString = format(ModelParams[par]['step'],ModelParams[par]['printformat'])
                print >> OutFile, par+'|'+ValString+'|'+ErrString+'|'+ErrString+'| True '
            else:
                ValString = format(ModelParams[par]['value'],ModelParams[par]['printformat'])
                ErrString = 'nan'
                print >> OutFile, par+'|'+ValString+'|'+ErrString+'|'+ErrString+'| False '

    OutFile.close()
Example #9
0
def makeStatLabels(Stats,DataFile):
    """     """
    
    parList = getPars(DataFile)
    Npar = len(parList)
    iplot = 1
    
    StatLabels = {}

    for iy in range(Npar):
        for ix in range(Npar):
            if not ix >= iy:
                parName1 = parList[ix]
                parName2 = parList[iy]
                cov = r'$|\sigma_{(x,y)}|$='+\
                format(abs(Stats['cov'][parName1][parName2]['value']),'0.2f')
                spe = r'$|\rho|$='+\
                format(abs(Stats['spear'][parName1][parName2]['value']),'0.2f')
                pea = r'$|r|$='+\
                format(abs(Stats['pear'][parName1][parName2]['value']),'0.2f')
                StatLabels[iplot] = {'cov':cov,'spe':spe,'pea':pea}
Example #10
0
def WriteLowestChisq(file,ModelParams,OutFileName,ShowOutput):
    """ Finds the lowest chisq point in MCMC and prints to a file
        of format similar to the start paramfile.
        Inputs 
            file - the MCMC file
            ModelPars
            OutFileName
        Output
            a file with OutFileName
    """
    
    ModelParamsCopy = {}
    
    if checkFileExists(file):
        FileObject = open(file,'r')
        FileObjectLines = FileObject.readlines()
        minchi = 1e308
        for line in FileObjectLines:
            if line.startswith('#'):
                ParNames = ReadHeaderMCMC(line)
            else:
                if line.endswith("|:\n"):
                    par0 = ReadMCMCline(line,ParNames)
                    if par0['chi1'] < minchi:
                        minchi = par0['chi1']
                        if ShowOutput: print 'Lowest chisq = ',format(minchi,'.2f'),\
                        ' @ step = ',format(par0['istep'],'n')
                        for key in ModelParams.keys():
                            if ModelParams[key]['open']:
                                ModelParamsCopy[key] = {'value':par0[key],'step':ModelParams[key]['step'],\
                                'printformat':ModelParams[key]['printformat'],'open':ModelParams[key]['open']}
                            else:
                                ModelParamsCopy[key] = {'value':ModelParams[key]['value'],'step':\
                                ModelParams[key]['step'],'printformat':ModelParams[key]\
                                ['printformat'],'open':ModelParams[key]['open']}
                    else:
                        pass
        PrintModelParams(ModelParamsCopy,OutFileName)
    else:
        print file, ' does not exist.'
Example #11
0
def getEntryString(value,lower,upper,useSingle,parformat):
    """                     """
    
    if useSingle:
        errVal = np.max([lower,upper])
        errStr = format(errVal,parformat)
        ValStr = format(value,parformat)
        EntryStr = r'%s $\pm$ %s' % (ValStr,errStr)
    else:
        if lower == upper:
            errVal = np.max([lower,upper])
            errStr = format(errVal,parformat)
            ValStr = format(value,parformat)
            EntryStr = r'%s $\pm$ %s' % (ValStr,errStr)
        else:
            errUpStr = format(upper,parformat)
            errLowStr = format(lower,parformat)
            ValStr = format(value,parformat)
            EntryStr = r'%s$^{+%s}_{-%s}$' % (ValStr,errUpStr,errLowStr)

    return EntryStr
Example #12
0
def printErrors(MCMCfile,BestfitFile,OutputFile):
    """ Gets data out from the MCMC data file and the BESTFIT parameters file and prints uncertainties """

    mcmcData = readMCMC(MCMCfile)
    BestFitParams = ReadStartParams(BestfitFile)
    
    erf15 = ((1e0 - scipy.special.erf(1e0/np.sqrt(2e0)))/2e0)
    erf84 = (1e0 - erf15)
    
    OutFileObject = open(OutputFile,'w')
    print >> OutFileObject, '#  Parameter   |  low15   | upp84  | use Single ?'
    for param in BestFitParams.keys():
        if BestFitParams[param]['open']:
            sort_list = sorted(mcmcData[param])
            Npoints = len(mcmcData[param])
            id84 = long(round(erf84*(Npoints-1)))
            id15 = long(round(erf15*(Npoints-1)))
            low15 = abs(BestFitParams[param]['value']-sort_list[id15])
            upp84 = abs(BestFitParams[param]['value']-sort_list[id84])
            if low15 >= upp84:
                greater = low15
                lower = upp84
            else:
                greater = upp84
                lower = low15
            useSingle = False
            if 1.1*lower >= 0.9*greater:
                useSingle = True
            print >> OutFileObject, param+'|'+\
            format(BestFitParams[param]['value'],BestFitParams[param]['printformat'])+'|'+\
            format(low15,BestFitParams[param]['printformat'])+'|'+\
            format(upp84,BestFitParams[param]['printformat'])+'|'+\
            str(useSingle)+'|:'
        else:
            if param.lower() != 'reffilt':
                print >> OutFileObject, param+'|'+\
                format(BestFitParams[param]['value'],BestFitParams[param]['printformat'])+'|'+\
                format(float('nan'),BestFitParams[param]['printformat'])+'|'+\
                format(float('nan'),BestFitParams[param]['printformat'])+'|'+\
                str(True)+'|:'
    OutFileObject.close()
Example #13
0
def WriteLCNUSoutlierRejection(OutFileTag,file,IngressTime,EgressTime):
    """ Writes a LC and Nuisance Data file (usable by tmcmc).
    Inputs
        OutFileName - String with the name of the file to write the nuisance data
        file - String with the name of the file with Input data (IDL data file) 

    Results
        Nuisance data is written to OutFileName
    """
    
    namesplit = map(str,file.split('.'))
    LCFileObject = open('LIGHTCURVE.'+OutFileTag+'.data','w')
    NusFileObject = open('NUISANCE.'+OutFileTag+'.data','w')
    DataDictionary, HeaderData = ReadData(file)
    outdata = {'time':[],'flux_ratio':[],'err_fluxratio':[]}
    
    Nlen = len(DataDictionary['(BJD)'])
    NheaderLen = len(HeaderData.keys())
    LineList = {}
    for i in range(Nlen):
        Line = ''
        outdata['time'].append(DataDictionary['JDprefix']+DataDictionary['(BJD)'][i])
        if DataDictionary['f1'][i] == 0.0 or DataDictionary['f2'][i] == 0.0 or \
        DataDictionary['f1'][i] < 0.0 or DataDictionary['f2'][i] < 0.0:
            # if there is no flux data write 'nans'
            outdata['flux_ratio'].append(float('+99'))
            outdata['err_fluxratio'].append(float('+99'))
        else:
            # compute flux ratios
            outdata['flux_ratio'].append(DataDictionary['f1'][i]/DataDictionary['f2'][i])
            outdata['err_fluxratio'].append((1e0/DataDictionary['f2'][i])*\
            np.sqrt(DataDictionary['erf1sq'][i] + DataDictionary['erf2sq'][i]*\
            (DataDictionary['f1'][i]/DataDictionary['f2'][i])**2))
        for j in range(NheaderLen):
            if HeaderData[j] != '(BJD)' and HeaderData[j] != 'f1' \
            and HeaderData[j] != 'f2' and HeaderData[j] != 'erf1sq' \
            and HeaderData[j] != 'erf2sq' and HeaderData[j] != 'ootfg':
                Line = Line+str(DataDictionary[HeaderData[j]][i])+' | '
        time = str(i)
        dist = np.sqrt((DataDictionary['x1'][i]-DataDictionary['x2'][i])**2 + (DataDictionary['y1'][i]-DataDictionary['y2'][i])**2)
        diff_sky = DataDictionary['msky1'][i]-DataDictionary['msky2'][i]
        sky_ratio1 = DataDictionary['msky1'][i]/DataDictionary['gsky'][i]
        sky_ratio2 = DataDictionary['msky2'][i]/DataDictionary['gsky'][i]
        Line = Line+time+' | '+str(diff_sky)+' | '+str(dist)+' | '+str(sky_ratio1)+' | '+str(sky_ratio2)
        LineList[i] = Line
    
    # normalize lightcurve
    dd = np.array(outdata['flux_ratio'])/np.median(outdata['flux_ratio'])
    #print np.isnan(dd)
    mm, sdv, ngood, goodindex, badindex = binning.MedianMeanOutlierRejection(dd,5.0,'median')
    #print mm, TT, file, np.shape(dd), np.shape(goodindex)
    import pylab
    t = np.array(outdata['time'])
    x = np.array(outdata['flux_ratio'])
    pylab.plot(t[goodindex],\
    x[goodindex], 'bo')
    pylab.plot(t[badindex],\
    x[badindex], 'ro')
    pylab.show()
    normalizing_factor = mm
    if np.isnan(normalizing_factor):
        normalizing_factor = 1e0

    for goodi in goodindex:
        time = format(outdata['time'][goodi],'.9f')
        fratio = format(outdata['flux_ratio'][goodi]/normalizing_factor,'.7f')
        err_fratio = format(outdata['err_fluxratio'][goodi]/normalizing_factor,'.7f')
        print >> LCFileObject,time+' | '+fratio+' | '+err_fratio
        print >> NusFileObject,LineList[goodi]
    LCFileObject.close()
    NusFileObject.close()
Example #14
0
def autocorMCMC_old(File, lowtol, jmax, OutStatFile, mkPlotsFlag, **keywords):
    """ Compute the auto-correlation of parameters in a chain """
    
    #ftag = ''
    ftag = 'mcmc'
    silent = False
    for keyw in keywords:
        if keyw == 'ftag':
            ftag = keywords[keyw]
        if keyw == 'Silent':
            silent = keywords[keyw]
    
    chain_stats = {}
    OutFileObject = open(OutStatFile,'w')
    hdrKeys = readMCMChdr(File)
    for i in hdrKeys.keys():
        key = hdrKeys[i]
        # empty list for autocorrelation data
        x = []
        # skip the non-parameter data
        if isNonParam(key):
            pass
        else:
            data = read1parMCMC(File,key)
            #ChainLength = len(data['istep'])
            #print np.shape(data['istep'])
            if not silent: print 'Par '+key
            #x = np.array(data[key])
            #for i in range(ChainLength):
                #x.append(data[key][i])
            median_x = np.median(x)
            # removing median value
            x1 = np.array(data[key])-median_x
            cj = []             # auto-correlation
            jarr = []           # lag
            # starting auto-correlation, set to be much higher than tolerance
            cval = 1e0
            j = 0
            szx1 = len(x1)
            while cval > lowtol and j < jmax:
                sli0 = 0 
                sli1 = szx1-j-1
                slj0 = j
                slj1 = szx1-1
                next2 = x1[sli0:sli1]
                x1i_x1ipj = x1[sli0:sli1]*x1[slj0:slj1]
                x1i_sq = (x1[sli0:sli1]*x1[sli0:sli1])
                x1i = (x1[sli0:sli1])
                val = (np.mean(x1i_x1ipj) - (np.mean(x1i))**2)/\
                      (np.mean(x1i_sq) - (np.mean(x1i))**2)
                cj.append(val)
                jarr.append(j)
                cval = val
                j += 1
            cj = np.array(cj)
            jarr = np.array(jarr)
            corlen_index = np.where( np.abs(cj-0.5e0) == np.min( np.abs(cj-0.5e0)) )[0]
            corlen = jarr[corlen_index[0]]
            efflen = long(float(ChainLength)/float(corlen))
            if mkPlotsFlag:
                print 'plotting acor', ftag+'.ACOR.par_'+key+'.png'
                plt.plot(jarr,cj,'b.')
                plt.plot([corlen,corlen],[0,1],'k--')
                plt.plot([0,max(jarr)],[0.5,0.5],'k--')
                plt.xlabel('j (Lag)')
                plt.ylabel('C (Auto-Correlation)')
                plt.title('Auto-Correlation for "'+key+'"')
                plt.savefig(ftag+'.ACOR.par_'+key+'.png')
                plt.clf()

            print >> OutFileObject, '##'+key+'## Corr Length = '+format(corlen,'d')
            print >> OutFileObject, '##'+key+'## Eff Length = '+format(efflen,'d')
            chain_stats[key] = {'corlength':corlen,'efflength':efflen}
    
    clen = []
    elen = []
    for key in chain_stats.keys():
        clen.append(chain_stats[key]['corlength'])
        elen.append(chain_stats[key]['efflength'])
    if not silent:
        print '## ALL ## Chain Length = '+format(ChainLength,'d')
        print '## ALL ## Corr Length = '+format(max(clen),'d')
        print '## ALL ## Eff Length = '+format(min(elen),'d')
    print >> OutFileObject, '## ALL ## Chain Length = '+format(ChainLength,'d')
    print >> OutFileObject, '## ALL ## Corr Length = '+format(max(clen),'d')
    print >> OutFileObject, '## ALL ## Eff Length = '+format(min(elen),'d')
    OutFileObject.close()
Example #15
0
def autocorMCMC(File, lowtol, jmax, OutStatFile, mkPlotsFlag, **keywords):
    """ Compute the auto-correlation of parameters in a chain """
    
    #ftag = ''
    ftag = 'mcmc'
    silent = False
    Nres = 1
    Jstep = Nres
    SetDynamic = False
    for keyw in keywords:
        if keyw == 'ftag':
            ftag = keywords[keyw]
        if keyw.lower() == 'silent':
            silent = keywords[keyw]
        if keyw.lower().startswith('res'):
            Nres = keywords[keyw]
        if keyw.lower().startswith('dyn'):
            SetDynamic = keywords[keyw]
            
    chain_stats = {}
    OutFileObject = open(OutStatFile,'w')
    hdrKeys = readMCMChdr(File)
    
    ParList = []
    for i in hdrKeys.keys():
        key = hdrKeys[i]
        if not isNonParam(key):
            ParList.append(key)

    for key in ParList:
        data = read1parMCMC(File,key)
        if not silent: print 'Par '+key
        x1 = np.array(data[key])-np.median(data[key])
        cj = []             # auto-correlation
        jarr = []           # lag
        # starting auto-correlation, set to be much higher than tolerance
        val = 1e0
        j = 0
        szx1 = len(x1)
        sli0 = 0
        slj1 = szx1-1
        while val > lowtol and j < jmax:
            sli1 = szx1-j-1
            slj0 = j
            #t0 = time.time()
            x1i_x1ipj = x1[sli0:sli1]*x1[slj0:slj1]
            x1i_sq = (x1[sli0:sli1]*x1[sli0:sli1])
            x1i = (x1[sli0:sli1])
            mean_x1i_x1ipj = np.mean(x1i_x1ipj)
            mean_x1i = np.mean(x1i)
            mean_x1i_sq = np.mean(x1i_sq) 
            #mean_x1i_x1ipj = quickMean(x1i_x1ipj)
            #mean_x1i = quickMean(x1i)
            #mean_x1i_sq = quickMean(x1i_sq) 
            val = (mean_x1i_x1ipj - (mean_x1i)**2)/\
                  (mean_x1i_sq - (mean_x1i)**2)
            #t1 = time.time()
            if j > 0 and SetDynamic:
                dc_dj = np.abs( (cj[-1]-val)/(jarr[-1]-j))
                if (val > 0.45) and (val < 0.55):
                    scale = 0.005
                else:
                    scale = 0.01
                Jstep = long(round(scale/dc_dj))
                #print t1-t0,j,val, dc_dj, 0.01/(dc_dj), Jstep
                if Jstep < 1: 
                    Jstep = 1
                elif Jstep > 100:
                    Jstep = 100
            cj.append(val)
            jarr.append(j)
            if SetDynamic:
                j += Jstep
            else:
                j += Nres

        cj = np.array(cj)
        jarr = np.array(jarr)
        ChainLength = szx1
        corlen_index = np.where( np.abs(cj-0.5e0) == np.min( np.abs(cj-0.5e0)) )[0]
        corlen = jarr[corlen_index[0]]
        efflen = long(float(ChainLength)/float(corlen))
        if mkPlotsFlag:
            print 'plotting acor', ftag+'.ACOR.par_'+key+'.png'
            plt.plot(jarr,cj,'b.')
            plt.plot([corlen,corlen],[0,1],'k--')
            plt.plot([0,max(jarr)],[0.5,0.5],'k--')
            plt.xlabel('j (Lag)')
            plt.ylabel('C (Auto-Correlation)')
            plt.title('Auto-Correlation for "'+key+'"')
            plt.savefig(ftag+'.ACOR.par_'+key+'.png')
            plt.clf()

        print >> OutFileObject, '##'+key+'## Corr Length = '+format(corlen,'d')
        print >> OutFileObject, '##'+key+'## Eff Length = '+format(efflen,'d')
        chain_stats[key] = {'corlength':corlen,'efflength':efflen}

    clen = []
    elen = []
    for key in chain_stats.keys():
        clen.append(chain_stats[key]['corlength'])
        elen.append(chain_stats[key]['efflength'])
    if not silent:
        print '## ALL ## Chain Length = '+format(ChainLength,'d')
        print '## ALL ## Corr Length = '+format(max(clen),'d')
        print '## ALL ## Eff Length = '+format(min(elen),'d')
    print >> OutFileObject, '## ALL ## Chain Length = '+format(ChainLength,'d')
    print >> OutFileObject, '## ALL ## Corr Length = '+format(max(clen),'d')
    print >> OutFileObject, '## ALL ## Eff Length = '+format(min(elen),'d')
    OutFileObject.close()
Example #16
0
def covcorStats(File, FileTag, **kwargs):
    """ Given MCMC parameters, this function computes
        the covariance between parameters, the pearson's correlation 
        coefficient and spearman's rank correlation.
    """
    
    DFlag = False
    for key in kwargs:
        if key.lower().startswith('derive'):
            DFlag = kwargs[key]
    
    stats = {}
    OutFileObject_COV = open(FileTag+'_COV.data','w')
    OutFileObject_PR = open(FileTag+'_PEARSONSR.data','w')
    OutFileObject_SR = open(FileTag+'_SPEARMANSR.data','w')
    #print >> OutFileObject, '#  par1   |  par2   |  cov   | pearson\'s r  | spearman\'s r '
    topline = '#'
    passCount = 0
    hdrKeys = readMCMChdr(File)
    parList1 = []
    parList2 = []
    for i in hdrKeys.keys():
        key = hdrKeys[i]
        if not isNonParam(key):
            parList1.append(key)
            parList2.append(key)

    ComboList = []
    ComboDict = {}
    for par1 in parList1: 
        covline = ''
        pcorline = ''
        scorline = ''
        for par2 in parList2:
            t0 = time.time()
            #x1 = np.arange(10)
            Combo = (par1,par2)
            if not Combo[::-1] in ComboList:
                if par1 == par2:
                    d1 = read1parMCMC(File,par1,derived=DFlag)
                    x1 = np.array(d1[par1])
                    x2 = x1
                else:
                    d1 = read1parMCMC(File,par1,derived=DFlag)
                    x1 = np.array(d1[par1])
                    d2 = read1parMCMC(File,par2,derived=DFlag)
                    x2 = np.array(d2[par2])
                ComboList.append((par1,par2))
                cov = np.cov(x1,x2)
                pcor = scipy.stats.pearsonr(x1,x2)
                scor = scipy.stats.spearmanr(x1,x2)
                ComboDict[Combo] = {'cov':cov,'pcor':pcor,'scor':scor}
            else:
                cov = ComboDict[Combo[::-1]]['cov']
                pcor = ComboDict[Combo[::-1]]['pcor']
                scor = ComboDict[Combo[::-1]]['scor']

            #x2 = np.arange(10)
            t1 = time.time()
            print par1, par2, t1-t0
            covline = covline+' '+format(cov[0][1],'+.2e')
            pcorline = pcorline+' '+format(pcor[0],'+.2e')
            scorline = scorline+' '+format(scor[0],'+.2e')
            if passCount == 0: topline = topline+5*' '+par2 
        if passCount == 0:
            print >> OutFileObject_COV, topline
            print >> OutFileObject_COV, '#'+88*'-'
            print >> OutFileObject_PR, topline
            print >> OutFileObject_PR, '#'+88*'-'
            print >> OutFileObject_SR, topline
            print >> OutFileObject_SR, '#'+88*'-'
        lenkey = len(par1)
        if lenkey < 10:
            fac = 10-lenkey
        else:
            fac = 10
        print >> OutFileObject_COV, par1+fac*' '+' | '+covline
        print >> OutFileObject_PR, par1+fac*' '+' | '+pcorline
        print >> OutFileObject_SR, par1+fac*' '+' | '+scorline
        passCount += 1

    OutFileObject_COV.close()
    OutFileObject_PR.close()
    OutFileObject_SR.close()
Example #17
0
def RunMinuit(FunctionName,ObservedData,ModelParams,NuisanceData,BoundParams,tolnum,OutFile):
    """ Designed to run Minuit on tmcmc format Data dictionaries """
    
    OpenParNames = []
    for key in ModelParams.keys():
        if ModelParams[key]['open']:
            OpenParNames.append(key)
    startchi2 = f_chisquared(FunctionName,ObservedData,ModelParams,NuisanceData,BoundParams)
    dumpfile1 = open('dump1','wb')
    dumpfile2 = open('dump2','wb')
    dumpfile3 = open('dump3','wb')
    dumpfile4 = open('dump4','wb')
    dumpfile5 = open('dump5','wb')
    dumpfile6 = open('dump6','wb')
    pickle.dump(FunctionName,dumpfile1,-1)
    pickle.dump(ObservedData,dumpfile2,-1)
    pickle.dump(ModelParams,dumpfile3,-1)
    pickle.dump(NuisanceData,dumpfile4,-1)
    pickle.dump(BoundParams,dumpfile5,-1)
    pickle.dump(OpenParNames,dumpfile6,-1)
    dumpfile1.close()
    dumpfile2.close()
    dumpfile3.close()
    dumpfile4.close()
    dumpfile5.close()
    dumpfile6.close()

    # Begin running Minuit
    converge = False
    CountA1 = 0
    CountFail = 0
    AbsoluteFail = False
    while not converge:
        if CountFail > 20:
            print 'Failed'
            AbsoluteFail = True
            break
        try:
            m = minuit2.Minuit2(transit_chisquared(OpenParNames))
            m.tol = tolnum #55.4720096  #*1e8
            m.up = 1
            m.strategy = 2
            for key in m.values.keys():
                m.values[key] = ModelParams[key]['value']
                m.errors[key] = ModelParams[key]['step']
            m.migrad()
            #print 'edm = ',m.edm, '(edm < 1e-3 signifies convergence, edm > 1e2 is probably too high)'
            #print m.edm, converge, tolnum, 1e-3*m.tol*m.up, m.ncalls
            tolStatus = m.edm/(1e-3*m.tol*m.up)
            if tolStatus <= 1e0:
                if tolStatus == 1e0:
                    CountA1 = 100
                CountA1 += 1
                if CountA1 <= 4:
                    converge = False
                else:
                    converge = True
                tolnum *= tolStatus
                #CloseDict['a1'] = tolnum
                w = 'a1 Good Convergence'
            else:
                converge = False
                tolnum *= tolStatus
                w = 'a2 Poor Convergence'
                if np.isnan(tolStatus):
                    CountFail += 1 
            print tolStatus, m.edm, m.tol, tolnum, converge, w, CountFail
        except:
            if m.edm != None:
                edm = m.edm
            else:
                edm = 1e-2*m.tol*m.up
            if CountFail < 11:
                tolStatus = (edm/(1e-3*m.tol*m.up))
            elif CountFail > 11:
                tolStatus = (edm/(1e-3*m.tol*m.up))**(-1)
            else:
                tolStatus = 10e0/tolnum
            if tolStatus == 1.0:
                tolStatus = 10e0
            tolnum *= tolStatus
            converge = False
            CountFail += 1
            print tolStatus, m.edm, m.tol, tolnum, converge, 'b Failed Convergence', CountFail
            #raise

    for key in m.values.keys():
        ModelParams[key]['value'] = m.values[key]
        ModelParams[key]['step'] = m.errors[key]

    if not AbsoluteFail:
        DOF = len(ObservedData['all']['y']) - len(OpenParNames)
        PrintModelParams(ModelParams,OutFile)
        OutFileObjectAppend = open(OutFile,'a')
        print >> OutFileObjectAppend, \
        '# Best-fit ChiSQ = '+format(m.fval,'.2f')+' | Starting ChiSQ = ',format(startchi2,'.2f')
        print >> OutFileObjectAppend, '# DOF = '+format(DOF)
        print >> OutFileObjectAppend, \
        '# Best-fit reduced ChiSQ = '+format(m.fval/DOF,'.2f')+' | Starting reduced ChiSQ = '+format(startchi2/DOF,'.2f')
        OutFileObjectAppend.close()
Example #18
0
    def statsTable(self,Stats,stype, **kwargs):

        width, height = matplotlib.rcParams['figure.figsize']
        size = max([width,height])
        # make a square figure
        fig = plt.figure(figsize=(size,size))

        # default font sizes
        xfsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
        yfsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
        fsize = np.floor(300e0*size/(24*max(self.GridNX,self.GridNY)))
        yshift = 0
        xshift = 0
        PlotFile = False
        TitleString = ''
        #print xfsize,yfsize,fsize
        if fsize > 24.0:
            fsize = 24.0
        if xfsize > 18.0:
            xfsize = 18.0
        if yfsize > 18.0:
            yfsize = 18.0
            
        for key in kwargs:
            if key.lower().startswith('plotfile'):
                PlotFile = True
                FileName = kwargs[key]
            elif key.lower().startswith('numfsize'):
                fsize = kwargs[key]
            elif key.lower().startswith('fsizex'):
                xfsize = kwargs[key]
            elif key.lower().startswith('fsizey'):
                yfsize = kwargs[key]
            elif key.lower().startswith('xshift'):
                xshift = kwargs[key]
            elif key.lower().startswith('yshift'):
                yshift = kwargs[key]
            elif key.lower().startswith('title'):
                TitleString = kwargs[key]
            else:
                pass

        for grid in self.GridDict.keys():
            plotID = subID(grid,self.GridNX,self.GridNY)
            plt.subplot(self.GridNY,self.GridNX,plotID)

            Stat = Stats[stype]\
                        [self.GridDict[grid][0]]\
                        [self.GridDict[grid][1]]\
                        ['value']

            absStat = abs(Stat)

            cross = np.linspace(0,1,3)
            plt.plot(cross,cross,marker='o',\
                     markerfacecolor='w',\
                     markeredgecolor='w',\
                     linestyle='None')
            #numsplit = map(str,text.split('e'))
            #OutText = r'%s$\times 10^{%s}$' % (numsplit[0],numsplit[1])
            if absStat < 0.01:
                text = r'$< 0.01$'
                OutText = r'%s' % text
                plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize)
            elif absStat >= 0.5:
                text = format(abs(Stat),'0.2f')
                plt.subplot(self.GridNY,self.GridNX,plotID, axisbg='gray')
                OutText = r'%s' % text
                plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize,color='w')
            else:
                text = format(abs(Stat),'0.2f')
                plt.subplot(self.GridNY,self.GridNX,plotID)
                OutText = r'%s' % text
                plt.text(0.0-xshift,0.4-yshift,OutText,fontsize=fsize)

            plt.setp(plt.gca(),yticklabels=[],yticks=[])
            plt.setp(plt.gca(),xticklabels=[],xticks=[])
            if grid[1] == 0:
                #print self.Label[grid]['x'], grid
                plt.xlabel(self.Label[grid]['x'],fontsize=xfsize)
            if grid[0] == 0:
                #print self.Label[grid]['y'], grid
                plt.ylabel(self.Label[grid]['y'],fontsize=yfsize)
            
        plt.subplots_adjust(hspace=0)
        plt.subplots_adjust(wspace=0)
        plt.suptitle(TitleString,fontsize=24,y=0.95)

        if PlotFile:
            plt.savefig(FileName,pad_inches=0.5)
        else:
            plt.show()
Example #19
0
def returnDerivedLine_MTQ2011(ModelParams,istep,keyList0):
    """ print a file with Derived parameters from the MCMC ensemble. """
        
    derived = {}
    
    Period = computePeriod(ModelParams)
    derived['Period'] = {'value':Period,'printformat':'.9f'}
    RefFilt = ModelParams['RefFilt']['printformat']

    # get Filter and Transit time tags
    Tags = getTags(ModelParams)
    
    # compute parameters used to compute lightcurve using the reference filter
    Dref, v1ref, v2ref = MTQ_2011.MTQ_FilterParams(RefFilt,Tags,ModelParams)
    u1ref, u2ref = LDC_v2u(v1ref,v2ref)

    tT = ModelParams['tT']['value']
    tG = ModelParams['tG']['value']
    
    TransitRef = MTQ_2011.MTQ_getDerivedParams(Dref,tT,tG,u1ref,u2ref,Period)
    #print TransitRef
    derived['inc'] = {'value':TransitRef['inc']*180e0/np.pi,'printformat':'.4f'}
    derived['b'] = {'value':TransitRef['b'],'printformat':'.6f'}
    derived['aRs'] = {'value':TransitRef['aRs'],'printformat':'.6f'}
    derived['velRs'] = {'value':TransitRef['velRs'],'printformat':'.6f'}
    derived['rho_star'] = {'value':TransitRef['rho_star'],'printformat':'.6f'}
    derived['RpRs'+'.'+ModelParams['RefFilt']['printformat']] = \
    {'value':TransitRef['RpRs'],'printformat':'.9f'}
    
    for key in ModelParams.keys():
        if key.startswith('T0'):
            split_TT = map(str,key.split('.'))
            transit_tag = split_TT[1].strip()
    
            # compute D, v1, v2 and then u1 and u2 for a given transit tag
            D, v1, v2 = MTQ_2011.MTQ_FilterParams(transit_tag,Tags,ModelParams)
            u1, u2 = LDC_v2u(v1,v2)
            RpRs = computeRpRs(u1,u2,tT,tG,D)
            filterD = filterMatchD(transit_tag,Tags,ModelParams)
            #print filterD
            derived['RpRs'+'.'+filterD] = {'value':RpRs,'printformat':'.9f'}
    
    DerivedLine = ''
    if istep == 0:
        keylist = derived.keys()
    else:
        keylist = keyList0

    Nkeys = len(keylist)
    if istep == 0:
        keyorder = {}
        for i in range(len(keylist)):
            keyorder[i] = keylist[i]
        for i in range(Nkeys):
            if i == 0:
                DerivedLine =\
                str(format(derived[keylist[i]]['value'],\
                derived[keylist[i]]['printformat']))
            else:
                DerivedLine =\
                DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
                derived[keylist[i]]['printformat']))
    else:
        for i in range(Nkeys):
            if i == 0:
                DerivedLine =\
                str(format(derived[keylist[i]]['value'],\
                derived[keylist[i]]['printformat']))
            else:
                DerivedLine = DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
                derived[keylist[i]]['printformat']))

    DerivedLine = DerivedLine+'|'+str(format(istep,'.0f'))+'|:'
    
    return DerivedLine, keylist
Example #20
0
def printDerived_MTQ_2011(STARTFILE,MCMCfile,DerivedFile):
    """ print a file with Derived parameters from the MCMC ensemble. """
    
    hdrKeys = readMCMChdr(MCMCfile)
    ModelParams = ReadStartParams(STARTFILE)
    
    mcmcFile = open(MCMCfile,'r')
    mcmcFile = mcmcFile.readlines()
    OutFileObject = open(DerivedFile,'w')
    
    for line in mcmcFile:
        derived = {}
        if not line.startswith('#'):
            data_line = ReadMCMCline(line,hdrKeys)
            print 'step/line = ',format(long(data_line['istep']),'n')
            for key in ModelParams.keys():
                if ModelParams[key]['open']:
                    ModelParams[key]['value'] = data_line[key]
            
            Period = computePeriod(ModelParams)
            derived['Period'] = {'value':Period,'printformat':'.9f'}
            RefFilt = ModelParams['RefFilt']['printformat']
    
            # get Filter and Transit time tags
            Tags = getTags(ModelParams)
            
            # compute parameters used to compute lightcurve using the reference filter
            Dref, v1ref, v2ref = MTQ_2011.MTQ_FilterParams(RefFilt,Tags,ModelParams)
            u1ref, u2ref = LDC_v2u(v1ref,v2ref)

            tT = ModelParams['tT']['value']
            tG = ModelParams['tG']['value']
            
            TransitRef = MTQ_2011.MTQ_getDerivedParams(Dref,tT,tG,u1ref,u2ref,Period)
            #print TransitRef
            derived['inc'] = {'value':TransitRef['inc']*180e0/np.pi,'printformat':'.4f'}
            derived['b'] = {'value':TransitRef['b'],'printformat':'.6f'}
            derived['aRs'] = {'value':TransitRef['aRs'],'printformat':'.6f'}
            derived['velRs'] = {'value':TransitRef['velRs'],'printformat':'.6f'}
            derived['rho_star'] = {'value':TransitRef['rho_star'],'printformat':'.6f'}
            derived['RpRs'+'.'+ModelParams['RefFilt']['printformat']] = \
            {'value':TransitRef['RpRs'],'printformat':'.9f'}
            
            for key in ModelParams.keys():
                if key.startswith('T0'):
                    split_TT = map(str,key.split('.'))
                    transit_tag = split_TT[1].strip()
            
                    # compute D, v1, v2 and then u1 and u2 for a given transit tag
                    D, v1, v2 = MTQ_2011.MTQ_FilterParams(transit_tag,Tags,ModelParams)
                    u1, u2 = LDC_v2u(v1,v2)
                    RpRs = computeRpRs(u1,u2,tT,tG,D)
                    filterD = filterMatchD(transit_tag,Tags,ModelParams)
                    #print filterD
                    derived['RpRs'+'.'+filterD] = {'value':RpRs,'printformat':'.9f'}
            
            DerivedLine = ''
            keylist = derived.keys()
            Nkeys = len(keylist)
            if data_line['istep'] == 0:
                keyorder = {}
                for i in range(len(keylist)):
                    keyorder[i] = keylist[i]
                for i in range(Nkeys):
                    if i == 0:
                        DerivedLine =\
                        str(format(derived[keylist[i]]['value'],\
                        derived[keylist[i]]['printformat']))
                    else:
                        DerivedLine =\
                        DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
                        derived[keylist[i]]['printformat']))
            else:
                for i in range(Nkeys):
                    if i == 0:
                        DerivedLine =\
                        str(format(derived[keylist[i]]['value'],\
                        derived[keylist[i]]['printformat']))
                    else:
                        DerivedLine = DerivedLine+'|'+str(format(derived[keylist[i]]['value'],\
                        derived[keylist[i]]['printformat']))

            DerivedLine = DerivedLine+'|'+str(format(data_line['istep'],'.0f'))+'|:'
            print >> OutFileObject, DerivedLine
    
    keyline = ''
    for i in range(Nkeys):
        if i == 0:
            keyline = '#'+keylist[i]
        else:
            keyline = keyline+'|'+keylist[i]
    keyline = keyline+'|istep|:'
    
    OutFileObject.close()
    
    #print >> OutFileObject, keyline
    opencopy = open(DerivedFile,'r')
    opencopy = opencopy.readlines()
    os.system('rm -v %s' % (DerivedFile))
    rearranged = open(DerivedFile,'w')
    print >> rearranged, keyline
    for iline in range(len(opencopy)):
        print >> rearranged, opencopy[iline].strip('\n')
    rearranged.close()