def getSsim(ell, Cl, lmax=100, cutSky=False):
    """
  Purpose:
    create simulated S_{1/2} from input power spectrum
  Note:
    this calculates Jmn every time it is run so should not be used for ensembles
  Procedure:
    simulates full sky CMB, measures S_{1/2}
  Inputs:
    ell: the l values for the power spectrum
    Cl: the power spectrum
    lmax: the maximum ell value to use in calculation
      Default: 100
    cutSky: set to True to convert to real space, apply mask, etc.
      Default: False
      Note: true option not yet implemented
  Returns:
    simulated S_{1/2}
  """
    # get Jmn matrix for harmonic space S_{1/2} calc.
    myJmn = getJmn(lmax=lmax)[2:, 2:]  # do not include monopole, dipole

    #alm_prim,alm_late = hp.synalm((primCl,lateCl,crossCl),lmax=lmax,new=True)
    almSim = hp.synalm(
        Cl, lmax=lmax)  # question: does this need to start at ell[0]=1?
    ClSim = hp.alm2cl(almSim)

    return np.dot(ClSim[2:], np.dot(myJmn, ClSim[2:]))
Esempio n. 2
0
    def __init__(self, lmax=100, new=True):
        if new:
            print 'new Jmn init running'
        else:
            print 'old Jmn init running'

        nVals = 11  #101
        self.Jmnx = np.empty([nVals, lmax + 1, lmax + 1])
        self.xvalues = np.linspace(0.25, 0.75, nVals)
        for index, val in enumerate(self.xvalues):
            self.Jmnx[index] = legprodint.getJmn(endX=val,
                                                 lmax=lmax,
                                                 doSave=False)
def test(useCLASS=1,
         useLensing=1,
         classCamb=1,
         nSims=1000,
         lmax=100,
         lmin=2,
         newSMICA=False,
         newDeg=False,
         suppressC2=False,
         suppFactor=0.23,
         filterC2=False,
         filtFacLow=0.1,
         filtFacHigh=0.2,
         R1=False):
    """
    code for testing the other functions in this module
    Inputs:
      useCLASS: set to 1 to use CLASS, 0 to use CAMB
        CLASS Cl has early/late split at z=50
        CAMB Cl has ISWin/out split: ISWin: 0.4<z<0.75, ISWout: the rest
        Note: CAMB results include primary in ISWin and ISWout (not as intended)
        default: 1
      useLensing: set to 1 to use lensed Cl, 0 for non-lensed
        default: 1
      classCamb: if 1: use the CAMB format of CLASS output, if 0: use CLASS format
        Note: parameter not used if useCLASS = 0
        default: 1
      nSims: the number of simulations to do for ensemble
        default: 1000
      lmax: the highest l to include in Legendre transforms
        default: 100
      lmin: the lowest l to include in S_{1/2} = CIC calculations
        default: 2
      newSMICA: set to True to recalculate SMICA results
        default: False
      newDeg: set to True to recalculate map and mask degredations
        (only if newSMICA is also True)
        default: False
      suppressC2: set to True to suppress theoretical C_2 (quadrupole) by 
        suppFactor before creating a_lm.s
        Default: False
      suppFactor: multiplies C_2 if suppressC2 is True
        Default: 0.23 # from Tegmark et. al. 2003, figure 13 (WMAP)
      filterC2 : set to true to filter simulated CMBs after spice calculates
        cut sky C_l.  Sims will pass filter if C_2 * filtFacLow < C_2^sim <
        C_2 * filtFacHigh.
        Default: False
      filtFacLow,filtFacHigh: defines C_2 range for passing simulated CMBs
        Default: 0.1,0.2
      R1: set to True to use SMICA and Mask R1.  Otherwise, R2 used.
        Only affects calculation of newly degraded map.
        Default: False
  """

    # load data
    ell, fullCl, primCl, lateCl, crossCl = gcp.loadCls(useCLASS=useCLASS,
                                                       useLensing=useLensing,
                                                       classCamb=classCamb)

    # fill beginning with zeros
    startEll = int(ell[0])
    ell = np.append(np.arange(startEll), ell)
    fullCl = np.append(np.zeros(startEll), fullCl)
    primCl = np.append(np.zeros(startEll), primCl)
    lateCl = np.append(np.zeros(startEll), lateCl)
    crossCl = np.append(np.zeros(startEll), crossCl)

    # suppress C_2 to see what happens in enesmble
    if suppressC2:
        fullCl[2] *= suppFactor
        primCl[2] *= suppFactor
        lateCl[2] *= suppFactor
        crossCl[2] *= suppFactor

    conv = ell * (ell + 1) / (2 * np.pi)
    #print ell,conv #ell[0]=2.0
    """
  # verify statistical properties of alm realizations
  nSims = 1000
  lmax = 100
  Clprim_sum = np.zeros(lmax+1)
  Cllate_sum = np.zeros(lmax+1)
  Clcros_sum = np.zeros(lmax+1) # prim x late
  for nSim in range(nSims):
    print 'starting sim ',nSim+1, ' of ',nSims
    #alm_prim,alm_late = getAlms(A_lij,lmax=lmax) #AKW method defunct
    # see if synalm can do it
    alm_prim,alm_late = hp.synalm((primCl,lateCl,crossCl),lmax=lmax,new=True)
    Clprim_sum = Clprim_sum + hp.alm2cl(alm_prim)
    Cllate_sum = Cllate_sum + hp.alm2cl(alm_late)
    Clcros_sum = Clcros_sum + hp.alm2cl(alm_prim,alm_late)
  Cl_prim_avg = Clprim_sum/nSims
  Cl_late_avg = Cllate_sum/nSims
  Cl_cros_avg = Clcros_sum/nSims

  doPlot = True
  if doPlot:
    plt.plot(ell[:lmax+1],Cl_prim_avg*conv[:lmax+1])
    plt.plot(ell[:lmax+1],primCl[:lmax+1]*conv[:lmax+1])
    plt.title('primary')
    plt.ylabel('D_l')
    plt.show()

    plt.plot(ell[:lmax+1],Cl_late_avg*conv[:lmax+1])
    plt.plot(ell[:lmax+1],lateCl[:lmax+1]*conv[:lmax+1])
    plt.title('late')
    plt.ylabel('D_l')
    plt.show()

    plt.plot(ell[:lmax+1],Cl_cros_avg*conv[:lmax+1])
    plt.plot(ell[:lmax+1],crossCl[:lmax+1]*conv[:lmax+1])
    plt.title('cross')
    plt.ylabel('D_l')
    plt.show()
  """

    # get covariances from SMICA map and mask
    theta_i = 0.0  #degrees
    theta_f = 180.0  #degrees
    nSteps = 1800
    #lmax = 100
    """ # don't want anafast after all

  # get unmasked and masked SMICA covariances
  #   note: getSMICA uses linspace in theta for thetaArray
  #newSMICA = False#True
  thetaArray2, C_SMICA, C_SMICAmasked, S_SMICAnomask, S_SMICAmasked = \
    getSMICA(theta_i=theta_i,theta_f=theta_f,nSteps=nSteps,lmax=lmax,lmin=lmin,
             newSMICA=newSMICA,newDeg=newDeg,useSPICE=False,R1=R1)
  print ''
  print 'S_{1/2}(anafast): SMICA, no mask: ',S_SMICAnomask,', masked: ',S_SMICAmasked
  print ''
  """

    # get C_l from SPICE to compare to above method
    #   note: getSMICA uses linspace in theta for thetaArray
    #newSMICA = False#True
    thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp, S_SMICAnomasksp, S_SMICAmaskedsp = \
      getSMICA(theta_i=theta_i,theta_f=theta_f,nSteps=nSteps,lmax=lmax,lmin=lmin,
               newSMICA=newSMICA,newDeg=newDeg,useSPICE=True,R1=R1)
    print ''
    print 'S_{1/2}(spice): SMICA, no mask: ', S_SMICAnomasksp, ', masked: ', S_SMICAmaskedsp
    print ''

    # Find S_{1/2} in real space to compare methods
    nTerms = 10000
    #SSnm2   = SOneHalf(thetaArray2,  C_SMICA,         nTerms=nTerms)
    #SSmd2   = SOneHalf(thetaArray2,  C_SMICAmasked,   nTerms=nTerms)
    SSnm2sp = SOneHalf(thetaArray2sp, C_SMICAsp, nTerms=nTerms)
    SSmd2sp = SOneHalf(thetaArray2sp, C_SMICAmaskedsp, nTerms=nTerms)

    # create ensemble of realizations and gather statistics
    covEnsembleFull = np.zeros([nSims, nSteps + 1])  # for maskless
    covEnsembleCut = np.zeros([nSims, nSteps + 1])  # for masked
    sEnsembleFull = np.zeros(nSims)
    sEnsembleCut = np.zeros(nSims)
    covTheta = np.array([])
    #nSims = 1000

    # apply beam and pixel window functions to power spectra
    #   note: to ignore the non-constant pixel shape, W(l) must be > B(l)
    #     however, this is not true for NSIDE=128 and gauss_beam(5')
    #   Here I ignore this anyway and proceed
    myNSIDE = 128  # must be same NSIDE as in getSMICA function
    Wpix = hp.pixwin(myNSIDE)
    Bsmica = hp.gauss_beam(5. / 60 * np.pi / 180)  # 5 arcmin
    WlMax = Wpix.size
    if WlMax < lmax:
        print 'die screaming!!!'
        return 0
    primCl = primCl[:WlMax] * (Wpix * Bsmica)**2
    lateCl = lateCl[:WlMax] * (Wpix * Bsmica)**2
    crossCl = crossCl[:WlMax] * (Wpix * Bsmica)**2
    # note: i tried sims without this scaling, and results seemed the same at a glance

    # collect simulated Cl for comparison to model
    Clsim_full_sum = np.zeros(lmax + 1)

    # get Jmn matrix for harmonic space S_{1/2} calc.
    myJmn = getJmn(lmax=lmax)

    # set up ramdisk for SpICE
    # super lame that spice needs to read/write from disk, but here goes...
    RAMdisk = '/Volumes/ramdisk/'
    ClTempFile = RAMdisk + 'tempCl.fits'
    mapTempFile = RAMdisk + 'tempMap.fits'
    mapDegFile = RAMdisk + 'smicaMapDeg.fits'  # this should have been created by sims.getSMICA
    maskDegFile = RAMdisk + 'maskMapDeg.fits'  # this should have been created by sims.getSMICA

    # create RAM Disk for SpICE and copy these files there using bash
    RAMsize = 4  #Mb
    ramDiskOutput = subprocess.check_output('./ramdisk.sh create ' +
                                            str(RAMsize),
                                            shell=True)
    print ramDiskOutput
    diskID = ramDiskOutput[
        31:41]  # this might not grab the right part; works for '/dev/disk1'
    subprocess.call('cp smicaMapDeg.fits ' + RAMdisk, shell=True)
    subprocess.call('cp maskMapDeg.fits ' + RAMdisk, shell=True)

    doTime = True  # to time the run and print output
    startTime = time.time()
    #for nSim in range(nSims):
    nSim = 0
    while nSim < nSims:
        print 'starting sim ', nSim + 1, ' of ', nSims
        alm_prim, alm_late = hp.synalm((primCl, lateCl, crossCl),
                                       lmax=lmax,
                                       new=True)

        # calculate C(theta) of simulation
        Clsim_prim = hp.alm2cl(alm_prim)
        Clsim_late = hp.alm2cl(alm_late)
        Clsim_cros = hp.alm2cl(alm_prim, alm_late)
        Clsim_full = Clsim_prim + 2 * Clsim_cros + Clsim_late
        # use Cl_sim_full to omit prim/late distinction for now

        # start with a mask
        #   -> for optional C2 filtering based on cut sky map
        #   alm2map should create map with default RING ordering
        #   pixel window and beam already accounted for in true Cls
        #mapSim = hp.alm2map(alm_prim+alm_late,myNSIDE,lmax=lmax,pixwin=True,sigma=5./60*np.pi/180)
        mapSim = hp.alm2map(alm_prim + alm_late, myNSIDE, lmax=lmax)

        hp.write_map(mapTempFile, mapSim)
        ispice(mapTempFile,
               ClTempFile,
               maskfile1=maskDegFile,
               subav="YES",
               subdipole="YES")
        Cl_masked = hp.read_cl(ClTempFile)
        ell2 = np.arange(Cl_masked.shape[0])

        # Check for low power of cut sky C_2
        if (filterC2 == True and fullCl[2] * filtFacHigh > Cl_masked[2] and
                Cl_masked[2] > fullCl[2] * filtFacLow) or filterC2 == False:

            #   note: getCovar uses linspace in x for thetaArray
            thetaArray, cArray2 = getCovar(ell2[:lmax + 1],
                                           Cl_masked[:lmax + 1],
                                           theta_i=theta_i,
                                           theta_f=theta_f,
                                           nSteps=nSteps,
                                           lmin=lmin)
            covEnsembleCut[nSim] = cArray2

            # S_{1/2}
            sEnsembleCut[nSim] = np.dot(
                Cl_masked[lmin:lmax + 1],
                np.dot(myJmn[lmin:, lmin:], Cl_masked[lmin:lmax + 1]))

            doPlot = False  #True
            if doPlot:
                plt.plot(thetaArray, cArray)
                plt.xlabel('theta (degrees)')
                plt.ylabel('C(theta)')
                plt.title('covariance of CMB simulation ' + str(nSim + 1))
                plt.show()

            # now without the mask
            # uses the same sims that passed the C2 filter
            Clsim_full_sum += Clsim_full

            #   note: getCovar uses linspace in x for thetaArray
            thetaArray, cArray = getCovar(ell[:lmax + 1],
                                          Clsim_full[:lmax + 1],
                                          theta_i=theta_i,
                                          theta_f=theta_f,
                                          nSteps=nSteps,
                                          lmin=lmin)
            covEnsembleFull[nSim] = cArray
            covTheta = thetaArray

            # S_{1/2}
            sEnsembleFull[nSim] = np.dot(
                Clsim_full[lmin:],
                np.dot(myJmn[lmin:, lmin:], Clsim_full[lmin:]))

            nSim += 1

    if doTime:
        print 'time elapsed: ', int(
            (time.time() - startTime) / 60.), ' minutes'

    # free the RAM used by SpICE's RAM disk
    ramDiskOutput = subprocess.check_output('./ramdisk.sh delete ' + diskID,
                                            shell=True)
    print ramDiskOutput

    avgEnsembleFull = np.average(covEnsembleFull, axis=0)
    stdEnsembleFull = np.std(covEnsembleFull, axis=0)
    # do I need a better way to describe confidence interval?
    avgEnsembleCut = np.average(covEnsembleCut, axis=0)
    stdEnsembleCut = np.std(covEnsembleCut, axis=0)

    Clsim_full_avg = Clsim_full_sum / nSims

    # save results
    saveFile1 = "simStatResultC.npy"
    np.save(
        saveFile1,
        np.vstack((thetaArray, avgEnsembleFull, stdEnsembleFull,
                   avgEnsembleCut, stdEnsembleCut)))
    saveFile2 = "simStatC_SMICA.npy"
    np.save(saveFile2, np.vstack((thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp)))

    saveFile3 = "simStatResultS.npy"
    np.save(
        saveFile3,
        np.vstack((np.hstack((np.array(S_SMICAnomasksp), sEnsembleFull)),
                   np.hstack((np.array(S_SMICAmaskedsp), sEnsembleCut)))))

    doPlot = True
    if doPlot:
        print 'plotting C_l... '
        #print ell.size,conv.size,primCl.size,crossCl.size,lateCl.size
        plt.plot(ell[:lmax + 1],
                 conv[:lmax + 1] * (primCl + 2 * crossCl + lateCl)[:lmax + 1],
                 label='model D_l')
        plt.plot(ell[:lmax + 1],
                 conv[:lmax + 1] * Clsim_full_avg,
                 label='ensemble average D_l')
        plt.legend()
        plt.show()

        makePlots(saveFile1=saveFile1,
                  saveFile2=saveFile2,
                  saveFile3=saveFile3)

    # S_{1/2} output
    print ''
    print 'using CIC method: '
    #print 'S_{1/2}(anafast): SMICA, no mask: ',S_SMICAnomask,', masked: ',S_SMICAmasked
    print 'S_{1/2}(spice): SMICA, no mask: ', S_SMICAnomasksp, ', masked: ', S_SMICAmaskedsp
    print ''
    print 'using CCdx method: '
    #print 'S_{1/2}(anafast): SMICA, no mask: ',SSnm2,', masked: ',SSmd2
    print 'S_{1/2}(spice): SMICA, no mask: ', SSnm2sp, ', masked: ', SSmd2sp
    print ''
def getSMICA(theta_i=0.0,
             theta_f=180.0,
             nSteps=1800,
             lmax=100,
             lmin=2,
             newSMICA=False,
             useSPICE=True,
             newDeg=False,
             R1=False):
    """
  Purpose:
    load CMB and mask maps from files, return correlation function for
    unmasked and masked CMB
    Mostly follows Copi et. al. 2013 for cut sky C(theta)
  Uses:
    get_crosspower.py (for plotting)
    C(theta) save file getSMICAfile.npy
  Inputs:
    theta_i,theta_f: starting and ending points for C(theta) in degrees
    nSteps: number of intervals between i,f points
    lmax: the maximum l value to include in legendre series for C(theta)
    lmin: the lowest l to use in C(theta,Cl) and S_{1/2} = CIC calculation
    newSMICA: set to True to reload data from files and recompute
      if False, will load C(theta) curves from file
    useSPICE: if True, will use SPICE to find power spectra
      if False, will use anafast, following Copi et. al. 2013
      Default: True
    newDeg: set to True to recalculate map and mask degredations
      Note: the saved files are dependent on the value of lmax that was used
      Default: False
    R1: set to True to use R1 versions of SMICA and mask.  Otherwise, R2 is used
      Only affects which Planck files are used; irrelevant if newDeg=False.
      Default: False
  Outupts:
    theta: nSteps+1 angles that C(theta) arrays are for (degrees)
    unmasked: C(theta) unmasked (microK^2)
    masked: C(theta) masked     (microK^2)

  """
    saveFile = 'getSMICAfile.npy'  #for anafast
    saveFile2 = 'getSMICAfile2.npy'  #for spice
    if newSMICA:
        # start with map degredations
        mapDegFile = 'smicaMapDeg.fits'
        maskDegFile = 'maskMapDeg.fits'
        if newDeg:
            # load maps; default files have 2048,NESTED,GALACTIC
            dataDir = '/Data/'
            if R1:
                smicaFile = 'COM_CompMap_CMB-smica-field-I_2048_R1.20.fits'
                maskFile = 'COM_Mask_CMB-union_2048_R1.10.fits'
            else:
                smicaFile = 'COM_CMB_IQU-smica-field-int_2048_R2.01_full.fits'
                maskFile = 'COM_CMB_IQU-common-field-MaskInt_2048_R2.01.fits'
            print 'opening file ', smicaFile, '... '
            smicaMap, smicaHead = hp.read_map(dataDir + smicaFile,
                                              nest=True,
                                              h=True)
            print 'opening file ', maskFile, '... '
            maskMap, maskHead = hp.read_map(dataDir + maskFile,
                                            nest=True,
                                            h=True)
            if R1:
                smicaMap *= 1e-6  #microK to K

            # degrade map and mask resolutions from 2048 to 128; convert NESTED to RING
            useAlm = True  # set to True to do harmonic space scaling, False for ud_grade
            NSIDE_big = 2048
            NSIDE_deg = 128
            while 4 * NSIDE_deg < lmax:
                NSIDE_deg *= 2
            print 'resampling maps at NSIDE = ', NSIDE_deg, '... '
            order_out = 'RING'
            if useAlm:
                # transform to harmonic space
                smicaMapRing = hp.reorder(smicaMap, n2r=True)
                maskMapRing = hp.reorder(maskMap, n2r=True)
                smicaCl, smicaAlm = hp.anafast(smicaMapRing,
                                               alm=True,
                                               lmax=lmax)
                maskCl, maskAlm = hp.anafast(maskMapRing, alm=True, lmax=lmax)
                # this gives 101 Cl values and 5151 Alm values.  Why not all 10201 Alm.s?

                # scale by pixel window functions
                bigWin = hp.pixwin(NSIDE_big)
                degWin = hp.pixwin(NSIDE_deg)
                winRatio = degWin / bigWin[:degWin.size]
                degSmicaAlm = hp.almxfl(smicaAlm, winRatio)
                degMaskAlm = hp.almxfl(maskAlm, winRatio)

                # re-transform back to real space
                smicaMapDeg = hp.alm2map(degSmicaAlm, NSIDE_deg)
                maskMapDeg = hp.alm2map(degMaskAlm, NSIDE_deg)

            else:
                smicaMapDeg = hp.ud_grade(smicaMap,
                                          nside_out=NSIDE_deg,
                                          order_in='NESTED',
                                          order_out=order_out)
                maskMapDeg = hp.ud_grade(maskMap,
                                         nside_out=NSIDE_deg,
                                         order_in='NESTED',
                                         order_out=order_out)
                # note: degraded resolution mask will no longer be only 0s and 1s.
                #   Should it be?  Yes.

            # turn smoothed mask back to 0s,1s mask
            threshold = 0.9
            maskMapDeg[np.where(maskMapDeg > threshold)] = 1
            maskMapDeg[np.where(maskMapDeg <= threshold)] = 0

            #testing
            #hp.mollview(smicaMapDeg)
            #plt.show()
            #hp.mollview(maskMapDeg)
            #plt.show()
            #return 0

            hp.write_map(mapDegFile, smicaMapDeg,
                         nest=False)  # use False if order_out='RING' above
            hp.write_map(maskDegFile, maskMapDeg, nest=False)

        else:  # just load previous degradations (dependent on previous lmax)
            print 'loading previously degraded map and mask...'
            smicaMapDeg = hp.read_map(mapDegFile, nest=False)
            maskMapDeg = hp.read_map(maskDegFile, nest=False)

        # find power spectra
        print 'find power spectra... '
        if useSPICE:
            ClFile1 = 'spiceCl_unmasked.fits'
            ClFile2 = 'spiceCl_masked.fits'

            # note: lmax for spice is 3*NSIDE-1 or less
            ispice(mapDegFile, ClFile1, subav="YES", subdipole="YES")
            Cl_unmasked = hp.read_cl(ClFile1)
            ispice(mapDegFile,
                   ClFile2,
                   maskfile1=maskDegFile,
                   subav="YES",
                   subdipole="YES")
            Cl_masked = hp.read_cl(ClFile2)
            Cl_mask = np.zeros(Cl_unmasked.shape[0])  # just a placeholder
            ell = np.arange(Cl_unmasked.shape[0])

        else:  # use anafast
            Cl_unmasked = hp.anafast(smicaMapDeg, lmax=lmax)
            Cl_masked = hp.anafast(smicaMapDeg * maskMapDeg, lmax=lmax)
            Cl_mask = hp.anafast(maskMapDeg, lmax=lmax)
            ell = np.arange(lmax + 1)  #anafast output seems to start at l=0

        # plot them
        doPlot = False  #True
        if doPlot:
            gcp.showCl(ell,
                       np.array([Cl_masked, Cl_unmasked]),
                       title='power spectra of unmasked, masked SMICA map')

        # Legendre transform to real space
        print 'Legendre transform to real space... '
        # note: getCovar uses linspace in x for thetaArray
        thetaDomain, CofTheta = getCovar(ell[:lmax + 1],
                                         Cl_unmasked[:lmax + 1],
                                         theta_i=theta_i,
                                         theta_f=theta_f,
                                         nSteps=nSteps,
                                         lmin=lmin)
        thetaDomain, CCutofThetaTA = getCovar(ell[:lmax + 1],
                                              Cl_masked[:lmax + 1],
                                              theta_i=theta_i,
                                              theta_f=theta_f,
                                              nSteps=nSteps,
                                              lmin=lmin)
        CofTheta *= 1e12  # K^2 to microK^2
        CCutofThetaTA *= 1e12  # K^2 to microK^2

        if useSPICE:
            CCutofTheta = CCutofThetaTA  #/(4*np.pi)
        else:
            thetaDomain, AofThetaInverse = getCovar(
                ell[:lmax + 1],
                Cl_mask[:lmax + 1],
                theta_i=theta_i,
                theta_f=theta_f,
                nSteps=nSteps,
                lmin=0)  # don't zilch the mask
            # note: zilching the mask's low power drastically changed C(theta) for masked anafast
            #   Not sure why.
            CCutofTheta = CCutofThetaTA / AofThetaInverse

        xArray = np.cos(thetaDomain * np.pi / 180.)

        # back to frequency space for S_{1/2} = CIC calculation
        if useSPICE:
            CCutofL = Cl_masked[:lmax + 1] * 1e12  #K^2 to microK^2
        else:
            legCoefs = legfit(xArray, CCutofTheta, lmax)
            CCutofL = legCoefs * (4 * np.pi) / (2 * ell[:lmax + 1] + 1)

        # S_{1/2}
        myJmn = getJmn(lmax=lmax)
        SMasked = np.dot(CCutofL[lmin:],
                         np.dot(myJmn[lmin:, lmin:], CCutofL[lmin:]))
        SNoMask = np.dot(
            Cl_unmasked[lmin:lmax + 1],
            np.dot(myJmn[lmin:, lmin:],
                   Cl_unmasked[lmin:lmax +
                               1])) * 1e24  #two factors of K^2 to muK^2

        # save results
        if useSPICE:
            np.save(
                saveFile2,
                np.array(
                    [thetaDomain, CofTheta, CCutofTheta, SNoMask, SMasked]))
        else:
            np.save(
                saveFile,
                np.array(
                    [thetaDomain, CofTheta, CCutofTheta, SNoMask, SMasked]))

    else:  # load from file
        if useSPICE:
            fileData = np.load(saveFile2)
        else:
            fileData = np.load(saveFile)
        thetaDomain = fileData[0]
        CofTheta = fileData[1]
        CCutofTheta = fileData[2]
        SNoMask = fileData[3]
        SMasked = fileData[4]

    return thetaDomain, CofTheta, CCutofTheta, SNoMask, SMasked
Esempio n. 5
0
def test(useCLASS=1,
         useLensing=1,
         classCamb=1,
         nSims=1000,
         lmax=100,
         lmin=2,
         newSMICA=False,
         newDeg=False,
         suppressC2=False,
         suppFactor=0.23):
    """
    code for testing the other functions in this module
    Inputs:
      useCLASS: set to 1 to use CLASS, 0 to use CAMB
        CLASS Cl has early/late split at z=50
        CAMB Cl has ISWin/out split: ISWin: 0.4<z<0.75, ISWout: the rest
        Note: CAMB results include primary in ISWin and ISWout (not as intended)
        default: 1
      useLensing: set to 1 to use lensed Cl, 0 for non-lensed
        default: 1
      classCamb: if 1: use the CAMB format of CLASS output, if 0: use CLASS format
        Note: parameter not used if useCLASS = 0
        default: 1
      nSims: the number of simulations to do for ensemble
        default: 1000
      lmax: the highest l to include in Legendre transforms
        default: 100
      lmin: the lowest l to include in S_{1/2} = CIC calculations
        default: 2
      newSMICA: set to True to recalculate SMICA results
        default: False
      newDeg: set to True to recalculate map and mask degredations
        (only if newSMICA is also True)
        default: False
      suppressC2: set to True to suppress theoretical C_2 by suppFactor
        before creating a_lm.s
        Default: False
      suppFactor: multiplies C_2 if suppressC2 is True
        Default: 0.23 # from Tegmark et. al. 2003, figure 13 (WMAP)
  """

    ##############################################################################
    # load theoretical power spectra

    # load data
    ell, fullCl, primCl, lateCl, crossCl = gcp.loadCls(useCLASS=useCLASS,
                                                       useLensing=useLensing,
                                                       classCamb=classCamb)

    # fill beginning with zeros
    startEll = int(ell[0])
    ell = np.append(np.arange(startEll), ell)
    fullCl = np.append(np.zeros(startEll), fullCl)
    primCl = np.append(np.zeros(startEll), primCl)
    lateCl = np.append(np.zeros(startEll), lateCl)
    crossCl = np.append(np.zeros(startEll), crossCl)

    # suppress C_2 to see what happens in enesmble
    #suppressC2 = False
    #suppFactor = 0.23 # from Tegmark et. al. 2003, figure 13 (WMAP)
    if suppressC2:
        fullCl[2] *= suppFactor
        primCl[2] *= suppFactor
        lateCl[2] *= suppFactor
        crossCl[2] *= suppFactor

    conv = ell * (ell + 1) / (2 * np.pi)
    #print ell,conv #ell[0]=2.0

    # apply beam and pixel window functions to power spectra
    #   note: to ignore the non-constant pixel shape, W(l) must be > B(l)
    #     however, this is not true for NSIDE=128 and gauss_beam(5')
    #   Here I ignore this anyway and proceed
    myNSIDE = 128  # must be same NSIDE as in sims.getSMICA function
    Wpix = hp.pixwin(myNSIDE)
    Bsmica = hp.gauss_beam(5. / 60 * np.pi / 180)  # 5 arcmin
    WlMax = Wpix.size
    if WlMax < lmax:
        print 'die screaming!!!'
        return 0
    fullCl = fullCl[:WlMax] * (Wpix * Bsmica)**2
    primCl = primCl[:WlMax] * (Wpix * Bsmica)**2
    lateCl = lateCl[:WlMax] * (Wpix * Bsmica)**2
    crossCl = crossCl[:WlMax] * (Wpix * Bsmica)**2
    # note: i tried sims without this scaling, and results seemed the same at a glance

    ##############################################################################
    # load SMICA data, converted to C(theta), via SpICE

    if newSMICA:
        theta_i = 0.0  #degrees
        theta_f = 180.0  #degrees
        nSteps = 1800
        thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp, S_SMICAnomasksp, S_SMICAmaskedsp = \
          sims.getSMICA(theta_i=theta_i,theta_f=theta_f,nSteps=nSteps,lmax=lmax,lmin=lmin,
                   newSMICA=newSMICA,newDeg=newDeg,useSPICE=True)

    # filenames for SpICE to use
    # super lame that spice needs to read/write from disk, but here goes...
    RAMdisk = '/Volumes/ramdisk/'
    ClTempFile = RAMdisk + 'tempCl.fits'
    mapTempFile = RAMdisk + 'tempMap.fits'
    mapDegFile = RAMdisk + 'smicaMapDeg.fits'  # this should have been created by sims.getSMICA
    maskDegFile = RAMdisk + 'maskMapDeg.fits'  # this should have been created by sims.getSMICA

    # create RAM Disk for SpICE and copy these files there using bash
    RAMsize = 4  #Mb
    ramDiskOutput = subprocess.check_output('./ramdisk.sh create ' +
                                            str(RAMsize),
                                            shell=True)
    print ramDiskOutput
    diskID = ramDiskOutput[
        31:41]  # this might not grab the right part; works for '/dev/disk1'
    subprocess.call('cp smicaMapDeg.fits ' + RAMdisk, shell=True)
    subprocess.call('cp maskMapDeg.fits ' + RAMdisk, shell=True)

    ispice(mapDegFile,
           ClTempFile,
           maskfile1=maskDegFile,
           subav="YES",
           subdipole="YES")
    ClsmicaCut = hp.read_cl(ClTempFile)

    # find S_{1/2} for SMICA.  Should actually optimize but see what happens here first.
    #myJmn = legprodint.getJmn(endX=0.5,lmax=lmax,doSave=False)
    #Ssmica = np.dot(ClsmicaCut[lmin:lmax+1],np.dot(myJmn[lmin:,lmin:],
    #                ClsmicaCut[lmin:lmax+1]))*1e24 #K^4 to microK^4

    ##############################################################################
    # create ensemble of realizations and gather statistics

    spiceMax = myNSIDE * 3  # should be lmax+1 for SpICE
    ClEnsembleCut = np.zeros([nSims, spiceMax])
    simEll = np.arange(spiceMax)

    doTime = True  # to time the run and print output
    startTime = time.time()
    for nSim in range(nSims):
        print 'starting masked Cl sim ', nSim + 1, ' of ', nSims
        alm_prim, alm_late = hp.synalm((primCl, lateCl, crossCl),
                                       lmax=lmax,
                                       new=True)
        mapSim = hp.alm2map(alm_prim + alm_late, myNSIDE, lmax=lmax)
        hp.write_map(mapTempFile, mapSim)

        ispice(mapTempFile,
               ClTempFile,
               maskfile1=maskDegFile,
               subav="YES",
               subdipole="YES")
        ClEnsembleCut[nSim] = hp.read_cl(ClTempFile)

        doPlot = False  #True
        if doPlot:
            gcp.showCl(simEll[:lmax + 1],
                       ClEnsembleCut[nSim, :lmax + 1],
                       title='power spectrum of simulation ' + str(nSim + 1))

    timeInterval1 = time.time() - startTime
    if doTime: print 'time elapsed: ', int(timeInterval1 / 60.), ' minutes'

    # free the RAM used by SpICE's RAM disk
    ramDiskOutput = subprocess.check_output('./ramdisk.sh delete ' + diskID,
                                            shell=True)
    print ramDiskOutput

    # put SMICA in as 0th member of the ensemble; 1e12 to convert K^2 to microK^2
    ClEnsembleCut = np.vstack((ClsmicaCut * 1e12, ClEnsembleCut))
    nSims += 1

    ##############################################################################
    # create S(x) for each C_l, using interpolation

    nXvals = 181
    thetaVals = np.linspace(0, 180, nXvals)  # one degree intervals
    xVals = np.cos(thetaVals * np.pi / 180)
    Jmnx = np.empty([nXvals, lmax + 1, lmax + 1])
    for index, xVal in enumerate(xVals):
        Jmnx[index] = legprodint.getJmn(endX=xVal, lmax=lmax, doSave=False)
    SxToInterpolate = np.empty(nXvals)

    # create list of functions
    dummy = lambda x: x**2
    SofXList = [dummy for i in range(nSims)]

    for nSim in range(nSims):
        print 'starting S(x) sim ', nSim + 1, ' of ', nSims
        for index, xVal in enumerate(xVals):
            SxToInterpolate[index] = np.dot(
                ClEnsembleCut[nSim, lmin:lmax + 1],
                np.dot(Jmnx[index, lmin:, lmin:],
                       ClEnsembleCut[nSim, lmin:lmax + 1]))
        SofX = interp1d(xVals, SxToInterpolate)
        #SofXList = SofXList.append(SofX)
        # Apparently appending a function to an empty list is not allowed. Instead:
        SofXList[nSim] = SofX

        #print SofXList#[nSim]
        doPlot = False  #True
        if doPlot:
            nplotx = (nXvals - 1) * 10 + 1
            plotTheta = np.linspace(0, 180, nplotx)
            plotx = np.cos(plotTheta * np.pi / 180)
            plotS = SofXList[nSim](plotx)
            plt.plot(plotx, plotS)
            plt.title('S(x) for simulation ' + str(nSim + 1))
            plt.show()

    doPlot = True
    if doPlot:
        for nSim in range(nSims):
            nplotx = (nXvals - 1) * 10 + 1
            plotTheta = np.linspace(0, 180, nplotx)
            plotx = np.cos(plotTheta * np.pi / 180)
            plotS = SofXList[nSim](plotx)
            plt.plot(plotx, plotS, label='sim ' + str(nSim + 1))
        #plt.legend()
        plt.title('S(x) for ' + str(nSims) + ' simulations')
        plt.xlabel('x')
        plt.ylabel('S_x')
        plt.show()

    ##############################################################################
    # create Pval(x) for each S(x), using ensemble
    # Pval: probability of result equal to or more extreme

    # create list of functions
    PvalOfXList = [dummy for i in range(nSims)]

    for nSim in range(nSims):
        print 'starting Pval(x) sim ', nSim + 1, ' of ', nSims

        def PvalOfX(x):
            nUnder = 0  # will also include nEqual
            nOver = 0
            threshold = SofXList[nSim](x)
            for nSim2 in range(nSims):
                Sx = SofXList[nSim2](x)
                if Sx > threshold:
                    nOver += 1
                    #print "Over! mySx: ",Sx,", threshold: ",threshold
                else:
                    nUnder += 1
                    #print "Under! mySx: ",Sx,", threshold: ",threshold
            #print "nUnder: ",nUnder,", nOver: ",nOver
            return nUnder / float(nUnder + nOver)

        PvalOfXList[nSim] = PvalOfX

    ##############################################################################
    # find global minimum for each Pval(x)
    # simply use same xVals as above, at one degree intervals
    # if there are equal p-values along the range, the one with the highest xVal
    #   will be reported

    PvalMinima = np.empty(nSims)
    xValMinima = np.empty(nSims)

    doTime = True  # to time the run and print output
    startTime = time.time()
    for nSim in range(nSims):
        print 'starting minimum Pval(x) search for sim ', nSim + 1, ' of ', nSims
        PvalOfX = PvalOfXList[nSim]
        #print 'function: ',PvalOfX
        PvalMinima[nSim] = PvalOfX(1.0)
        xValMinima[nSim] = 1.0

        Pvals = np.empty(nXvals)
        for index, xVal in enumerate(
                xVals):  # will start from 1 and go down to -1
            myPval = PvalOfX(xVal)
            Pvals[index] = myPval
            #print "nSim: ",nSim,", n: ",index,", myPval: ",myPval,", PvalMinima[nSim]: ",PvalMinima[nSim]
            if myPval < PvalMinima[
                    nSim] and xVal > -0.999:  #avoid the instabililility
                PvalMinima[nSim] = myPval
                xValMinima[nSim] = xVal
                #print 'nSim: ',nSim+1,', new x for minimum Pval: ',xVal
        #raw_input("Finished sim "+str(nSim+1)+" of "+str(nSims)+".  Press enter to continue")

        doPlot = True  #False#True
        if doPlot:  # and np.random.uniform() < 0.1: #randomly choose about 1/10 of them
            plt.plot(xVals, Pvals)
            plt.vlines(xValMinima[nSim], 0, 1)
            plt.xlabel('x = cos(theta), min at ' + str(xValMinima[nSim]))
            plt.ylabel('P-value')
            plt.title('P-values for simulation ' + str(nSim + 1) + ' of ' +
                      str(nSims) + ', p_min = ' + str(PvalMinima[nSim]))
            plt.xlim(-1.05, 1.05)
            plt.ylim(-0.05, 1.05)
            plt.show()

    timeInterval2 = time.time() - startTime
    if doTime: print 'time elapsed: ', int(timeInterval2 / 60.), ' minutes'
    """
  # A MYSTERY!  Something about the following code causes Pvals to always take 
  #   the values of PvalOfXList[nSims](xVals)  WTF?  Omit for now. 
  #   Testing seems to indicate that PvalOfXList functions still have different
  #   locations in memory, but they all seem to be evaluating the same.
  #   However, when the previous block of code is copied to come again after
  #   this one, it behaves properly again.
  # see how well it did
  doPlot = False#True
  if doPlot:
    nPlots = 10
    for nPlot in range(nPlots):
      print 'plot ',nPlot+1,' of ',nPlots
      toPlot = nPlot#np.random.randint(0,high=nSims)
      #for nSim in range(nSims):
      Pvals = np.empty(nXvals)
      PvalOfX = PvalOfXList[nPlot]
      print 'function: ',PvalOfX
      for index, xVal in enumerate(xVals):
        Pvals[index] = PvalOfX(xVal)
        #print index,Pvals[index]
      #print Pvals
      plt.plot(xVals,Pvals)
      plt.vlines(xValMinima[toPlot],0,1)
      plt.xlabel('x = cos(theta), min at '+str(xValMinima[toPlot]))
      plt.ylabel('P-value')
      plt.title('P-values for simulation '+str(toPlot+1)+' of '+str(nSims))
      plt.show()
  """

    ##############################################################################
    # create distribution of S(xValMinima)

    SxEnsembleMin = np.empty(nSims)
    for nSim in range(nSims):
        SxEnsembleMin[nSim] = SofXList[nSim](xValMinima[nSim])

    # extract SMICA result
    Ssmica = SxEnsembleMin[0]

    ##############################################################################
    # plot/print results

    print 'plotting S_x distribution... '
    myBins = np.logspace(1, 7, 100)
    plt.axvline(x=Ssmica, color='g', linewidth=3, label='SMICA masked')
    plt.hist(SxEnsembleMin[1:], bins=myBins, histtype='step', label='cut sky')
    # [1:] to omit SMICA value

    plt.gca().set_xscale("log")
    plt.legend()
    plt.xlabel('S_x (microK^4)')
    plt.ylabel('Counts')
    plt.title('S_x of ' + str(nSims - 1) +
              ' simulated CMBs')  #-1 due to SMICA in zero position
    plt.show()

    print ' '
    print 'nSims = ', nSims - 1
    print 'time interval 1: ', timeInterval1, 's, time interval 2: ', timeInterval2, 's'
    print '  => ', timeInterval1 / (nSims - 1), ' s/sim, ', timeInterval2 / (
        nSims - 1), ' s/sim'
    print 'SMICA optimized S_x: S = ',Ssmica,', for x = ',xValMinima[0], \
          ', with p-value ',PvalMinima[0]
    print ' '

    print 'step 3: profit'
    print ''
def test(useCLASS=1,
         useLensing=1,
         classCamb=1,
         nSims=1000,
         lmax=100,
         lmin=2,
         newSMICA=False,
         newDeg=False,
         suppressC2=False,
         suppFactor=0.23,
         filterC2=False,
         filtFacLow=0.1,
         filtFacHigh=0.2,
         doCovar=False):
    """
    code for testing the other functions in this module
    Inputs:
      useCLASS: set to 1 to use CLASS, 0 to use CAMB
        CLASS Cl has early/late split at z=50
        CAMB Cl has ISWin/out split: ISWin: 0.4<z<0.75, ISWout: the rest
        Note: CAMB results include primary in ISWin and ISWout (not as intended)
        default: 1
      useLensing: set to 1 to use lensed Cl, 0 for non-lensed
        default: 1
      classCamb: if 1: use the CAMB format of CLASS output, if 0: use CLASS format
        Note: parameter not used if useCLASS = 0
        default: 1
      nSims: the number of simulations to do for ensemble
        default: 1000
      lmax: the highest l to include in Legendre transforms
        default: 100
      lmin: the lowest l to include in S_{1/2} = CIC calculations
        default: 2
      newSMICA: set to True to recalculate SMICA results
        default: False
      newDeg: set to True to recalculate map and mask degredations
        (only if newSMICA is also True)
        default: False
      suppressC2: set to True to suppress theoretical C_2 (quadrupole) by 
        suppFactor before creating a_lm.s
        Default: False
      suppFactor: multiplies C_2 if suppressC2 is True
        Default: 0.23 # from Tegmark et. al. 2003, figure 13 (WMAP)
      filterC2 : set to true to filter simulated CMBs after spice calculates
        cut sky C_l.  Sims will pass filter if C_2 * filtFacLow < C_2^sim <
        C_2 * filtFacHigh.
        Default: False
      filtFacLow,filtFacHigh: defines C_2 range for passing simulated CMBs
        Default: 0.1,0.2
      doCovar: set to True to calculate C(theta) and S_{1/2} distritutions for ensemble
        Note: meant to capture functionality from sim_stats.py; ZK 2016.11.13
        Default: False
  """

    ##############################################################################
    # load theoretical power spectra

    # load data
    ell, fullCl, primCl, lateCl, crossCl = gcp.loadCls(useCLASS=useCLASS,
                                                       useLensing=useLensing,
                                                       classCamb=classCamb)

    # fill beginning with zeros
    startEll = int(ell[0])
    ell = np.append(np.arange(startEll), ell)
    fullCl = np.append(np.zeros(startEll), fullCl)
    primCl = np.append(np.zeros(startEll), primCl)
    lateCl = np.append(np.zeros(startEll), lateCl)
    crossCl = np.append(np.zeros(startEll), crossCl)

    # suppress C_2 to see what happens in enesmble
    if suppressC2:
        fullCl[2] *= suppFactor
        primCl[2] *= suppFactor
        lateCl[2] *= suppFactor
        crossCl[2] *= suppFactor

    conv = ell * (ell + 1) / (2 * np.pi)
    #print ell,conv #ell[0]=2.0

    # apply beam and pixel window functions to power spectra
    #   note: to ignore the non-constant pixel shape, W(l) must be > B(l)
    #     however, this is not true for NSIDE=128 and gauss_beam(5')
    #   Here I ignore this anyway and proceed
    myNSIDE = 128  # must be same NSIDE as in sims.getSMICA function
    Wpix = hp.pixwin(myNSIDE)
    Bsmica = hp.gauss_beam(5. / 60 * np.pi / 180)  # 5 arcmin
    WlMax = Wpix.size
    if WlMax < lmax:
        print 'die screaming!!!'
        return 0
    fullCl = fullCl[:WlMax] * (Wpix * Bsmica)**2
    primCl = primCl[:WlMax] * (Wpix * Bsmica)**2
    lateCl = lateCl[:WlMax] * (Wpix * Bsmica)**2
    crossCl = crossCl[:WlMax] * (Wpix * Bsmica)**2
    # note: i tried sims without this scaling, and results seemed the same at a glance

    ##############################################################################
    # load SMICA data, converted to C_l, via SpICE

    if newSMICA or doCovar:
        theta_i = 0.0  #degrees
        theta_f = 180.0  #degrees
        nSteps = 1800
        thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp, S_SMICAnomasksp, S_SMICAmaskedsp = \
          sims.getSMICA(theta_i=theta_i,theta_f=theta_f,nSteps=nSteps,lmax=lmax,lmin=lmin,
                   newSMICA=newSMICA,newDeg=newDeg,useSPICE=True)

    # filenames for SpICE to use
    # super lame that spice needs to read/write from disk, but here goes...
    RAMdisk = '/Volumes/ramdisk/'
    ClTempFile = RAMdisk + 'tempCl.fits'
    mapTempFile = RAMdisk + 'tempMap.fits'
    mapDegFile = RAMdisk + 'smicaMapDeg.fits'  # this should have been created by sims.getSMICA
    maskDegFile = RAMdisk + 'maskMapDeg.fits'  # this should have been created by sims.getSMICA

    # create RAM Disk for SpICE and copy these files there using bash
    RAMsize = 4  #Mb
    ramDiskOutput = subprocess.check_output('./ramdisk.sh create ' +
                                            str(RAMsize),
                                            shell=True)
    print ramDiskOutput
    diskID = ramDiskOutput[
        31:41]  # this might not grab the right part; works for '/dev/disk1'
    subprocess.call('cp smicaMapDeg.fits ' + RAMdisk, shell=True)
    subprocess.call('cp maskMapDeg.fits ' + RAMdisk, shell=True)

    ispice(mapDegFile,
           ClTempFile,
           maskfile1=maskDegFile,
           subav="YES",
           subdipole="YES")
    ClsmicaCut = hp.read_cl(ClTempFile)

    # find S_{1/2} for SMICA.  Should actually optimize but see what happens here first.
    if doCovar:
        myJmn = legprodint.getJmn(endX=0.5, lmax=lmax, doSave=False)
        #Ssmica = np.dot(ClsmicaCut[lmin:lmax+1],np.dot(myJmn[lmin:,lmin:],
        #                ClsmicaCut[lmin:lmax+1]))*1e24 #K^4 to microK^4

    ##############################################################################
    # create ensemble of realizations and gather statistics

    spiceMax = myNSIDE * 3  # should be lmax+1 for SpICE
    ClEnsembleCut = np.zeros([nSims, spiceMax])
    if doCovar:
        ClEnsembleFull = np.zeros([nSims, lmax + 1])
    simEll = np.arange(spiceMax)

    # option for creating C(\theta) and S_{1/2} ensembles
    if doCovar:
        cEnsembleCut = np.zeros([nSims, nSteps + 1])
        cEnsembleFull = np.zeros([nSims, nSteps + 1])
        sEnsembleCut = np.zeros(nSims)
        sEnsembleFull = np.zeros(nSims)

    doTime = True  # to time the run and print output
    startTime = time.time()
    #for nSim in range(nSims):
    nSim = 0
    while nSim < nSims:
        print 'starting masked Cl sim ', nSim + 1, ' of ', nSims
        alm_prim, alm_late = hp.synalm((primCl, lateCl, crossCl),
                                       lmax=lmax,
                                       new=True)
        mapSim = hp.alm2map(alm_prim + alm_late, myNSIDE, lmax=lmax)
        hp.write_map(mapTempFile, mapSim)
        if doCovar:
            ClEnsembleFull[nSim] = hp.alm2cl(alm_prim + alm_late)

        ispice(mapTempFile,
               ClTempFile,
               maskfile1=maskDegFile,
               subav="YES",
               subdipole="YES")
        ClEnsembleCut[nSim] = hp.read_cl(ClTempFile)

        # Check for low power of cut sky C_2
        if (filterC2 == True
                and fullCl[2] * filtFacHigh > ClEnsembleCut[nSim, 2]
                and ClEnsembleCut[nSim, 2] > fullCl[2] * filtFacLow
            ) or filterC2 == False:

            doPlot = False  #True
            if doPlot:
                gcp.showCl(simEll[:lmax + 1],
                           ClEnsembleCut[nSim, :lmax + 1],
                           title='power spectrum of simulation ' +
                           str(nSim + 1))

            if doCovar:
                #   note: getCovar uses linspace in x for thetaArray
                thetaArray, cArray = sims.getCovar(simEll[:lmax + 1],
                                                   ClEnsembleCut[nSim, :lmax +
                                                                 1],
                                                   theta_i=theta_i,
                                                   theta_f=theta_f,
                                                   nSteps=nSteps,
                                                   lmin=lmin)
                cEnsembleCut[nSim] = cArray
                thetaArray, cArray = sims.getCovar(simEll[:lmax + 1],
                                                   ClEnsembleFull[nSim, :lmax +
                                                                  1],
                                                   theta_i=theta_i,
                                                   theta_f=theta_f,
                                                   nSteps=nSteps,
                                                   lmin=lmin)
                cEnsembleFull[nSim] = cArray

                # S_{1/2}
                sEnsembleCut[nSim] = np.dot(
                    ClEnsembleCut[nSim, lmin:lmax + 1],
                    np.dot(myJmn[lmin:, lmin:], ClEnsembleCut[nSim,
                                                              lmin:lmax + 1]))
                sEnsembleFull[nSim] = np.dot(
                    ClEnsembleFull[nSim, lmin:lmax + 1],
                    np.dot(myJmn[lmin:, lmin:], ClEnsembleFull[nSim,
                                                               lmin:lmax + 1]))

            nSim += 1

    timeInterval1 = time.time() - startTime
    if doTime: print 'time elapsed: ', int(timeInterval1 / 60.), ' minutes'

    # free the RAM used by SpICE's RAM disk
    ramDiskOutput = subprocess.check_output('./ramdisk.sh delete ' + diskID,
                                            shell=True)
    print ramDiskOutput

    # put SMICA in as 0th member of the ensemble; 1e12 to convert K^2 to microK^2
    ClEnsembleCut = np.vstack((ClsmicaCut * 1e12, ClEnsembleCut))
    nSims += 1

    ##############################################################################
    # create S(x) for each C_l, using interpolation

    nXvals = 181
    thetaVals = np.linspace(0, 180, nXvals)  # one degree intervals
    xVals = np.cos(thetaVals * np.pi / 180)

    Jmnx = np.empty([nXvals, lmax + 1, lmax + 1])
    for index, xVal in enumerate(xVals):
        Jmnx[index] = legprodint.getJmn(endX=xVal, lmax=lmax, doSave=False)
    SxToInterpolate = np.empty(nXvals)

    # create list of functions
    #dummy = lambda x: x**2
    #SofXList = [dummy for i in range(nSims)]

    # here is where this program starts to diverge from the purely python version
    # create array to hold S_x values
    SxValsArray = np.empty([nSims, nXvals])

    for nSim in range(nSims):
        print 'starting S(x) sim ', nSim + 1, ' of ', nSims
        for index, xVal in enumerate(xVals):  #not using xVal?
            SxToInterpolate[index] = np.dot(
                ClEnsembleCut[nSim, lmin:lmax + 1],
                np.dot(Jmnx[index, lmin:, lmin:],
                       ClEnsembleCut[nSim, lmin:lmax + 1]))
        #SofX = interp1d(xVals,SxToInterpolate)
        #SofXList[nSim] = SofX

        SxValsArray[nSim] = SxToInterpolate
    """
    #print SofXList#[nSim]
    doPlot=False#True
    if doPlot:
      nplotx = (nXvals-1)*10+1
      plotTheta = np.linspace(0,180,nplotx)
      plotx = np.cos(plotTheta*np.pi/180)
      plotS = SofXList[nSim](plotx)
      plt.plot(plotx,plotS)
      plt.title('S(x) for simulation '+str(nSim+1))
      plt.show()

  doPlot = False#True
  if doPlot:
    for nSim in range(nSims):
      nplotx = (nXvals-1)*10+1
      plotTheta = np.linspace(0,180,nplotx)
      plotx = np.cos(plotTheta*np.pi/180)
      plotS = SofXList[nSim](plotx)
      plt.plot(plotx,plotS,label='sim '+str(nSim+1))
      #plt.plot(xVals,SxValsArray[nSim],label='sim '+str(nSim+1))
    #plt.legend()
    plt.title('S(x) for '+str(nSims)+ 'simulations')
    plt.xlabel('x')
    plt.ylabel('S_x')
    plt.show()
  """

    # Kludge for extracting the S(x) ensemble to disk for Jackknife testing later
    saveAndExit = False  #True
    saveAndExitFile = 'SofXEnsemble.npy'
    if saveAndExit:
        np.save(saveAndExitFile, np.vstack((xVals, SxValsArray)))
        print 'saving file ', saveAndExitFile, ' and exiting.'
        return 0

    ##############################################################################
    # send data to c library function in optimizeSx.so

    xStart = -1.0
    xEnd = 1.0
    nSearch = 181  # same num as nXvals for now, but spaced equally in x, not theta
    PvalMinima = np.empty(nSims)  # for return values
    XvalMinima = np.empty(nSims)  # for return values

    doTime = True  # to time the run and print output
    startTime = time.time()
    optSx(xVals, nXvals, SxValsArray, nSims, xStart, xEnd, nSearch, PvalMinima,
          XvalMinima)
    timeInterval2 = time.time() - startTime
    if doTime: print 'time elapsed: ', int(timeInterval2 / 60.), ' minutes'

    ##############################################################################
    # create distribution of S(XvalMinima)

    SxEnsembleMin = np.empty(nSims)
    for nSim in range(nSims):
        # need to interpolate since optSx uses interpolation
        SofX = interp1d(xVals, SxValsArray[nSim])
        SxEnsembleMin[nSim] = SofX(XvalMinima[nSim])

    ##############################################################################
    # save S_x, P(x), x results
    saveFile = "optSxResult.npy"
    np.save(saveFile, np.vstack((PvalMinima, XvalMinima, SxEnsembleMin)))
    saveFileC2 = "optSxC2.npy"
    np.save(saveFileC2, ClEnsembleCut[:, 2])  #for C_2

    # save C(theta) and S{1/2} results
    if doCovar:
        avgEnsembleFull = np.average(cEnsembleFull, axis=0)
        stdEnsembleFull = np.std(cEnsembleFull, axis=0)
        # do I need a better way to describe confidence interval?
        avgEnsembleCut = np.average(cEnsembleCut, axis=0)
        stdEnsembleCut = np.std(cEnsembleCut, axis=0)

        saveFile1 = "simStatResultC.npy"
        np.save(
            saveFile1,
            np.vstack((thetaArray, avgEnsembleFull, stdEnsembleFull,
                       avgEnsembleCut, stdEnsembleCut)))
        saveFile2 = "simStatC_SMICA.npy"
        np.save(saveFile2,
                np.vstack((thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp)))

        saveFile3 = "simStatResultS.npy"
        np.save(
            saveFile3,
            np.vstack((np.hstack((np.array(S_SMICAnomasksp), sEnsembleFull)),
                       np.hstack((np.array(S_SMICAmaskedsp), sEnsembleCut)))))

    ##############################################################################
    # plot/print results
    makePlots(saveFile=saveFile, suppressC2=suppressC2)
    #makeCornerPlot(saveFile=saveFile,suppressC2=suppressC2)
    makeCornerPlotSmall(saveFile=saveFile, suppressC2=suppressC2)
    c2pval = makeC2Plot(saveFile=saveFileC2)
    if doCovar:
        sims.makePlots(saveFile1=saveFile1,
                       saveFile2=saveFile2,
                       saveFile3=saveFile3)

    pv = PvalPval(saveFile=saveFile)
    print ' '
    print 'nSims = ', nSims - 1
    print 'time interval 1: ', timeInterval1, 's, time interval 2: ', timeInterval2, 's'
    print '  => ', timeInterval1 / (nSims - 1), ' s/sim, ', timeInterval2 / (
        nSims - 1), ' s/sim'
    print 'SMICA optimized S_x: S = ',SxEnsembleMin[0],', for x = ',XvalMinima[0], \
          ', with p-value ',PvalMinima[0]
    print 'P-value of P-value for SMICA: ', pv
    print ' '
    print 'p-value of C_2^SMICA in distribution: ', c2pval
    print ' '

    print 'step 3: profit'
    print ''
def test(nSims=100,
         lmax=100,
         lmin=2,
         partialMax=4,
         useCLASS=1,
         useLensing=1,
         cutSky=True,
         myNSIDE=128,
         newSC2=True,
         saveFile='simpleSonehalfC2.npy',
         nGrid=100):
    """
    Purpose:
      function for testing S_{1/2} calculations
    Inputs:
      nSims: the number of simulations to do
        Overriden if newSC2 = False
        Default: 100
      lmax: the highest l to use in the calculation
        Default: 100
      lmin: the lowest l to use in the calculation
        Default: 2
      partialMax: the maximum l to use for partial Sonehalf plots
        must be more than lmin
        Overriden if newSC2 = False
        Default: 4
      useCLASS: set to 1 to use CLASS Cl, 0 for CAMB
        Default: 1
      useLensing: set to 1 to use lensed Cls
        Default: 1
      cutSky: set to True to do cut-sky sims
        Default: True
      myNSIDE: HEALPix parameter for simulated maps if cutSky=True
        Default: 128
      newSC2: set to True to simulate new ensemble and save S,C2 results 
        in file, False to skip simulation and load previous results
        If false, values of nSims and partialMax will come from file
        Default: True
      saveFile: filename to save S,C2 result if newSC2 is true, to load if false
        Default: 'simpleSonehalfC2.npy'
      nGrid: to pass to plot2Ddist; controls grid for binning for contours
        Default: 100
  """
    # get power spectrum
    # starts with ell[0]=2
    ell, fullCl, primCl, lateCl, crossCl = gcp.loadCls(useCLASS=useCLASS,
                                                       useLensing=useLensing)

    # fill beginning with zeros
    startEll = int(ell[0])
    ell = np.append(np.arange(startEll), ell)
    Cl = np.append(np.zeros(startEll), fullCl)
    #conv = ell*(ell+1)/(2*np.pi)

    # Note: optimizeSx2 includes a multiplication of Cl by (beam*window)**2 at this point,
    #   but in this program I'm omitting it.  Why?  Effects are small, esp. at low ell

    # get Jmn matrix for harmonic space S_{1/2} calc.
    myJmn = getJmn(lmax=lmax)  # do not include monopole, dipole

    if cutSky:
        # yeah.. disk access is annoying so...
        RAMdisk = '/Volumes/ramdisk/'
        ClTempFile = RAMdisk + 'tempCl.fits'
        mapTempFile = RAMdisk + 'tempMap.fits'
        mapDegFile = RAMdisk + 'smicaMapDeg.fits'  #created by sim_stats.getSMICA
        maskDegFile = RAMdisk + 'maskMapDeg.fits'  #created by sim_stats.getSMICA

        # create RAM Disk for SpICE and copy these files there using bash
        RAMsize = 4  #Mb
        ramDiskOutput = subprocess.check_output('./ramdisk.sh create ' +
                                                str(RAMsize),
                                                shell=True)
        print ramDiskOutput
        diskID = ramDiskOutput[
            31:
            41]  # this might not grab the right part; works for '/dev/disk1'
        subprocess.call('cp smicaMapDeg.fits ' + RAMdisk, shell=True)
        subprocess.call('cp maskMapDeg.fits ' + RAMdisk, shell=True)

        ispice(mapDegFile,
               ClTempFile,
               maskfile1=maskDegFile,
               subav="YES",
               subdipole="YES")
        Clsmica = hp.read_cl(ClTempFile)
    else:
        ClTempFile = 'tempCl.fits'
        mapTempFile = 'tempMap.fits'
        mapDegFile = 'smicaMapDeg.fits'  #created by sim_stats.getSMICA
        maskDegFile = 'maskMapDeg.fits'  #created by sim_stats.getSMICA
        ispice(mapDegFile, ClTempFile, subav="YES", subdipole="YES")
        Clsmica = hp.read_cl(ClTempFile)

    # collect results
    if newSC2:
        sEnsemblePartial = np.zeros([nSims, partialMax + 1])
        C2Ensemble = np.zeros(nSims)
        for i in range(nSims):
            print "starting sim ", i + 1, " of ", nSims, "... "

            almSim = hp.synalm(Cl, lmax=lmax)  # should start with ell[0] = 0
            if cutSky:
                mapSim = hp.alm2map(almSim, myNSIDE, lmax=lmax)
                hp.write_map(mapTempFile, mapSim)
                ispice(mapTempFile,
                       ClTempFile,
                       maskfile1=maskDegFile,
                       subav="YES",
                       subdipole="YES")
                ClSim = hp.read_cl(ClTempFile)
            else:
                ClSim = hp.alm2cl(almSim)

            for myLmin in range(lmin, partialMax + 1):
                sEnsemblePartial[i, myLmin] = np.dot(
                    ClSim[myLmin:lmax + 1],
                    np.dot(myJmn[myLmin:, myLmin:], ClSim[myLmin:lmax + 1]))
            C2Ensemble[i] = ClSim[2]
        # save results
        np.save(saveFile,
                np.hstack((np.array([C2Ensemble]).T, sEnsemblePartial)))

    else:  # load from file
        sEnsemblePartial = np.load(saveFile)
        C2Ensemble = sEnsemblePartial[:, 0]
        sEnsemblePartial = sEnsemblePartial[:, 1:]
        nSims = sEnsemblePartial.shape[0]
        partialMax = sEnsemblePartial.shape[1] - 1

    if cutSky:
        # free the RAM used by SpICE's RAM disk
        ramDiskOutput = subprocess.check_output('./ramdisk.sh delete ' +
                                                diskID,
                                                shell=True)
        print ramDiskOutput

    # plot results

    print 'plotting S_{1/2} distributions... '

    #myBins = np.logspace(2,7,100)
    myBins = np.logspace(2, 6, 100)
    #plt.axvline(x=6763,color='b',linewidth=3,label='SMICA inpainted')
    #plt.axvline(x=2145,color='g',linewidth=3,label='SMICA masked')
    #plt.hist(sEnsembleFull,bins=myBins,color='b',histtype='step',label='full sky')
    #plt.hist(sEnsembleCut, bins=myBins,color='g',histtype='step',label='cut sky')

    myColors = ('g', 'b', 'r', 'c', 'm', 'k')  #need more?  prob. not.
    myLines = ('-', '--', '-.')  #need more?
    for myEll in range(lmin, partialMax + 1):
        plt.hist(sEnsemblePartial[:, myEll],
                 bins=myBins,
                 histtype='step',
                 label=r'sims: $\ell_{\rm min}$ = ' + str(myEll),
                 color=myColors[myEll - lmin],
                 linestyle=myLines[myEll - lmin],
                 linewidth=2)

        Sonehalf = np.dot(
            Clsmica[myEll:lmax + 1],
            np.dot(myJmn[myEll:, myEll:], Clsmica[myEll:lmax + 1])) * 1e24
        plt.axvline(x=Sonehalf,
                    linewidth=3,
                    label=r'SMICA: $\ell_{\rm min}$=' + str(myEll),
                    color=myColors[myEll - lmin],
                    linestyle=myLines[myEll - lmin])
        # calculate and print p-value
        pval = pValue(sEnsemblePartial[:, myEll], Sonehalf)
        print 'l_min: ', myEll, ', Sonehalf: ', Sonehalf, ', p-value: ', pval

    plt.gca().set_xscale("log")
    plt.legend()
    myfs = 16  # font size for labels
    plt.xlabel(r'$S_{1/2} (\mu K^4)$', fontsize=myfs)
    plt.ylabel('Counts', fontsize=myfs)
    plt.xlim((500, 10**6))
    if cutSky:
        sName = ' cut-sky'
    else:
        sName = ' full-sky'
    #plt.title(r'$S_{1/2}$ of '+str(nSims)+sName+' simulated CMBs')
    plt.show()

    print 'plotting C_2 vs. S_{1/2} histogram... '

    SMICAvals = (np.log10(2145), 171.8
                 )  # KLUDGE!!! #moved to earlier in program
    SonehalfLabel = "$log_{10}(\ S_{1/2}\ /\ (\mu K)^4\ )$"
    C2Label = "$C_2\ /\ (\mu K)^2$"
    C2Label3 = "$C_2\ /\ (10^3 (\mu K)^2)$"

    log10SonehalfEnsemble = np.log10(sEnsemblePartial[:, lmin])
    myBinsLog10S = np.linspace(2, 6, 100)
    myBinsC2 = np.linspace(0, 3000, 100)
    cmap = cm.magma  #Greens#Blues

    plt.hist2d(log10SonehalfEnsemble,
               C2Ensemble,
               bins=[myBinsLog10S, myBinsC2],
               cmap=cmap)
    plt.plot(SMICAvals[0], SMICAvals[1], 'cD')
    plt.colorbar()
    #myfs = 16 # font size for labels
    plt.xlabel(SonehalfLabel, fontsize=myfs)
    plt.ylabel(C2Label, fontsize=myfs)
    plt.show()

    print 'plotting C_2 vs. S_{1/2} contours... '

    H, xedges, yedges = np.histogram2d(log10SonehalfEnsemble,
                                       C2Ensemble,
                                       bins=(myBinsLog10S, myBinsC2))
    H = H.T  # Let each row list bins with common y range
    myXedges = (xedges[1:] +
                xedges[:-1]) / 2  #find midpoint of linspace for plotting
    myYedges = (yedges[1:] + yedges[:-1]) / 2
    hMax = np.max(H)
    #levels = [hMax*0.0009,hMax*0.009,hMax*0.09,hMax*0.9,hMax]
    #levels = [hMax*0.01,hMax*0.05,hMax*0.1,hMax*0.5,hMax*0.9,hMax]
    levels = np.logspace(np.log10(0.01 * hMax), np.log10(0.9 * hMax), 5)

    norm = cm.colors.Normalize(vmax=abs(H).max(), vmin=0)
    #cmap = cm.PRGn

    #plt.figure()
    #plt.imshow(H,origin='lower',norm=norm,cmap=cmap)#,extent=extent) #extent is a coordinate zoom
    #plt.imshow(H,norm=norm,cmap=cmap,extent=(2,6,0,3000)) #should match linspace above
    #v = plt.axis()
    CS = plt.contour(myXedges, myYedges, H, levels, colors='k', thickness=2)
    plt.clabel(CS, inline=1, fontsize=10)
    #plt.axis(v)

    plt.colorbar()
    #plt.title('do i want a title here?')
    plt.xlim(2.8, 5.8)
    #myfs = 16 # font size for labels
    plt.xlabel(SonehalfLabel, fontsize=myfs)
    plt.ylabel(C2Label, fontsize=myfs)
    plt.plot(SMICAvals[0], SMICAvals[1], 'cD')
    plt.show()

    print 'plotting corner plot... '

    toPlot = np.vstack((log10SonehalfEnsemble, C2Ensemble))
    toPlot = toPlot.T
    figure = corner.corner(toPlot,
                           labels=[SonehalfLabel, C2Label],
                           show_titles=False,
                           truths=SMICAvals,
                           range=((2.5, 6), (0, 3000)),
                           label_kwargs={'fontsize': myfs})
    plt.show()

    print 'plotting contours again but now using plot2Ddist (please wait)... '

    doTime = True
    startTime = time.time()
    scatterstyle = {'color': 'r', 'alpha': 0.5}
    styleargs = {'color': 'k', 'scatterstyle': scatterstyle}
    bw_method = 0.05  #'scott'
    axSize = "20%"  #1.5
    nstart = 600

    # create separate figures to contain separate plots
    """plt.figure(1)
  ax1=plt.gca()
  plt.figure(2)
  ax2=plt.gca()
  plt.figure(3)
  ax3=plt.gca()"""
    #fig = plt.figure() #should be the same one used by plot2Ddist

    # divide C2Ensemble by 1000 since that is approximate factor between ranges of C2,Sonehalf
    # presumably useful for accuracy in contour plotting via kernel density estimation
    fig1, axeslist = plot2Ddist.plot2Ddist(
        [log10SonehalfEnsemble, C2Ensemble / 1000],
        truevalues=[SMICAvals[0], SMICAvals[1] / 1000],
        labels=[SonehalfLabel, C2Label3],
        contourNGrid=nGrid,
        bw_method=bw_method,
        axSize=axSize,
        nstart=nstart,
        returnfigure=True,
        **styleargs)
    #bw_method=bw_method,axSize=axSize,axeslist=[ax1,ax2,ax3],**styleargs)
    ax1, ax2, ax3 = axeslist
    timeInterval1 = time.time() - startTime
    if doTime:
        print 'time elapsed: ', int(timeInterval1), ' seconds'
        print 'starting second plot2Ddist call... '

    ax1.set_xlim(left=2.9, right=6.1)
    ax1.set_ylim(top=5.5)
    ax1.plot(SMICAvals[0], SMICAvals[1] / 1000, 'cD')

    #inset plot
    left, bottom, width, height = [0.2, 0.4, 0.3, 0.3]
    ax4 = fig1.add_axes([left, bottom, width, height])
    #ax4.plot(range(10))
    plt.figure(5)
    ax5 = plt.gca()
    plt.figure(6)
    ax6 = plt.gca()

    plot2Ddist.plot2Ddist([log10SonehalfEnsemble, C2Ensemble / 1000],
                          truevalues=[SMICAvals[0], SMICAvals[1] / 1000],
                          contourNGrid=nGrid,
                          bw_method=bw_method,
                          axSize=axSize,
                          nstart=nstart,
                          axeslist=[ax4, ax5, ax6],
                          contourFractions=[0.91, 0.93, 0.95, 0.97, 0.99],
                          labelcontours=False,
                          **styleargs)

    timeInterval2 = time.time() - startTime
    if doTime:
        print 'time elapsed for both: ', int(timeInterval2), ' seconds'

    ax4.set_xlim(left=3.15, right=3.45)
    ax4.set_ylim(top=0.5)
    ax4.plot(SMICAvals[0], SMICAvals[1] / 1000, 'cD')
    ax4.xaxis.set_ticks((3.2, 3.3, 3.4))

    #plt.figure(1)
    #plt.xlim(2.9,6.1)
    #plt.ylim(-0.03,5.5)
    plt.show()

    # calculate and print 1D p-values
    pValueS12 = pValue(log10SonehalfEnsemble, SMICAvals[0])
    pValueC2 = pValue(C2Ensemble, SMICAvals[1])

    print 'S_{1/2} p-value = ', pValueS12
    print 'C_2 p-value     = ', pValueC2
    print ''
Esempio n. 8
0
def test(useCLASS=1,
         useLensing=1,
         classCamb=1,
         nSims=1000,
         lmax=3,
         lmin=2,
         newSMICA=True,
         newDeg=False):
    """
    code for testing the other functions in this module
    Inputs:
      useCLASS: set to 1 to use CLASS, 0 to use CAMB
        CLASS Cl has early/late split at z=50
        CAMB Cl has ISWin/out split: ISWin: 0.4<z<0.75, ISWout: the rest
        Note: CAMB results include primary in ISWin and ISWout (not as intended)
        default: 1
      useLensing: set to 1 to use lensed Cl, 0 for non-lensed
        default: 1
      classCamb: if 1: use the CAMB format of CLASS output, if 0: use CLASS format
        Note: parameter not used if useCLASS = 0
        default: 1
      nSims: the number of simulations to do for ensemble
        default: 1000
      lmax: the highest l to include in Legendre transforms
        default: 3
      lmin: the lowest l to include in S_{1/2} = CIC calculations
        default: 2
      newSMICA: set to True to recalculate SMICA results
        default: True
      newDeg: set to True to recalculate map and mask degredations
        default: False
  """

    ##############################################################################
    # load theoretical power spectra

    # load data
    ell, fullCl, primCl, lateCl, crossCl = gcp.loadCls(useCLASS=useCLASS,
                                                       useLensing=useLensing,
                                                       classCamb=classCamb)

    # fill beginning with zeros
    startEll = ell[0]
    ell = np.append(np.arange(startEll), ell)
    fullCl = np.append(np.zeros(startEll), fullCl)
    primCl = np.append(np.zeros(startEll), primCl)
    lateCl = np.append(np.zeros(startEll), lateCl)
    crossCl = np.append(np.zeros(startEll), crossCl)

    # suppress C_2 to see what happens in enesmble
    suppressC2 = False
    suppFactor = 0.23  # from Tegmark et. al. 2003, figure 13 (WMAP)
    if suppressC2:
        fullCl[2] *= suppFactor
        primCl[2] *= suppFactor
        lateCl[2] *= suppFactor
        crossCl[2] *= suppFactor

    conv = ell * (ell + 1) / (2 * np.pi)
    #print ell,conv #ell[0]=2.0

    # apply beam and pixel window functions to power spectra
    #   note: to ignore the non-constant pixel shape, W(l) must be > B(l)
    #     however, this is not true for NSIDE=128 and gauss_beam(5')
    #   Here I ignore this anyway and proceed
    myNSIDE = 128  # must be same NSIDE as in sims.getSMICA function
    Wpix = hp.pixwin(myNSIDE)
    Bsmica = hp.gauss_beam(5. / 60 * np.pi / 180)  # 5 arcmin
    WlMax = Wpix.size
    if WlMax < lmax:
        print 'die screaming!!!'
        return 0
    fullCl = fullCl[:WlMax] * (Wpix * Bsmica)**2
    primCl = primCl[:WlMax] * (Wpix * Bsmica)**2
    lateCl = lateCl[:WlMax] * (Wpix * Bsmica)**2
    crossCl = crossCl[:WlMax] * (Wpix * Bsmica)**2
    # note: i tried sims without this scaling, and results seemed the same at a glance

    # extract the part I want
    myL = ell[:lmax]
    myCl = fullCl[:lmax]

    ##############################################################################
    # load SMICA data and filter out all but low-l a_lm.s

    theta_i = 0.0  #degrees
    theta_f = 180.0  #degrees
    nSteps = 1800
    # default: lmax=3,lmin=2
    #newSMICA = True # so I don't use lmax=100 from previous calc.
    thetaArray2sp, C_SMICAsp, C_SMICAmaskedsp, S_SMICAnomasksp, S_SMICAmaskedsp = \
      sims.getSMICA(theta_i=theta_i,theta_f=theta_f,nSteps=nSteps,lmax=lmax,lmin=lmin,
               newSMICA=newSMICA,newDeg=newDeg,useSPICE=True)

    ##############################################################################
    # create ensemble of realizations and gather statistics

    covEnsembleFull = np.zeros([nSims, nSteps + 1])  # for maskless
    covEnsembleCut = np.zeros([nSims, nSteps + 1])  # for masked
    sEnsembleFull = np.zeros(nSims)
    sEnsembleCut = np.zeros(nSims)
    covTheta = np.array([])

    # get Jmn matrix for harmonic space S_{1/2} calc.
    myJmn = getJmn(lmax=lmax)

    doTime = True  # to time the run and print output
    startTime = time.time()
    for nSim in range(nSims):
        print 'starting sim ', nSim + 1, ' of ', nSims
        alm_prim, alm_late = hp.synalm((primCl, lateCl, crossCl),
                                       lmax=lmax,
                                       new=True)

        # calculate C(theta) of simulation
        Clsim_prim = hp.alm2cl(alm_prim)
        Clsim_late = hp.alm2cl(alm_late)
        Clsim_cros = hp.alm2cl(alm_prim, alm_late)
        Clsim_full = Clsim_prim + 2 * Clsim_cros + Clsim_late
        # use Cl_sim_full to omit prim/late distinction for now
        #Clsim_full_sum += Clsim_full

        # first without mask
        #   note: getCovar uses linspace in x for thetaArray
        thetaArray, cArray = sims.getCovar(ell[:lmax + 1],
                                           Clsim_full[:lmax + 1],
                                           theta_i=theta_i,
                                           theta_f=theta_f,
                                           nSteps=nSteps,
                                           lmin=lmin)
        covEnsembleFull[nSim] = cArray
        covTheta = thetaArray

        # S_{1/2}
        sEnsembleFull[nSim] = np.dot(
            Clsim_full[lmin:], np.dot(myJmn[lmin:, lmin:], Clsim_full[lmin:]))

        # now with a mask
        # should have default RING ordering
        # pixel window and beam already accounted for in true Cls
        #mapSim = hp.alm2map(alm_prim+alm_late,myNSIDE,lmax=lmax,pixwin=True,sigma=5./60*np.pi/180)
        mapSim = hp.alm2map(alm_prim + alm_late, myNSIDE, lmax=lmax)

        # super lame that spice needs to read/write from disk, but here goes...
        mapTempFile = 'tempMap.fits'
        ClTempFile = 'tempCl.fits'
        maskDegFile = 'maskMapDeg.fits'  # this should have been created by sims.getSMICA
        hp.write_map(mapTempFile, mapSim)
        ispice(mapTempFile,
               ClTempFile,
               maskfile1=maskDegFile,
               subav="YES",
               subdipole="YES")
        Cl_masked = hp.read_cl(ClTempFile)
        ell2 = np.arange(Cl_masked.shape[0])
        #   note: getCovar uses linspace in x for thetaArray
        thetaArray, cArray2 = sims.getCovar(ell2[:lmax + 1],
                                            Cl_masked[:lmax + 1],
                                            theta_i=theta_i,
                                            theta_f=theta_f,
                                            nSteps=nSteps,
                                            lmin=lmin)
        covEnsembleCut[nSim] = cArray2

        # S_{1/2}
        sEnsembleCut[nSim] = np.dot(
            Cl_masked[lmin:lmax + 1],
            np.dot(myJmn[lmin:, lmin:], Cl_masked[lmin:lmax + 1]))

        doPlot = False  #True
        if doPlot:
            plt.plot(thetaArray, cArray)
            plt.xlabel('theta (degrees)')
            plt.ylabel('C(theta)')
            plt.title('covariance of simulated CMB')
            plt.show()

    if doTime:
        print 'time elapsed: ', int(
            (time.time() - startTime) / 60.), ' minutes'
    avgEnsembleFull = np.average(covEnsembleFull, axis=0)
    stdEnsembleFull = np.std(covEnsembleFull, axis=0)
    # do I need a better way to describe confidence interval?
    avgEnsembleCut = np.average(covEnsembleCut, axis=0)
    stdEnsembleCut = np.std(covEnsembleCut, axis=0)

    #Clsim_full_avg = Clsim_full_sum / nSims

    ##############################################################################
    # plot/print results

    doPlot = True
    if doPlot:

        print 'plotting correlation functions... '
        # first the whole sky statistics
        plt.plot(thetaArray,
                 avgEnsembleFull,
                 label='sim. ensemble average (no mask)')
        plt.fill_between(thetaArray,
                         avgEnsembleFull + stdEnsembleFull,
                         avgEnsembleFull - stdEnsembleFull,
                         alpha=0.25,
                         label='simulation 1sigma envelope')
        #plt.plot(thetaArray2,C_SMICA,label='SMICA R2 (inpainted,anafast)')
        plt.plot(thetaArray2sp, C_SMICAsp, label='SMICA R2 (inpainted,spice)')

        plt.xlabel('theta (degrees)')
        plt.ylabel('C(theta)')
        plt.title('whole sky covariance of ' + str(nSims) +
                  ' simulated CMBs, lmax=' + str(lmax))
        plt.ylim([-500, 1000])
        plt.plot([0, 180], [0, 0])  #horizontal line
        plt.legend()
        plt.show()

        # now the cut sky
        plt.plot(thetaArray,
                 avgEnsembleCut,
                 label='sim. ensemble average (masked)')
        plt.fill_between(thetaArray,
                         avgEnsembleCut + stdEnsembleCut,
                         avgEnsembleCut - stdEnsembleCut,
                         alpha=0.25,
                         label='simulation 1sigma envelope')
        #plt.plot(thetaArray2,C_SMICAmasked,label='SMICA R2 (masked ,anafast)')
        plt.plot(thetaArray2sp,
                 C_SMICAmaskedsp,
                 label='SMICA R2 (masked ,spice)')

        plt.xlabel('theta (degrees)')
        plt.ylabel('C(theta)')
        plt.title('cut sky covariance of ' + str(nSims) +
                  ' simulated CMBs, lmax=' + str(lmax))
        plt.ylim([-500, 1000])
        plt.plot([0, 180], [0, 0])  #horizontal line
        plt.legend()
        plt.show()

        print 'plotting S_{1/2} distributions... '
        myBins = np.logspace(2, 7, 100)
        plt.axvline(x=S_SMICAnomasksp,
                    color='b',
                    linewidth=3,
                    label='SMICA inpainted')
        plt.axvline(x=S_SMICAmaskedsp,
                    color='g',
                    linewidth=3,
                    label='SMICA masked')
        plt.hist(sEnsembleFull, bins=myBins, histtype='step', label='full sky')
        plt.hist(sEnsembleCut, bins=myBins, histtype='step', label='cut sky')

        plt.gca().set_xscale("log")
        plt.legend()
        plt.xlabel('S_{1/2} (microK^4)')
        plt.ylabel('Counts')
        plt.title('S_{1/2} of ' + str(nSims) + ' simulated CMBs')
        plt.show()

    # S_{1/2} output
    print ''
    print 'using CIC method: '
    #print 'S_{1/2}(anafast): SMICA, no mask: ',S_SMICAnomask,', masked: ',S_SMICAmasked
    print 'S_{1/2}(spice): SMICA, no mask: ', S_SMICAnomasksp, ', masked: ', S_SMICAmaskedsp
    print ''
    #print 'using CCdx method: '
    #print 'S_{1/2}(anafast): SMICA, no mask: ',SSnm2,', masked: ',SSmd2
    #print 'S_{1/2}(spice): SMICA, no mask: ',SSnm2sp,', masked: ',SSmd2sp
    #print ''

    ##############################################################################
    # step 3
    print 'step 3: profit'