示例#1
0
def make_uv_grid(u,v,tol=0.01,do_fof=True):
    isconj=(v<0)|((v<tol)&(u<0))
    u=u.copy()
    v=v.copy()
    u[isconj]=-1*u[isconj]
    v[isconj]=-1*v[isconj]
    if (have_fof & do_fof):
        uv=numpy.stack([u,v]).transpose()
        groups=pyfof.friends_of_friends(uv,tol)
        myind=numpy.zeros(len(u))
        for j,mygroup in enumerate(groups):
            myind[mygroup]=j
        ii=numpy.argsort(myind)
        edges=numpy.where(numpy.diff(myind[ii])!=0)[0]+1
    else:
        #break up uv plane into tol-sized blocks
        u_int=numpy.round(u/tol)
        v_int=numpy.round(v/tol)
        uv=u_int+numpy.complex(0,1)*v_int
        ii=numpy.argsort(uv)
        uv_sort=uv[ii]
        edges=numpy.where(numpy.diff(uv_sort)!=0)[0]+1
    edges=numpy.append(0,edges)
    edges=numpy.append(edges,len(u))

    #map isconj into post-sorting indexing
    isconj=isconj[ii]
    return ii,edges,isconj
示例#2
0
    def fit(self, x, y=None, sample_weight=None):
        """Perform FoF clustering from features, or distance matrix.

        Parameters
        ----------
        x : {array-like, sparse matrix} of shape (n_samples, n_features), or \
            (n_samples, n_samples)
            Training instances to cluster, or distances between instances if

        y : Ignored
            Not used, present here for API consistency by convention.

        sample_weights : Ignored
            Not used, present here for API consistency by convention.

        Returns
        -------
        self

        """
        # pyfof returns a list of N elements, where N is the number of groups
        # found. Each list contains the indices of x that belongs to that group
        groups = pyfof.friends_of_friends(x, self.linking_length)

        # to turn the groups into sklearn-like labels, we create an array of
        # "labels" with the same size as data is in x
        x_len = len(x)
        labels = np.empty(x_len, dtype=int)

        # then we iterate over the groups, and assign the position of the group
        # as a label in the group indexes
        for idx, group in enumerate(groups):
            labels[group] = idx

        # Finally we assing he labels as an attribute of the object.
        self.labels_ = labels

        return self
示例#3
0
def build_ABC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build covariance matrix used in ABC for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the multidark simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    Notes 
    -----
    * This covariance matrix is the covariance matrix calculated from the *entire* multidark 
        box. So this does _not_ account for the sample variance, which the MCMC covariance does. 
    '''
    nbars, xir, gmfs = [], [], []

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname='multidark',
                                redshift=0,
                                halo_finder='rockstar')
    rbins = xi_binedges()  # some setting for tpcf calculations

    rmax = rbins.max()
    approx_cell1_size = [rmax, rmax, rmax]
    approx_cellran_size = [rmax, rmax, rmax]

    # load randoms and RRs for the ENTIRE MultiDark volume
    ###randoms = data_random(box='md_all')
    ###RR = data_RR(box='md_all')
    ###NR = len(randoms)

    for i in xrange(1, 125):
        print 'mock#', i
        # populate the mock subvolume
        model.populate_mock(halocat)
        # returning the positions of galaxies
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')

        # calculate nbar
        nbars.append(len(pos) / 1000**3.)

        # calculate xi(r) for the ENTIRE MultiDark volume
        # using the natural estimator DD/RR - 1
        xi = tpcf(pos,
                  rbins,
                  period=model.mock.Lbox,
                  max_sample_size=int(3e5),
                  estimator='Natural',
                  approx_cell1_size=approx_cell1_size)
        xir.append(xi)

        # calculate gmf
        nbar = len(pos) / 1000**3.
        b = b_normal * (nbar)**(-1. / 3)
        groups = pyfof.friends_of_friends(pos, b)
        w = np.array([len(x) for x in groups])
        gbins = gmf_bins()
        gmf = np.histogram(w, gbins)[0] / (1000.**3.)
        gmfs.append(gmf)  # GMF

    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join([util.obvs_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])

    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack(
        (np.array(nbars).reshape(len(nbars),
                                 1), np.array(xir), np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)
    # and save the covariance matrix
    nopoisson_file = ''.join([
        util.obvs_dir(), 'ABC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
        str(Mr), '.bnorm',
        str(round(b_normal, 2)), '.dat'
    ])
    np.savetxt(nopoisson_file, fullcov)
    return None
示例#4
0
def build_MCMC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build covariance matrix used in MCMC for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the other subvolumes (subvolume 1 to subvolume 125) of
    the simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    '''
    nbars = []
    xir = []
    gmfs = []

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname='multidark',
                                redshift=0,
                                halo_finder='rockstar')
    ###model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}

    #some settings for tpcf calculations
    rbins = xi_binedges()
    rmax = rbins.max()
    approx_cell1_size = [rmax, rmax, rmax]
    approx_cellran_size = [rmax, rmax, rmax]

    #load randoms and RRs

    randoms = data_random(box='md_sub')
    RR = data_RR(box='md_sub')
    NR = len(randoms)

    for i in xrange(1, 125):
        print 'mock#', i

        # populate the mock subvolume
        ###mocksubvol = lambda x: util.mask_func(x, i)
        ###model.populate_mock(halocat,
        ###                    masking_function=mocksubvol,
        ###                    enforce_PBC=False)
        model.populate_mock(halocat)
        # returning the positions of galaxies in the entire volume
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
        # masking out the galaxies outside the subvolume i
        pos = util.mask_galaxy_table(pos, i)
        # calculate nbar
        print "shape of pos", pos.shape
        nbars.append(len(pos) / 200**3.)
        # translate the positions of randoms to the new subbox
        xi0, yi0, zi0 = util.random_shifter(i)
        temp_randoms = randoms.copy()
        temp_randoms[:, 0] += xi0
        temp_randoms[:, 1] += yi0
        temp_randoms[:, 2] += zi0
        #calculate xi(r)
        xi = tpcf(pos,
                  rbins,
                  pos,
                  randoms=temp_randoms,
                  period=None,
                  max_sample_size=int(3e5),
                  estimator='Natural',
                  approx_cell1_size=approx_cell1_size,
                  approx_cellran_size=approx_cellran_size,
                  RR_precomputed=RR,
                  NR_precomputed=NR)
        xir.append(xi)
        # calculate gmf

        nbar = len(pos) / 200**3.
        b = b_normal * (nbar)**(-1. / 3)
        groups = pyfof.friends_of_friends(pos, b)
        w = np.array([len(x) for x in groups])
        gbins = gmf_bins()
        gmf = np.histogram(w, gbins)[0] / 200.**3.
        gmfs.append(gmf)

    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join([util.obvs_dir(), 'nbar_var.Mr', str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])

    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack(
        (np.array(nbars).reshape(len(nbars),
                                 1), np.array(xir), np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)

    # and save the covariance matrix
    nopoisson_file = ''.join([
        util.obvs_dir(), 'MCMC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
        str(Mr), '.bnorm',
        str(round(b_normal, 2)), '.dat'
    ])
    np.savetxt(nopoisson_file, fullcov)
    return None
示例#5
0
def build_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build data vector [nbar, xi, gmf] and save to file 
    This data vector is built from the zeroth slice of the multidark
    The other slices will be used for building the covariance matrix.

    Parameters
    ----------
    Mr : (int) 
        Absolute magnitude cut off M_r. Default M_r = -21.

    b_normal : (float) 
        FoF Linking length
    '''
    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname='multidark',
                                redshift=0,
                                halo_finder='rockstar')
    ####model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}

    ####datsubvol = lambda x: util.mask_func(x, 0)
    ####model.populate_mock(halocat, masking_function=datsubvol, enforce_PBC=False)
    model.populate_mock(halocat)

    #all the things necessary for tpcf calculation
    pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
    #masking the galaxies outside the subvolume 0
    pos = util.mask_galaxy_table(pos, 0)
    rbins = xi_binedges()
    rmax = rbins.max()
    approx_cell1_size = [rmax, rmax, rmax]
    approx_cellran_size = [rmax, rmax, rmax]

    #compute number density
    nbar = len(pos) / 200**3.

    # load MD subvolume randoms and RRs
    randoms = data_random(box='md_sub')
    RR = data_RR(box='md_sub')
    NR = len(randoms)

    #compue tpcf with Natural estimator
    data_xir = tpcf(pos,
                    rbins,
                    pos,
                    randoms=randoms,
                    period=None,
                    max_sample_size=int(2e5),
                    estimator='Natural',
                    approx_cell1_size=approx_cell1_size,
                    approx_cellran_size=approx_cellran_size,
                    RR_precomputed=RR,
                    NR_precomputed=NR)

    fullvec = np.append(nbar, data_xir)

    #compute gmf
    b = b_normal * (nbar)**(-1. / 3)
    groups = pyfof.friends_of_friends(pos, b)
    w = np.array([len(x) for x in groups])
    gbins = gmf_bins()
    gmf = np.histogram(w, gbins)[0] / (200.**3.)
    fullvec = np.append(fullvec, gmf)

    output_file = data_file(Mr=Mr, b_normal=b_normal)
    np.savetxt(output_file, fullvec)
    return None
示例#6
0
    def _sum_stat(self, theta, prior_range=None, observables=['nbar', 'gmf']):
        '''
        Given theta, sum_stat calculates the observables from our forward model

        Parameters
        ----------
        theta : (self explanatory)
        prior_range : If specified, checks to make sure that theta is within the prior range.
        '''
        self.model.param_dict['logM0'] = theta[0]
        self.model.param_dict['sigma_logM'] = np.exp(theta[1])
        self.model.param_dict['logMmin'] = theta[2]
        self.model.param_dict['alpha'] = theta[3]
        self.model.param_dict['logM1'] = theta[4]

        rbins = xi_binedges()
        rmax = rbins.max()
        approx_cell1_size = [rmax, rmax, rmax]
        approx_cellran_size = [rmax, rmax, rmax]

        if prior_range is None:

            self.model.populate_mock(self.halocat, enforce_PBC=False)
            pos = three_dim_pos_bundle(self.model.mock.galaxy_table, 'x', 'y',
                                       'z')
            obvs = []

            for obv in observables:
                if obv == 'nbar':
                    obvs.append(len(pos) /
                                1000.**3.)  # nbar of the galaxy catalog
                elif obv == 'gmf':
                    nbar = len(pos) / 1000**3.
                    b = self.b_normal * (nbar)**(-1. / 3)
                    groups = pyfof.friends_of_friends(pos, b)
                    w = np.array([len(x) for x in groups])
                    gbins = data_gmf_bins()
                    gmf = np.histogram(w, gbins)[0] / (1000.**3.)
                    obvs.append(gmf)
                elif obv == 'xi':
                    greek_xi = tpcf(pos,
                                    rbins,
                                    pos,
                                    randoms=randoms,
                                    period=None,
                                    max_sample_size=int(3e5),
                                    estimator='Natural',
                                    approx_cell1_size=approx_cell1_size,
                                    approx_cellran_size=approx_cellran_size,
                                    RR_precomputed=self.RR,
                                    NR_precomputed=self.NR)
                    obvs.append(greek_xi)
                else:
                    raise NotImplementedError(
                        'Only nbar 2pcf, gmf implemented so far')

            return obvs

        else:
            if np.all((prior_range[:, 0] < theta)
                      & (theta < prior_range[:, 1])):
                # if all theta_i is within prior range ...
                try:

                    self.model.populate_mock(self.halocat)
                    pos = three_dim_pos_bundle(self.model.mock.galaxy_table,
                                               'x', 'y', 'z')
                    obvs = []
                    for obv in observables:
                        if obv == 'nbar':
                            obvs.append(len(pos) /
                                        1000**3.)  # nbar of the galaxy catalog
                        elif obv == 'gmf':
                            nbar = len(pos) / 1000**3.
                            b = self.b_normal * (nbar)**(-1. / 3)
                            groups = pyfof.friends_of_friends(pos, b)
                            w = np.array([len(x) for x in groups])
                            gbins = data_gmf_bins()
                            gmf = np.histogram(w, gbins)[0] / (1000.**3.)
                            obvs.append(gmf)
                        elif obv == 'xi':
                            greek_xi = tpcf(
                                pos,
                                rbins,
                                pos,
                                randoms=randoms,
                                period=None,
                                max_sample_size=int(3e5),
                                estimator='Natural',
                                approx_cell1_size=approx_cell1_size,
                                approx_cellran_size=approx_cellran_size,
                                RR_precomputed=self.RR,
                                NR_precomputed=self.NR)
                            obvs.append(greek_xi)
                        else:
                            raise NotImplementedError(
                                'Only nbar, tpcf, and gmf are implemented so far'
                            )

                    return obvs

                except ValueError:

                    obvs = []
                    for obv in observables:
                        if obv == 'nbar':
                            obvs.append(10.)
                        elif obv == 'gmf':
                            bins = data_gmf_bins()
                            obvs.append(np.ones_like(bins)[:-1] * 1000.)
                        elif obv == 'xi':
                            obvs.append(np.zeros(len(xi_binedges()[:-1])))
                    return obvs
            else:
                obvs = []
                for obv in observables:
                    if obv == 'nbar':
                        obvs.append(10.)
                    elif obv == 'gmf':
                        bins = data_gmf_bins()
                        obvs.append(np.ones_like(bins)[:-1] * 1000.)
                    elif obv == 'xi':
                        obvs.append(np.zeros(len(xi_binedges()[:-1])))
                return obvs
示例#7
0
def build_nbar_xi_gmf_cov(Mr=21):
    ''' Build covariance matrix for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the multidark simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    '''
    nbars = []
    xir = []
    gmfs = []

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname='multidark',
                                redshift=0,
                                halo_finder='rockstar')
    #some settings for tpcf calculations
    rbins = hardcoded_xi_bins()

    for i in xrange(1, 125):
        print 'mock#', i

        # populate the mock subvolume
        model.populate_mock(halocat)
        # returning the positions of galaxies
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
        # calculate nbar
        nbars.append(len(pos) / 1000**3.)
        # translate the positions of randoms to the new subbox
        #calculate xi(r)
        xi = tpcf(pos,
                  rbins,
                  period=model.mock.Lbox,
                  max_sample_size=int(2e5),
                  estimator='Landy-Szalay')
        xir.append(xi)
        # calculate gmf
        nbar = len(pos) / 1000**3.
        b_normal = 0.75
        b = b_normal * (nbar)**(-1. / 3)
        groups = pyfof.friends_of_friends(pos, b)
        w = np.array([len(x) for x in groups])
        gbins = gmf_bins()
        gmf = np.histogram(w, gbins)[0] / (1000.**3.)
        gmfs.append(gmf)  # GMF
    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join(
        [util.multidat_dir(), 'abc_nbar_var.Mr',
         str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])

    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack(
        (np.array(nbars).reshape(len(nbars),
                                 1), np.array(xir), np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)
    # and save the covariance matrix
    nopoisson_file = ''.join([
        util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr',
        str(Mr), '.dat'
    ])
    np.savetxt(nopoisson_file, fullcov)

    # and a correlation matrix
    full_corr_file = ''.join(
        [util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr',
         str(Mr), '.dat'])
    np.savetxt(full_corr_file, fullcorr)

    return None
示例#8
0
def build_nbar_xi_gmf_cov(Mr=21):
    ''' Build covariance matrix for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the multidark simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    '''
    nbars = []
    xir = []
    gmfs = []

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
    #some settings for tpcf calculations
    rbins = hardcoded_xi_bins()

    for i in xrange(1,125):
        print 'mock#', i

        # populate the mock subvolume
        model.populate_mock(halocat)
        # returning the positions of galaxies
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
        # calculate nbar
        nbars.append(len(pos) / 1000**3.)
        # translate the positions of randoms to the new subbox
        #calculate xi(r)        
        xi = tpcf(pos, rbins, period = model.mock.Lbox, 
                  max_sample_size=int(2e5), estimator='Landy-Szalay')
        xir.append(xi)
        # calculate gmf
        nbar = len(pos) / 1000**3.
        b_normal = 0.75
        b = b_normal * (nbar)**(-1./3) 
        groups = pyfof.friends_of_friends(pos , b)
        w = np.array([len(x) for x in groups])
        gbins = gmf_bins()
        gmf = np.histogram(w , gbins)[0] / (1000.**3.)
        gmfs.append(gmf)  # GMF
    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join([util.multidat_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])


    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
                            np.array(xir),
                            np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)
    # and save the covariance matrix
    nopoisson_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr', str(Mr), '.dat'])
    np.savetxt(nopoisson_file, fullcov)

    # and a correlation matrix
    full_corr_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr', str(Mr), '.dat'])
    np.savetxt(full_corr_file, fullcorr)

    return None
示例#9
0
文件: data.py 项目: mjvakili/ccppabc
def build_ABC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build covariance matrix used in ABC for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the multidark simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    Notes 
    -----
    * This covariance matrix is the covariance matrix calculated from the *entire* multidark 
        box. So this does _not_ account for the sample variance, which the MCMC covariance does. 
    '''
    nbars, xir, gmfs = [], [], [] 

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
    rbins = xi_binedges()  # some setting for tpcf calculations

    rmax = rbins.max()
    approx_cell1_size = [rmax , rmax , rmax]
    approx_cellran_size = [rmax , rmax , rmax]
    
    # load randoms and RRs for the ENTIRE MultiDark volume 
    ###randoms = data_random(box='md_all')
    ###RR = data_RR(box='md_all')
    ###NR = len(randoms)

    for i in xrange(1,125):
        print 'mock#', i
        # populate the mock subvolume
        model.populate_mock(halocat)
        # returning the positions of galaxies
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')

        # calculate nbar
        nbars.append(len(pos) / 1000**3.)

        # calculate xi(r) for the ENTIRE MultiDark volume 
        # using the natural estimator DD/RR - 1
        xi = tpcf(
                pos, rbins, period=model.mock.Lbox, 
                max_sample_size=int(3e5), estimator='Natural', 
                approx_cell1_size=approx_cell1_size)
        xir.append(xi)

        # calculate gmf
        nbar = len(pos) / 1000**3.
        b = b_normal * (nbar)**(-1./3) 
        groups = pyfof.friends_of_friends(pos , b)
        w = np.array([len(x) for x in groups])
        gbins = gmf_bins()
        gmf = np.histogram(w , gbins)[0] / (1000.**3.)
        gmfs.append(gmf)  # GMF

    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join([util.obvs_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])

    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
                            np.array(xir),
                            np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)
    # and save the covariance matrix
    nopoisson_file = ''.join([util.obvs_dir(),
        'ABC.nbar_xi_gmf_cov', '.no_poisson', '.Mr', str(Mr), '.bnorm', str(round(b_normal, 2)), '.dat'])
    np.savetxt(nopoisson_file, fullcov)
    return None
示例#10
0
文件: data.py 项目: mjvakili/ccppabc
def build_MCMC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build covariance matrix used in MCMC for the full nbar, xi, gmf data vector
    using realisations of galaxy mocks for "data" HOD parameters in the 
    halos from the other subvolumes (subvolume 1 to subvolume 125) of
    the simulation. Covariance matrices for different sets of observables
    can be extracted from the full covariance matrix by slicing through 
    the indices. 

    '''
    nbars = []
    xir = []
    gmfs = []

    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
    ###model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
    
    #some settings for tpcf calculations
    rbins = xi_binedges()
    rmax = rbins.max()
    approx_cell1_size = [rmax , rmax , rmax]
    approx_cellran_size = [rmax , rmax , rmax]
    
    #load randoms and RRs
    
    randoms = data_random(box='md_sub')
    RR = data_RR(box='md_sub')
    NR = len(randoms)

    for i in xrange(1,125):
        print 'mock#', i

        # populate the mock subvolume
        ###mocksubvol = lambda x: util.mask_func(x, i)
        ###model.populate_mock(halocat,
        ###                    masking_function=mocksubvol,
        ###                    enforce_PBC=False)
        model.populate_mock(halocat)
        # returning the positions of galaxies in the entire volume
        pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
        # masking out the galaxies outside the subvolume i
        pos = util.mask_galaxy_table(pos , i)
        # calculate nbar
        print "shape of pos" , pos.shape 
        nbars.append(len(pos) / 200**3.)
        # translate the positions of randoms to the new subbox
        xi0 , yi0 , zi0 = util.random_shifter(i)
        temp_randoms = randoms.copy()
        temp_randoms[:,0] += xi0
        temp_randoms[:,1] += yi0
        temp_randoms[:,2] += zi0
        #calculate xi(r)        
        xi=tpcf(
             pos, rbins, pos, 
             randoms=temp_randoms, period=None, 
             max_sample_size=int(3e5), estimator='Natural', 
             approx_cell1_size=approx_cell1_size, 
             approx_cellran_size=approx_cellran_size,
             RR_precomputed = RR,
	     NR_precomputed = NR)
        xir.append(xi)
        # calculate gmf

        nbar = len(pos) / 200**3.
        b = b_normal * (nbar)**(-1./3) 
        groups = pyfof.friends_of_friends(pos , b)
    	w = np.array([len(x) for x in groups])
    	gbins = gmf_bins()
    	gmf = np.histogram(w , gbins)[0] / 200.**3.
        gmfs.append(gmf)

    # save nbar variance
    nbar_var = np.var(nbars, axis=0, ddof=1)
    nbar_file = ''.join([util.obvs_dir(), 'nbar_var.Mr', str(Mr), '.dat'])
    np.savetxt(nbar_file, [nbar_var])

    # write full covariance matrix of various combinations of the data
    # and invert for the likelihood evaluations

    # --- covariance for all three ---
    fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
                            np.array(xir),
                            np.array(gmfs)))
    fullcov = np.cov(fulldatarr.T)
    fullcorr = np.corrcoef(fulldatarr.T)

    # and save the covariance matrix
    nopoisson_file = ''.join([util.obvs_dir(),
        'MCMC.nbar_xi_gmf_cov', '.no_poisson', '.Mr', str(Mr), '.bnorm', str(round(b_normal,2)), '.dat'])
    np.savetxt(nopoisson_file, fullcov)
    return None
示例#11
0
文件: data.py 项目: mjvakili/ccppabc
def build_nbar_xi_gmf(Mr=21, b_normal=0.25):
    ''' Build data vector [nbar, xi, gmf] and save to file 
    This data vector is built from the zeroth slice of the multidark
    The other slices will be used for building the covariance matrix.

    Parameters
    ----------
    Mr : (int) 
        Absolute magnitude cut off M_r. Default M_r = -21.

    b_normal : (float) 
        FoF Linking length
    '''
    thr = -1. * np.float(Mr)
    model = PrebuiltHodModelFactory('zheng07', threshold=thr)
    halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
    ####model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}

    ####datsubvol = lambda x: util.mask_func(x, 0)
    ####model.populate_mock(halocat, masking_function=datsubvol, enforce_PBC=False)
    model.populate_mock(halocat)
    
    #all the things necessary for tpcf calculation
    pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
    #masking the galaxies outside the subvolume 0
    pos = util.mask_galaxy_table(pos , 0)
    rbins = xi_binedges()
    rmax = rbins.max()
    approx_cell1_size = [rmax , rmax , rmax]
    approx_cellran_size = [rmax , rmax , rmax]

    #compute number density    
    nbar = len(pos) / 200**3.
    
    # load MD subvolume randoms and RRs
    randoms = data_random(box='md_sub')
    RR = data_RR(box='md_sub')
    NR = len(randoms)
 
    #compue tpcf with Natural estimator
    data_xir = tpcf(
                 pos, rbins, pos, 
                 randoms=randoms, period=None, 
                 max_sample_size=int(2e5), estimator='Natural', 
                 approx_cell1_size=approx_cell1_size, 
                 approx_cellran_size=approx_cellran_size, 
                 RR_precomputed=RR, 
                 NR_precomputed=NR)

    fullvec = np.append(nbar, data_xir)
    
    #compute gmf
    b = b_normal * (nbar)**(-1./3) 
    groups = pyfof.friends_of_friends(pos , b)
    w = np.array([len(x) for x in groups])
    gbins = gmf_bins()
    gmf = np.histogram(w , gbins)[0] / (200.**3.)
    fullvec = np.append(fullvec, gmf)

    output_file = data_file(Mr=Mr, b_normal=b_normal)
    np.savetxt(output_file, fullvec)
    return None
示例#12
0
    def _sum_stat(self, theta, prior_range=None, observables=['nbar', 'gmf']):
        '''
        Given theta, sum_stat calculates the observables from our forward model

        Parameters
        ----------
        theta : (self explanatory)
        prior_range : If specified, checks to make sure that theta is within the prior range.
        '''
        self.model.param_dict['logM0'] = theta[0]
        self.model.param_dict['sigma_logM'] = np.exp(theta[1])
        self.model.param_dict['logMmin'] = theta[2]
        self.model.param_dict['alpha'] = theta[3]
        self.model.param_dict['logM1'] = theta[4]

        rbins = xi_binedges()
        rmax = rbins.max()
        approx_cell1_size = [rmax , rmax , rmax]
        approx_cellran_size = [rmax , rmax , rmax]

        if prior_range is None:
            
            self.model.populate_mock(self.halocat)
            pos =three_dim_pos_bundle(self.model.mock.galaxy_table, 'x', 'y', 'z')
            obvs = []

            for obv in observables:
                if obv == 'nbar':
                    obvs.append(len(pos) / 1000.**3.)       # nbar of the galaxy catalog
                elif obv == 'gmf':
                    nbar = len(pos) / 1000**3.
    		    b = self.b_normal * (nbar)**(-1./3) 
    		    groups = pyfof.friends_of_friends(pos , b)
    		    w = np.array([len(x) for x in groups])
    		    gbins =data_gmf_bins()
    		    gmf = np.histogram(w , gbins)[0] / (1000.**3.)
                    obvs.append(gmf)   
                elif obv == 'xi':
                    greek_xi = tpcf(
                            pos, rbins,  
                            period=self.model.mock.Lbox, 
                            max_sample_size=int(3e5), estimator='Natural', 
                            approx_cell1_size=approx_cell1_size)
                    obvs.append(greek_xi)
                else:
                    raise NotImplementedError('Only nbar 2pcf, gmf implemented so far')

            return obvs

        else:
            if np.all((prior_range[:,0] < theta) & (theta < prior_range[:,1])):
                # if all theta_i is within prior range ...
                try:


                    self.model.populate_mock(self.halocat) 
                    pos=three_dim_pos_bundle(self.model.mock.galaxy_table, 'x', 'y', 'z')
            	    obvs = []
            	    for obv in observables:
                        if obv == 'nbar':
                    	    obvs.append(len(pos) / 1000**3.)       # nbar of the galaxy catalog
                        elif obv == 'gmf':
                    	    nbar = len(pos) / 1000**3.
    		    	    b = self.b_normal * (nbar)**(-1./3) 
    		    	    groups = pyfof.friends_of_friends(pos , b)
    		    	    w = np.array([len(x) for x in groups])
    		    	    gbins =data_gmf_bins()
    		    	    gmf = np.histogram(w , gbins)[0] / (1000.**3.)
                    	    obvs.append(gmf)   
                        elif obv == 'xi':
                            greek_xi = tpcf(
                                    pos, rbins, period=self.model.mock.Lbox, 
                                    max_sample_size=int(3e5), estimator='Natural', 
                                    approx_cell1_size=approx_cell1_size)
                            obvs.append(greek_xi)
                        else:
                            raise NotImplementedError('Only nbar, tpcf, and gmf are implemented so far')

                    return obvs

                except ValueError:

                    obvs = []
                    for obv in observables:
                        if obv == 'nbar':
                            obvs.append(10.)
                        elif obv == 'gmf':
                            bins = data_gmf_bins()
                            obvs.append(np.ones_like(bins)[:-1]*1000.)
                        elif obv == 'xi':
                            obvs.append(np.zeros(len(xi_binedges()[:-1])))
                    return obvs
            else:
                obvs = []
                for obv in observables:
                    if obv == 'nbar':
                        obvs.append(10.)
                    elif obv == 'gmf':
                        bins = data_gmf_bins()
                        obvs.append(np.ones_like(bins)[:-1]*1000.)
                    elif obv == 'xi':
                        obvs.append(np.zeros(len(xi_binedges()[:-1])))
                return obvs
示例#13
0
    def _sum_stat(self, theta, prior_range=None, observables=['nbar', 'gmf']):
        '''
        Given theta, sum_stat calculates the observables from our forward model

        Parameters
        ----------
        theta : (self explanatory)
        prior_range : If specified, checks to make sure that theta is within the prior range.
        '''
        self.model.param_dict['logM0'] = theta[0]
        self.model.param_dict['sigma_logM'] = np.exp(theta[1])
        self.model.param_dict['logMmin'] = theta[2]
        self.model.param_dict['alpha'] = theta[3]
        self.model.param_dict['logM1'] = theta[4]

        rbins = xi_binedges()
        rmax = rbins.max()
        period = None
        approx_cell1_size = [rmax , rmax , rmax]
        approx_cellran_size = [rmax , rmax , rmax]

        if prior_range is None:
            rint = np.random.randint(1, 125)
            ####simsubvol = lambda x: util.mask_func(x, rint)
            ####self.model.populate_mock(self.halocat,
            ####                masking_function=simsubvol,
            ####                enforce_PBC=False)
            self.model.populate_mock(self.halocat)
                        
            pos =three_dim_pos_bundle(self.model.mock.galaxy_table, 'x', 'y', 'z')
            pos = util.mask_galaxy_table(pos , rint) 

            xi , yi , zi = util.random_shifter(rint)
            temp_randoms = self.randoms.copy()
            temp_randoms[:,0] += xi
            temp_randoms[:,1] += yi
            temp_randoms[:,2] += zi

            obvs = []
            for obv in observables:
                if obv == 'nbar':
                    obvs.append(len(pos) / 200**3.)       # nbar of the galaxy catalog
                elif obv == 'gmf':
                    #compute group richness 
                    nbar = len(pos) / 200**3.
    		    b = self.b_normal * (nbar)**(-1./3) 
    		    groups = pyfof.friends_of_friends(pos , b)
    		    w = np.array([len(x) for x in groups])
    		    gbins = data_gmf_bins()
    		    gmf = np.histogram(w , gbins)[0] / (200.**3.)
                    obvs.append(gmf)   
                elif obv == 'xi':
                    greek_xi = tpcf(
                        pos, rbins, pos, 
                        randoms=temp_randoms, period = period, 
                        max_sample_size=int(1e5), estimator='Natural', 
                        approx_cell1_size=approx_cell1_size, 
                        approx_cellran_size=approx_cellran_size,
                        RR_precomputed = self.RR,
	                NR_precomputed = self.NR)

                    obvs.append(greek_xi)
                else:
                    raise NotImplementedError('Only nbar 2pcf, gmf implemented so far')

            return obvs

        else:
            if np.all((prior_range[:,0] < theta) & (theta < prior_range[:,1])):
                # if all theta_i is within prior range ...
                try:
                    rint = np.random.randint(1, 125)
                    simsubvol = lambda x: util.mask_func(x, rint)
                    self.model.populate_mock(self.halocat,
                                masking_function=simsubvol,
                                enforce_PBC=False)
           
                    pos =three_dim_pos_bundle(self.model.mock.galaxy_table, 'x', 'y', 'z')
                    #imposing mask on the galaxy table
                    pos = util.mask_galaxy_table(pos , rint) 
            	    xi , yi , zi = util.random_shifter(rint)
            	    temp_randoms = self.randoms.copy()
            	    temp_randoms[:,0] += xi
            	    temp_randoms[:,1] += yi
            	    temp_randoms[:,2] += zi
            	    obvs = []

            	    for obv in observables:
                        if obv == 'nbar':
                    	    obvs.append(len(pos) / 200**3.)       # nbar of the galaxy catalog
                        elif obv == 'gmf':
                            nbar = len(pos) / 200**3.
    		            b = self.b_normal * (nbar)**(-1./3) 
    		            groups = pyfof.friends_of_friends(pos , b)
    		            w = np.array([len(x) for x in groups])
    		    	    gbins =data_gmf_bins()
    		            gmf = np.histogram(w , gbins)[0] / (200.**3.)
                    	    obvs.append(gmf)   
                        elif obv == 'xi':
                    	    greek_xi = tpcf(
                                     pos, rbins, pos, 
                        	     randoms=temp_randoms, period = period, 
                                     max_sample_size=int(1e5), estimator='Natural', 
                                     approx_cell1_size=approx_cell1_size, 
                                     approx_cellran_size=approx_cellran_size,
                                     RR_precomputed = self.RR,
	                             NR_precomputed = self.NR)

                    	    obvs.append(greek_xi)
                        else:
                            raise NotImplementedError('Only nbar, tpcf, and gmf are implemented so far')

                    return obvs

                except ValueError:

                    obvs = []
                    for obv in observables:
                        if obv == 'nbar':
                            obvs.append(10.)
                        elif obv == 'gmf':
                            bins = data_gmf_bins()
                            obvs.append(np.ones_like(bins)[:-1]*1000.)
                        elif obv == 'xi':
                            obvs.append(np.zeros(len(xi_binedges()[:-1])))
                    return obvs
            else:
                obvs = []
                for obv in observables:
                    if obv == 'nbar':
                        obvs.append(10.)
                    elif obv == 'gmf':
                        bins = data_gmf_bins()
                        obvs.append(np.ones_like(bins)[:-1]*1000.)
                    elif obv == 'xi':
                        obvs.append(np.zeros(len(xi_binedges()[:-1])))
                return obvs
示例#14
0
文件: fof.py 项目: mjvakili/ccppabc
    return grp_richness
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')

model.populate_mock(halocat)
pos =three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')

#groups = FoFGroups(pos, 0.75, 0.75, Lbox = model.mock.Lbox, num_threads=1)
#gids = groups.group_ids

#print richness(gids)
import time

a = time.time()

groups = pyfof.friends_of_friends(pos, 0.75*(len(pos)/1000**3.)**(-1./3))

w = np.array([len(x) for x in groups])

bins = np.array([2.,3.,4.,5.,6.,7,9,11,14,17,20])


gmeff = np.histogram(w , bins)[0] / 1000.**3.

print gmeff

print time.time() - a