def xi_proj_analytic(x, y, z, L, r_edges, ncomponents, proj_type, projfn=None, nthreads=24): mumax = 1.0 nmubins = 1 verbose = False # MAKE SURE THIS IS FALSE otherwise breaks periodic = True # Periodic must be true for analytic computation! _, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, proj_type=proj_type, ncomponents=ncomponents, projfn=projfn, boxsize=L, periodic=periodic) #dd_proj = np.random.rand(ncomponents) rmin = min(r_edges) rmax = max(r_edges) volume = float(L**3) #nrbins = len(r_edges)-1 nd = len(x) # works up to 100 thru here # hangs up to 15 when through next line print(rmin, rmax, nd, volume, ncomponents, r_edges, proj_type, projfn) rr_ana, trr_ana = trr_analytic(rmin, rmax, nd, volume, ncomponents, proj_type, rbins=r_edges, projfn=projfn) print(rr_ana) rcont = np.linspace(rmin, rmax, 1000) numerator = dd_proj - rr_ana amps_periodic_ana = np.linalg.solve(trr_ana, numerator) #xi_periodic_ana = evaluate_xi(nrbins, amps_periodic_ana, len(rcont), rcont, nrbins, r_edges, proj_type, projfn=projfn) xi_periodic_ana = evaluate_xi(amps_periodic_ana, rcont, proj_type, rbins=r_edges, projfn=projfn) print("DD proj:", dd_proj) print(numerator) print(amps_periodic_ana) print("First 10 xi vals:", xi_periodic_ana[:10]) return rcont, xi_periodic_ana, amps_periodic_ana
def xi_proj_numeric(x, y, z, rx, ry, rz, rr_proj, trr_proj, L, r_edges, ncomponents, proj_type, projfn=None, weights=None, weights_r=None, weight_type=None, nthreads=24, periodic=True): mumax = 1.0 nmubins = 1 verbose = False # MAKE SURE THIS IS FALSE otherwise breaks print("computing dd...") _, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, proj_type=proj_type, ncomponents=ncomponents, projfn=projfn, boxsize=L, periodic=periodic, weights1=weights, weight_type=weight_type) print("DD proj:", dd_proj) print("computing dr...") _, dr_proj, _ = DDsmu(0, nthreads, r_edges, mumax, nmubins, x, y, z, X2=rx, Y2=ry, Z2=rz, proj_type=proj_type, ncomponents=ncomponents, projfn=projfn, boxsize=L, periodic=periodic, weights1=weights, weights2=weights_r, weight_type=weight_type) print("DR proj:", dr_proj) nd = len(x) nr = len(rx) rmin = min(r_edges) rmax = max(r_edges) r_cont = np.linspace(rmin, rmax, 1000) amps = compute_amps(ncomponents, nd, nd, nr, nr, dd_proj, dr_proj, dr_proj, rr_proj, trr_proj) # for gradient, this will just return the tophat for now; we'll do the evaluation separately xi_proj = evaluate_xi(amps, r_cont, proj_type, rbins=r_edges, projfn=projfn) return r_cont, xi_proj, amps
def run_estimator_analytic(self): # TODO: make that can pass rbins as None to DDsmu for e.g. generalr when dont need! _, dd_proj, _ = DDsmu(1, self.nthreads, self.rbins, self.mumax, self.nmubins, self.x, self.y, self.z, proj_type=self.proj_type, nprojbins=self.nprojbins, projfn=self.projfn, verbose=self.verbose, boxsize=self.boxsize, periodic=self.periodic, isa='fallback') volume = float(self.boxsize**3) rr_ana, qq_ana = qq_analytic(self.rmin, self.rmax, self.nd, volume, self.nprojbins, self.proj_type, rbins=self.rbins, projfn=self.projfn) numerator = dd_proj - rr_ana amps_periodic_ana = np.linalg.solve(qq_ana, numerator) print("AMPS:", amps_periodic_ana) xi_periodic_ana = evaluate_xi(amps_periodic_ana, self.rcont, self.proj_type, rbins=self.rbins, projfn=self.projfn) return xi_periodic_ana, amps_periodic_ana
rbins, ra_rand, dec_rand, z_rand, is_comoving_dist=True, proj_type=proj_type, nprojbins=nbins, verbose=verbose, weights1=weights_rand, weight_type=weight_type) # In[7]: amps = compute_amps(nbins, nd, nd, nr, nr, dd_proj, dr_proj, dr_proj, rr_proj, qq_proj) xi_proj = evaluate_xi(nbins, amps, len(rcont), rcont, len(rbins) - 1, rbins, proj_type) # ### Perform xi(s, mu) standard estimation # In[8]: def extract_counts(res, weight_type=None): counts = np.array([x[4] for x in res], dtype=float) if weight_type: weights = np.array([x[5] for x in res], dtype=float) counts *= weights return counts # In[9]:
def run_dr7_LRGs(): print "Running corrfunc estimator on LRGs" #sample = 'Full' #DR = 7 sample = 'ns' DR = 3 nproc = 24 frac = 1 randfrac = 1 #proj_type = 'tophat' #proj_type = 'piecewise' proj_type = 'generalr' proj_label = proj_type + '_fast' #projfn = "/home/users/ksf293/Corrfunc/utils/dcosmos.dat" #projfn = None #projfn = "/home/users/ksf293/vectorizedEstimator/tables/dcosmos.dat" projfn = "/home/users/ksf293/vectorizedEstimator/tables/dcosmos_norm.dat" #proj_type = 'dcosmo' #proj_label = proj_type+'_const' projtag = '_' + proj_label if randfrac == 1: randfractag = '' else: randfractag = '_rand' + str(randfrac) saveto = "../results/dcosmo/xis_dr{}_{}LRG_frac{}{}{}.npy".format( DR, sample, frac, randfractag, projtag) K = 14 smin = 40 smax = 180 #K=2 #smin = 5 #smax = 15 sbins = np.linspace(smin, smax, K + 1) sbinsavg = np.array(0.5 * (sbins[1:] + sbins[:-1])) print "bins:", sbins if proj_type == "tophat" or proj_type == "piecewise": nprojbins = len(sbins) - 1 elif proj_type == "powerlaw": nprojbins = 5 elif proj_type == "dcosmo": nprojbins = 3 #FOR NOW elif proj_type == "generalr": nprojbins = 3 else: raise ValueError("Proj type {} not recognized".format(proj_type)) print "nprojbins:", nprojbins datafn = '../data/DR{}-{}.ascii'.format(DR, sample) randfn = '../data/random-DR{}-{}.ascii'.format(DR, sample) print "Loading data..." data = pd.read_csv(datafn, index_col=0) rand = pd.read_csv(randfn, index_col=0) #saveto = None if DR == 7: cosmo = LambdaCDM(H0=70, Om0=0.25, Ode0=0.75) elif DR == 3: cosmo = LambdaCDM(H0=70, Om0=0.30, Ode0=0.70) else: raise ValueError("DR {} not recognized".format(DR)) #check if need to return or in place utils.write_comoving_dist(data, datafn, cosmo) utils.write_comoving_dist(rand, randfn, cosmo) print 'ndata=', len(data.index) print 'nrand=', len(rand.index) #data = data.sample(frac=frac) #rand = rand.sample(frac=frac*randfrac) step = int(1 / frac) data = data[::step] rand = rand[::step] #data = data[:int(frac*len(data.index))] #rand = rand[:int(frac*len(rand.index))] nd = len(data.index) nr = len(rand.index) print 'ndata=', nd print 'nrand=', nr #weights_data = data['radial_weight']*data['fiber_coll_weight'] #weights_rand = rand['radial_weight'] weights_data = None weights_rand = None mumax = 1.0 #max of cosine ss = [] xis = [] labels = [] counts = [] aa = [] print "Running corrfunc..." start = time.time() dcm_col = 'dcm_Om0-{:.2f}'.format(cosmo.Om0) data_cz = data[dcm_col].values rand_cz = rand[dcm_col].values res = corrfuncproj.counts_smu(data['ra'].values, data['dec'].values, data_cz, rand['ra'].values, rand['dec'].values, rand_cz, sbins, mumax, cosmo, nproc=nproc, weights_data=weights_data, weights_rand=weights_rand, comoving=True, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn) print("Completed counts") dd, dr, rr, qq, dd_orig, dr_orig, rr_orig = res # Note: dr twice because cross-correlations will be possible amps = compute_amps(nprojbins, nd, nd, nr, nr, dd, dr, dr, rr, qq) print 'Computed amplitudes' amps = np.array(amps) svals = np.linspace(smin, smax, 300) #print "svals:",svals #svals = np.array(0.5*(sbins[1:]+sbins[:-1])) nsvals = len(svals) sbins = np.array(sbins) nsbins = len(sbins) - 1 xi_proj = evaluate_xi(nprojbins, amps, nsvals, svals, nsbins, sbins, proj_type, projfn=projfn) #print("xi:", xi_proj) end = time.time() print 'Time for dr7 {} LRGs, nd={} nr={}: {}'.format( sample, nd, nr, end - start) xi_orig = convert_3d_counts_to_cf(nd, nd, nr, nr, dd_orig, dr_orig, dr_orig, rr_orig) #print "savg:", sbinsavg #print "xi_orig", xi_orig #print "svals", svals #print "xi_proj", xi_proj ss.append(sbinsavg) xis.append(xi_orig) labels.append("orig") counts.append([dd_orig, dr_orig, rr_orig]) aa.append(None) ss.append(svals) xis.append(xi_proj) labels.append(proj_label) counts.append([dd, dr, rr, qq]) aa.append(amps) if saveto: print "Saving to {}".format(saveto) np.save(saveto, [ss, xis, labels, counts, aa])
x, y, z, X2=x_rand, Y2=y_rand, Z2=z_rand, proj_type=proj_type, nprojbins=nbins, verbose=verbose, weights1=weights, weights2=weights_rand, weight_type=weight_type, periodic=periodic) _, rr_proj, qq_proj = DDsmu(1, nthreads, rbins, mumax, nmubins, x_rand, y_rand, z_rand, proj_type=proj_type, nprojbins=nbins, verbose=verbose, weights1=weights_rand, weight_type=weight_type, periodic=periodic) # In[12]: amps = compute_amps(nbins, nd, nd, nr, nr, dd_proj, dr_proj, dr_proj, rr_proj, qq_proj) xi_proj = evaluate_xi(nbins, amps, len(rcont), rcont, len(rbins)-1, rbins, proj_type) # In[ ]: # In[ ]: # ### Perform xi(s, mu) standard estimation
def counts_proj(ra_data, dec_data, z_data, ra_rand, dec_rand, z_rand, rpbins, losmax, cosmo, nproc=1, weights_data=None, weights_rand=None, comoving=False): assert (len(ra_data) == len(dec_data) and len(ra_data) == len(z_data)) assert (len(ra_rand) == len(dec_rand) and len(ra_rand) == len(z_rand)) nd1 = len(ra_data) nr1 = len(ra_rand) nd2 = nd1 nr2 = nr1 print "counts proj" print "comoving:", comoving if comoving: print "To comoving:" zdf = pd.DataFrame(z_data) print min(z_data), max(z_data) z_data = zdf.apply(get_comoving_dist, args=(cosmo, ))[0].values print min(z_data), max(z_data) rzdf = pd.DataFrame(z_rand) print min(z_rand), max(z_rand) z_rand = rzdf.apply(get_comoving_dist, args=(cosmo, ))[0].values print min(z_rand), max(z_rand) if weights_data is None: weights_data = np.ones(nd1) if weights_rand is None: weights_rand = np.ones(nr1) cosmology = 1 nthreads = nproc verbose = False weight_type = 'pair_product' output_rpavg = True isa = 'fallback' mumax = losmax nmubins = 1 print 'test' # dd_res_corrfunc, dd_proj, dd_projt = s_mu_mocks(1, cosmology, nthreads, mumax, nmubins, rpbins, ra_data, dec_data, z_data, # weights1=weights_data, is_comoving_dist=comoving, verbose=verbose, # weight_type=weight_type, isa=isa) print 'Computing DD pairs' start = time.time() dd_res_corrfunc, dd_proj, dd_projt = DDsmu_mocks(1, cosmology, nthreads, mumax, nmubins, rpbins, ra_data, dec_data, z_data, weights1=weights_data, is_comoving_dist=comoving, verbose=verbose, weight_type=weight_type, isa=isa) end = time.time() print "Time DD pairs:", end - start print "DD:", dd_proj print dd_res_corrfunc print 'Computing DR pairs' start = time.time() dr_res_corrfunc, dr_proj, dr_projt = DDsmu_mocks(0, cosmology, nthreads, mumax, nmubins, rpbins, ra_data, dec_data, z_data, RA2=ra_rand, DEC2=dec_rand, CZ2=z_rand, weights1=weights_data, weights2=weights_rand, is_comoving_dist=comoving, verbose=verbose, weight_type=weight_type, isa=isa) end = time.time() print "Time DD pairs:", end - start print "DR:", dr_proj print dr_res_corrfunc print 'Computing RR pairs' start = time.time() rr_res_corrfunc, rr_proj, rr_projt = DDsmu_mocks(1, cosmology, nthreads, mumax, nmubins, rpbins, ra_rand, dec_rand, z_rand, weights1=weights_rand, is_comoving_dist=comoving, verbose=verbose, weight_type=weight_type, isa=isa) end = time.time() print "Time RR pairs:", end - start print "RR:", rr_proj print "QQ:", rr_projt print rr_res_corrfunc # a = [] # dd = dd_proj * 1. / (nd1 * nd2) # dr = dr_proj * 1. / (nd1 * nr2) # #TODO: allow cross-correlations # rd = dr_proj * 1. / (nd2 * nr1) # rr = rr_proj * 1. / (nr1 * nr2) # qq = rr_projt * 1. / (nr1 * nr2) #a.append(calc_amplitudes(dd, dr, rd, rr, qq)) nprojbins = len(rpbins) - 1 amps = compute_amps(nprojbins, nd1, nd2, nr1, nr2, dd_proj, dr_proj, dr_proj, rr_proj, rr_projt) print 'Computed amplitudes' print amps print amps amps = np.array(amps) svals = np.array(0.5 * (rpbins[1:] + rpbins[:-1])) nsvals = len(svals) sbins = np.array(rpbins) nsbins = len(rpbins) - 1 print nprojbins print amps print nsvals print svals print nsbins print sbins xi = evaluate_xi(nprojbins, amps, nsvals, svals, nsbins, sbins) print "Evaluated xi" print xi return dd_res_corrfunc, dr_res_corrfunc, rr_res_corrfunc, svals, xi, amps
def main(): print("Go!") nrealizations = 1 L = 750 #L = 450.0 N = 1000 # number of points in mock cat_tag = 'cattest' proj = 'gradient' kwargs = {} binwidth = 10 cf_tag = f"_{proj}_top_bw{binwidth}" #proj = 'tophat' #kwargs = {} #binwidth = 5 #cf_tag = f"_{proj}_bw{binwidth}_anatest" #proj = None #kwargs = {} #binwidth = 10 #cf_tag = f"_{proj}_bw{binwidth}" rmin = 40.0 rmax = 60.0 r_edges = np.arange(rmin, rmax + binwidth, binwidth) rmax = max(r_edges) nrbins = len(r_edges) - 1 r_avg = 0.5 * (r_edges[1:] + r_edges[:-1]) result_dir = '../results/results_lognormal{}'.format(cat_tag) if not os.path.exists(result_dir): os.makedirs(result_dir) proj_type, nprojbins, projfn = get_proj_parameters(proj, r_edges=r_edges, cf_tag=cf_tag, **kwargs) nthreads = 2 for Nr in range(nrealizations): print(f"Realization {Nr}") save_fn = f'{result_dir}/cf{cf_tag}_rlz{Nr}.npy' # Generate cubic mock x = np.random.rand(N) * float(L) y = np.random.rand(N) * float(L) z = np.random.rand(N) * float(L) ones = np.ones(N) if proj_type == 'gradient': weights = np.array([ones, x, y, z]) else: weights = ones # Generate random catalog randmult = 2 Nr = N * randmult xr = np.random.rand(Nr) * float(L) yr = np.random.rand(Nr) * float(L) zr = np.random.rand(Nr) * float(L) ones = np.ones(Nr) if proj_type == 'gradient': weights_r = np.array([ones, xr, yr, zr]) else: weights_r = ones print("Run DDsmu, {} basis".format(proj)) nmubins = 1 mumax = 1.0 periodic = True verbose = True weight_type = 'pair_product_gradient' np.set_printoptions(precision=6) dd_res, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, weights1=weights, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn, periodic=periodic, boxsize=L, verbose=verbose, weight_type=weight_type) print("DD:", np.array(dd_proj)) dr_res, dr_proj, _ = DDsmu(0, nthreads, r_edges, mumax, nmubins, x, y, z, weights1=weights, X2=xr, Y2=yr, Z2=zr, weights2=weights_r, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn, periodic=periodic, boxsize=L, verbose=verbose, weight_type=weight_type) print("DR:", np.array(dr_proj)) rr_res, rr_proj, qq_proj = DDsmu(1, nthreads, r_edges, mumax, nmubins, xr, yr, zr, weights1=weights_r, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn, periodic=periodic, boxsize=L, verbose=verbose, weight_type=weight_type) print("RR:", np.array(rr_proj)) if proj_type is None: print(dd_res) continue amps = compute_amps(nprojbins, N, N, Nr, Nr, dd_proj, dr_proj, dr_proj, rr_proj, qq_proj) print(amps) #weights1 = np.array([1.0, 325., 325., 325.]) #weights2 = np.array([1.0, 325., 325., 325.]) weights1 = np.array([1.0, 750., 750., 750.]) weights2 = np.array([1.0, 750., 750., 750.]) rcont = np.linspace(rmin, rmax, 10) #weights1=None #weights2=None #weight_type=None print(amps, rcont, proj_type, r_edges, projfn, weights1, weights2, weight_type) xi = evaluate_xi(amps, rcont, proj_type, rbins=r_edges, projfn=projfn, weights1=weights1, weights2=weights2, weight_type=weight_type) # for now! print(xi) r = [] xi = [] extra_dict = {} np.save(save_fn, [r, xi, amps, proj, extra_dict])
def run_estimator_numeric(self): _, dd_proj, _ = DDsmu(1, self.nthreads, self.rbins, self.mumax, self.nmubins, self.x, self.y, self.z, proj_type=self.proj_type, nprojbins=self.nprojbins, projfn=self.projfn, verbose=self.verbose, boxsize=self.boxsize, periodic=self.periodic) _, dr_proj, _ = DDsmu(0, self.nthreads, self.rbins, self.mumax, self.nmubins, self.x, self.y, self.z, X2=self.x_rand, Y2=self.y_rand, Z2=self.z_rand, proj_type=self.proj_type, nprojbins=self.nprojbins, projfn=self.projfn, verbose=self.verbose, boxsize=self.boxsize, periodic=self.periodic) _, rr_proj, qq_proj = DDsmu(1, self.nthreads, self.rbins, self.mumax, self.nmubins, self.x_rand, self.y_rand, self.z_rand, proj_type=self.proj_type, nprojbins=self.nprojbins, projfn=self.projfn, verbose=self.verbose, boxsize=self.boxsize, periodic=self.periodic) print("nd nr", self.nd, self.nr) amps = compute_amps(self.nprojbins, self.nd, self.nd, self.nr, self.nr, dd_proj, dr_proj, dr_proj, rr_proj, qq_proj) print("AMPS:", amps) xi_proj = evaluate_xi(self.nprojbins, amps, len(self.rcont), self.rcont, len(self.rbins) - 1, self.rbins, self.proj_type, projfn=self.projfn) return xi_proj, amps
def main(): print("Go!") print(Corrfunc.__version__) print(Corrfunc.__file__) nrealizations = 1 #L = 750 L = 450.0 N = 10000 # number of points in mock proj = 'spline' kwargs = {'order': 3} binwidth = 10 cf_tag = f"_{proj}{kwargs['order']}_bw{binwidth}_bintest" #proj = 'tophat' #kwargs = {} #binwidth = 5 #cf_tag = f"_{proj}_bw{binwidth}_basetest" #proj = None #kwargs = {} #binwidth = 10 #cf_tag = f"_{proj}_bw{binwidth}" rmin = 40.0 rmax = 150.0 r_edges = np.arange(rmin, rmax+binwidth, binwidth) rmax = max(r_edges) r_avg = 0.5*(r_edges[1:]+r_edges[:-1]) proj_type, nprojbins, projfn = get_proj_parameters(proj, r_edges=r_edges, cf_tag=cf_tag, **kwargs) nthreads = 24 for Nr in range(nrealizations): print(f"Realization {Nr}") # Generate cubic mock x = np.random.rand(N)*float(L) y = np.random.rand(N)*float(L) z = np.random.rand(N)*float(L) pos = np.array([x, y, z]).T print("Run DDsmu, {} basis".format(proj)) nmubins = 1 mumax = 1.0 periodic=True r_edges = None dd_res, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn, periodic=periodic, boxsize=L, verbose=True) print("DD:", dd_proj) if proj_type is None: print(dd_res) continue print("Run trr_analytic") rmin = min(r_edges) rmax = max(r_edges) nd = len(x) volume = float(L**3) nrbins = len(r_edges)-1 rr_ana, trr_ana = trr_analytic(rmin, rmax, nd, volume, nprojbins, proj_type, rbins=r_edges, projfn=projfn) print("RR_ana:", rr_ana) print("Run evaluate_xi") rcont = np.linspace(rmin, rmax, 1000) numerator = dd_proj - rr_ana amps_periodic_ana = np.linalg.solve(trr_ana, numerator) print("amplitudes:", amps_periodic_ana) xi_periodic_ana = evaluate_xi(amps_periodic_ana, rcont, proj_type, rbins=r_edges, projfn=projfn) print("evaluate_xi done")
x, y, z = read_lognormal_catalog(n='5e-5') boxsize = 750.0 nd = len(x) print("Read in lognormal catalog, number of data points:",nd) rmin = 40.0 rmax = 150.0 nbins = 11 r_edges = np.linspace(rmin, rmax, nbins+1) periodic = True nthreads = 1 proj_type = 'tophat' ncomponents = nbins nmubins = 1 mumax = 1.0 dd_res, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, boxsize=boxsize, periodic=periodic, proj_type=proj_type, ncomponents=ncomponents) print("Corrfunc result:", dd_res) print("Smoothcorrfunc result:", dd_proj) volume = boxsize**3 rr_ana, trr_ana = trr_analytic(rmin, rmax, nd, volume, ncomponents, proj_type, rbins=r_edges) numerator = dd_proj - rr_ana amps_ana, *_ = np.linalg.lstsq(trr_ana, numerator, rcond=None) # Use linalg.lstsq instead of actually computing inverse! r_fine = np.linspace(rmin, rmax, 200) xi_ana = evaluate_xi(amps_ana, r_fine, proj_type, rbins=r_edges) print(xi_ana) print("Successfully ran suave on lognormal mock!")