def periodic_RR_counts(X,Y,Z,W,X2,Y2,Z2,W2,distinct=False):
# Compute the RR pair counts for two possibly distinct fields
RR_aA=np.zeros([N_jack,nrbins*nmu_bins]);
# Iterate over jackknife regions
for i,j in enumerate(J_regions):
print("Computing pair counts for non-empty jackknife %s of %s." %(i+1,N_jack))
# Compute pair counts between jackknife region and entire survey volume
# First compute contribution from JK region in data1 and all of data2:
filt=np.where(J==j)
print(filt,len(filt))
if len(filt[0])>0:
cross_RR=DDsmu(0,nthreads,binfile,mu_max,nmu_bins,X2,Y2,Z2,weights1=W2,weight_type='pair_product',
X2=X[filt],Y2=Y[filt],Z2=Z[filt],weights2=W[filt],periodic=True,verbose=False)
# Weight by average particle weighting
RR_aA[i]+=cross_RR[:]['npairs']*cross_RR[:]['weightavg']
if distinct:
# Now compute contribution from JK region in data2 and all of data1:
filt2 = np.where(J2==j)
if len(filt2[0])>0:
cross_RR=DDsmu(0,nthreads,binfile,mu_max,nmu_bins,X,Y,Z,weights1=W,weight_type='pair_product',
X2=X2[filt],Y2=Y2[filt],Z2=Z2[filt],weights2=W2[filt],periodic=True,verbose=False)
# Weight by average particle weighting
RR_aA[i]+=cross_RR[:]['npairs']*cross_RR[:]['weightavg']
RR_aA[i]/=2.
return RR_aA
def xi_proj_numeric(x, y, z, rx, ry, rz, rr_proj, trr_proj, L, r_edges, ncomponents, proj_type, projfn=None, weights=None, weights_r=None, weight_type=None, nthreads=24, periodic=True):
mumax = 1.0
nmubins = 1
verbose = False # MAKE SURE THIS IS FALSE otherwise breaks
print("computing dd...")
_, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z,
proj_type=proj_type, ncomponents=ncomponents, projfn=projfn,
boxsize=L, periodic=periodic,
weights1=weights, weight_type=weight_type)
print("DD proj:", dd_proj)
print("computing dr...")
_, dr_proj, _ = DDsmu(0, nthreads, r_edges, mumax, nmubins, x, y, z,
X2=rx, Y2=ry, Z2=rz,
proj_type=proj_type, ncomponents=ncomponents, projfn=projfn,
boxsize=L, periodic=periodic,
weights1=weights, weights2=weights_r, weight_type=weight_type)
print("DR proj:", dr_proj)
nd = len(x)
nr = len(rx)
rmin = min(r_edges)
rmax = max(r_edges)
r_cont = np.linspace(rmin, rmax, 1000)
amps = compute_amps(ncomponents, nd, nd, nr, nr, dd_proj, dr_proj, dr_proj, rr_proj, trr_proj)
# for gradient, this will just return the tophat for now; we'll do the evaluation separately
xi_proj = evaluate_xi(amps, r_cont, proj_type, rbins=r_edges, projfn=projfn)
return r_cont, xi_proj, amps
def xi_proj_analytic(x, y, z, L, r_edges, ncomponents, proj_type, projfn=None, nthreads=24):
mumax = 1.0
nmubins = 1
verbose = False # MAKE SURE THIS IS FALSE otherwise breaks
periodic = True # Periodic must be true for analytic computation!
_, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z,
proj_type=proj_type, ncomponents=ncomponents, projfn=projfn,
boxsize=L, periodic=periodic)
#dd_proj = np.random.rand(ncomponents)
rmin = min(r_edges)
rmax = max(r_edges)
volume = float(L**3)
#nrbins = len(r_edges)-1
nd = len(x)
# works up to 100 thru here
# hangs up to 15 when through next line
print(rmin, rmax, nd, volume, ncomponents, r_edges, proj_type, projfn)
rr_ana, trr_ana = trr_analytic(rmin, rmax, nd, volume, ncomponents, proj_type, rbins=r_edges, projfn=projfn)
print(rr_ana)
rcont = np.linspace(rmin, rmax, 1000)
numerator = dd_proj - rr_ana
amps_periodic_ana = np.linalg.solve(trr_ana, numerator)
#xi_periodic_ana = evaluate_xi(nrbins, amps_periodic_ana, len(rcont), rcont, nrbins, r_edges, proj_type, projfn=projfn)
xi_periodic_ana = evaluate_xi(amps_periodic_ana, rcont, proj_type, rbins=r_edges, projfn=projfn)
print("DD proj:", dd_proj)
print(numerator)
print(amps_periodic_ana)
print("First 10 xi vals:", xi_periodic_ana[:10])
return rcont, xi_periodic_ana, amps_periodic_ana
def xi_wp_cubic_mock(self,
mock,
size=1000,
xi_name=None,
wp_name=None,
verbose=False):
if type(mock) == str:
mock = np.loadtxt(mock)
results_wp = wp(size,
80,
1,
self.rbins,
mock[:, 0],
mock[:, 1],
np.clip(mock[:, 2] + mock[:, 5] / 100, 10**-5,
0.99998 * size),
verbose=verbose,
output_rpavg=True)
results_DDsmu = DDsmu(1,
1,
self.rbins,
1,
20,
mock[:, 0],
mock[:, 1],
np.clip(mock[:, 2] + mock[:, 5] / 100, 10**-5,
0.99998 * size),
boxsize=size,
verbose=verbose,
output_savg=True)
density = len(mock) / 1000**3
rmin = np.array([line[0] for line in results_DDsmu])
rmax = np.array([line[1] for line in results_DDsmu])
ravg = np.array([line[2] for line in results_DDsmu])
mu_max = np.array([line[3] for line in results_DDsmu])
mu_min = mu_max - 0.05
DD = np.array([line[4] for line in results_DDsmu])
vol = 2 / 3 * np.pi * (rmax**3 - rmin**3)
vol *= 2 * (mu_max - mu_min)
xi = DD / (density * len(mock) * vol) - 1
r = ravg.reshape(20, 20).mean(axis=1)
mono = xi.reshape(20, 20).mean(axis=1)
quad = (2.5 * (3 * (mu_max - 0.025)**2 - 1) * xi).reshape(
20, 20).mean(axis=1)
if wp_name:
np.savetxt(wp_name, results_wp, fmt="%.6f")
if xi_name:
np.savetxt(xi_name,
np.array([(self.rbins[:-1] + self.rbins[1:]) / 2, mono,
quad]).T,
fmt="%.6f")
return mono, quad, np.array([line[3] for line in results_wp])
def compute_rr_trr_numeric(rr_trr_fn, random, L, r_edges, ncomponents, proj, proj_type, projfn=None, weights_r=None, weight_type=None, nthreads=24, periodic=True):
print("computing rr...")
rx, ry, rz = random.T
mumax = 1.0
nmubins = 1
verbose = False # MAKE SURE THIS IS FALSE otherwise breaks
_, rr_proj, trr_proj = DDsmu(1, nthreads, r_edges, mumax, nmubins, rx, ry, rz,
proj_type=proj_type, ncomponents=ncomponents, projfn=projfn,
boxsize=L, periodic=periodic, weights1=weights_r, weight_type=weight_type)
print("RR proj:", rr_proj)
print("QQ proj:", trr_proj)
print(f"Saving RR and QQ to {rr_trr_fn}")
np.save(rr_trr_fn, [rr_proj, trr_proj, proj])
return rr_proj, trr_proj
def do_auto_count(model, mode='smu-pre-recon'):
'''
mode is a string that contains 'smu' or 'wp', and can include
pre/post-recon tag which gets saved as part of filename
'''
# do pair counting in 1 Mpc bins
sim_name = model.mock.header['SimName']
redshift = model.mock.redshift
model_name = model.model_name
model.ND = ND = len(model.mock.galaxy_table)
x = model.mock.galaxy_table['x']
y = model.mock.galaxy_table['y']
z = model.mock.galaxy_table['z']
filedir = os.path.join(save_dir, sim_name,
'z{}-r{}'.format(redshift, model.r))
if not os.path.exists(filedir):
os.makedirs(filedir)
# c_api_timer needs to be set to false to make sure the DD variable
# and the npy file saved are exactly the same and can be recovered
# otherwise, with c_api_timer enabled, file is in "object" format,
# not proper datatypes, and will cause indexing issues
if 'smu' in mode:
# save counting results as original structured array in npy format
DDnpy = DDsmu(1, N_threads, s_bins_counts, mu_max, n_mu_bins,
x, y, z, periodic=True, verbose=False, boxsize=L,
output_savg=True, c_api_timer=False)
DDnpy = rfn.append_fields(DDnpy, ['DD'],
[DDnpy['npairs'].astype(np.float64)/ND/ND],
usemask=False)
np.save(os.path.join(filedir, '{}-auto-paircount-DD-{}.npy'
.format(model_name, mode)),
DDnpy)
elif 'wp' in mode: # also returns wp in addition to counts
DD = wp(L, pi_max, N_threads, rp_bins, x, y, z,
output_rpavg=True, verbose=False, c_api_timer=False)
wp_txt = np.vstack([DD['rpavg'], DD['wp']]).T
np.savetxt(os.path.join(filedir, '{}-auto-{}.txt'
.format(model_name, mode)),
wp_txt, fmt=txtfmt)
def run_estimator_analytic(self):
# TODO: make that can pass rbins as None to DDsmu for e.g. generalr when dont need!
_, dd_proj, _ = DDsmu(1,
self.nthreads,
self.rbins,
self.mumax,
self.nmubins,
self.x,
self.y,
self.z,
proj_type=self.proj_type,
nprojbins=self.nprojbins,
projfn=self.projfn,
verbose=self.verbose,
boxsize=self.boxsize,
periodic=self.periodic,
isa='fallback')
volume = float(self.boxsize**3)
rr_ana, qq_ana = qq_analytic(self.rmin,
self.rmax,
self.nd,
volume,
self.nprojbins,
self.proj_type,
rbins=self.rbins,
projfn=self.projfn)
numerator = dd_proj - rr_ana
amps_periodic_ana = np.linalg.solve(qq_ana, numerator)
print("AMPS:", amps_periodic_ana)
xi_periodic_ana = evaluate_xi(amps_periodic_ana,
self.rcont,
self.proj_type,
rbins=self.rbins,
projfn=self.projfn)
return xi_periodic_ana, amps_periodic_ana
if normed:
RR_counts22 /= np.sum(W2)**2.
else:
# Compute RR counts for the periodic case (measuring mu from the Z-axis)
print("\nUsing periodic input data")
from Corrfunc.theory.DDsmu import DDsmu
# Iterate over jackknife regions
print("\nComputing pair counts for field 1 x field 1")
RR11 = DDsmu(1,
nthreads,
binfile,
mu_max,
nmu_bins,
X,
Y,
Z,
weights1=W,
weight_type='pair_product',
periodic=False,
verbose=False)
print("\nComputing pair counts for field 1 x field 2")
RR12 = DDsmu(0,
nthreads,
binfile,
mu_max,
nmu_bins,
X,
Y,
Z,
weights1=W,
def do_subcross_count(model, n_sub, do_DD=True, do_DR=True):
sim_name = model.mock.header['SimName']
model_name = model.model_name
L = model.mock.BoxSize
Lbox = model.mock.Lbox
redshift = model.mock.redshift
l_sub = L / n_sub # length of subvolume box
gt = model.mock.galaxy_table
x1, y1, z1 = gt['x'], gt['y'], gt['z']
ND1 = len(gt) # number of galaxies in entire volume of periodic box
filedir = os.path.join(save_dir, sim_name,
'z{}-r{}'.format(redshift, model.r))
for i, j, k in itertools.product(range(n_sub), repeat=3):
linind = i * n_sub**2 + j * n_sub + k # linearised index of subvolumes
mask = ((l_sub * i < gt['x']) & (gt['x'] <= l_sub * (i + 1)) &
(l_sub * j < gt['y']) & (gt['y'] <= l_sub * (j + 1)) &
(l_sub * k < gt['z']) & (gt['z'] <= l_sub * (k + 1)))
gt_sub = gt[mask] # select a subvolume of the galaxy table
ND2 = len(gt_sub) # number of galaxies in the subvolume
if do_DD:
# print('Subvolume {} of {} mask created, '
# '{} of {} galaxies selected'
# .format(linind+1, np.power(n_sub, 3),
# len(gt_sub), len(gt)))
# print('Actual min/max x, y, z of galaxy sample:'
# '({:3.2f}, {:3.2f}, {:3.2f}),'
# '({:3.2f}, {:3.2f}, {:3.2f})'
# .format(np.min(gt_sub['x']),
# np.min(gt_sub['y']),
# np.min(gt_sub['z']),
# np.max(gt_sub['x']),
# np.max(gt_sub['y']),
# np.max(gt_sub['z'])))
# print('Cross-correlation pair counting for subvolume'
# + 'with {} threads...'.format(n_threads))
# calculate D1D2, where D1 is entire box, and D2 is subvolume
x2, y2, z2 = gt_sub['x'], gt_sub['y'], gt_sub['z']
D1D2 = DDsmu(0,
n_threads,
s_bins_counts,
mu_max,
n_mu_bins,
x1,
y1,
z1,
periodic=True,
X2=x2,
Y2=y2,
Z2=z2,
verbose=False,
boxsize=L,
c_api_timer=False)
np.save(
os.path.join(
filedir, '{}-cross_{}-paircount-D1D2.npy'.format(
model_name, linind)), D1D2)
if do_DR:
NR1 = ND1
# the time trade off is ~5 minutes
if use_analytic_randoms or debug_mode:
# calculate cross-correlation DR analytically
# similar code can be found in halotools:
# https://goo.gl/W9Njbv
R1D2 = cross_analytic_random(NR1, ND2, s_bins, mu_bins,
Lbox.prod())
np.savetxt(os.path.join(
filedir, '{}-cross_{}-paircount-R1D2.txt'.format(
model_name, linind)),
R1D2,
fmt=txtfmt)
if not use_analytic_randoms or debug_mode:
# calculate D2Rbox by brute force sampling, where
# Rbox sample is homogeneous in volume of box
print('Pair counting R1D2 with Corrfunc.DDsmu, '
'{} threads...'.format(n_threads))
# force float32 (a.k.a. single precision float in C)
# to be consistent with galaxy table precision
# otherwise corrfunc raises an error
xr = np.random.uniform(0, L, NR1).astype(np.float32)
yr = np.random.uniform(0, L, NR1).astype(np.float32)
zr = np.random.uniform(0, L, NR1).astype(np.float32)
R1D2 = DDsmu(0,
n_threads,
s_bins_counts,
mu_max,
n_mu_bins,
xr,
yr,
zr,
periodic=True,
X2=x2,
Y2=y2,
Z2=z2,
verbose=False,
boxsize=L,
c_api_timer=False)
np.save(
os.path.join(
filedir, '{}-cross_{}-paircount-R1D2.npy'.format(
model_name, linind)), R1D2)
def main():
print("Go!")
print(Corrfunc.__version__)
print(Corrfunc.__file__)
nrealizations = 1
#L = 750
L = 450.0
N = 10000 # number of points in mock
proj = 'spline'
kwargs = {'order': 3}
binwidth = 10
cf_tag = f"_{proj}{kwargs['order']}_bw{binwidth}_bintest"
#proj = 'tophat'
#kwargs = {}
#binwidth = 5
#cf_tag = f"_{proj}_bw{binwidth}_basetest"
#proj = None
#kwargs = {}
#binwidth = 10
#cf_tag = f"_{proj}_bw{binwidth}"
rmin = 40.0
rmax = 150.0
r_edges = np.arange(rmin, rmax+binwidth, binwidth)
rmax = max(r_edges)
r_avg = 0.5*(r_edges[1:]+r_edges[:-1])
proj_type, nprojbins, projfn = get_proj_parameters(proj, r_edges=r_edges, cf_tag=cf_tag, **kwargs)
nthreads = 24
for Nr in range(nrealizations):
print(f"Realization {Nr}")
# Generate cubic mock
x = np.random.rand(N)*float(L)
y = np.random.rand(N)*float(L)
z = np.random.rand(N)*float(L)
pos = np.array([x, y, z]).T
print("Run DDsmu, {} basis".format(proj))
nmubins = 1
mumax = 1.0
periodic=True
r_edges = None
dd_res, dd_proj, _ = DDsmu(1, nthreads, r_edges, mumax, nmubins, x, y, z, proj_type=proj_type, nprojbins=nprojbins, projfn=projfn, periodic=periodic, boxsize=L, verbose=True)
print("DD:", dd_proj)
if proj_type is None:
print(dd_res)
continue
print("Run trr_analytic")
rmin = min(r_edges)
rmax = max(r_edges)
nd = len(x)
volume = float(L**3)
nrbins = len(r_edges)-1
rr_ana, trr_ana = trr_analytic(rmin, rmax, nd, volume, nprojbins, proj_type, rbins=r_edges, projfn=projfn)
print("RR_ana:", rr_ana)
print("Run evaluate_xi")
rcont = np.linspace(rmin, rmax, 1000)
numerator = dd_proj - rr_ana
amps_periodic_ana = np.linalg.solve(trr_ana, numerator)
print("amplitudes:", amps_periodic_ana)
xi_periodic_ana = evaluate_xi(amps_periodic_ana, rcont, proj_type, rbins=r_edges, projfn=projfn)
print("evaluate_xi done")
if all_bins[0,0]>2:
raise Exception("Radial binfile should extend close to zero")
print('%s radial bins are used in this file.' %nrbins)
print("Using periodic input data");
from Corrfunc.theory.DDsmu import DDsmu
# Compute RR counts analytically
RR_counts = 4.*np.pi/3.*(all_bins[:,1]**3-all_bins[:,0]**3)/boxsize**3*mu_max/nmu_bins
import time
init = time.time()
# Now compute DD counts
print("Computing DD pair counts")
tmpDD=DDsmu(1,nthreads,binfile,mu_max,nmu_bins,dX,dY,dZ,weights1=dW,weight_type='pair_product',verbose=True,periodic=True)
DD_counts = tmpDD[:]['npairs']*tmpDD[:]['weightavg']
DD_counts/=np.sum(dW)**2.
print("Finished after %d seconds"%(time.time()-init))
# Now use CF estimator:
xi_reshape = DD_counts.reshape(nrbins,nmu_bins)/RR_counts.reshape(nrbins,1) - 1.
if multifield:
# Compute cross fields
init = time.time()
print("Computing DD pair counts for cross pair counts")
tmpDD12=DDsmu(0,nthreads,binfile,mu_max,nmu_bins,dX,dY,dZ,weights1=dW,weight_type='pair_product',
X2=dX2,Y2=dY2,Z2=dZ2,weights2=dW2,
verbose=True,periodic=True)
DD12_counts = tmpDD12[:]['npairs']*tmpDD12[:]['weightavg']
RR_aA = np.zeros([N_jack, nrbins * nmu_bins])
# Iterate over jackknife regions
for i, j in enumerate(J_regions):
filt = np.where(J == j)
print("Computing pair counts for non-empty jackknife %s of %s." %
(i + 1, N_jack))
# Compute pair counts between jackknife region and entire survey volume
cross_RR = DDsmu(0,
nthreads,
binfile,
mu_max,
nmu_bins,
X,
Y,
Z,
weights1=W,
weight_type='pair_product',
X2=X[filt],
Y2=Y[filt],
Z2=Z[filt],
weights2=W[filt],
periodic=False,
verbose=False)
# Weight by average particle weighting
RR_aA[i] = cross_RR[:]['npairs'] * cross_RR[:]['weightavg']
# Now compute weights from pair counts
w_aA = np.zeros_like(RR_aA)
RR_a = np.sum(RR_aA, axis=0)
for i, j in enumerate(J_regions):
w_aA[i] = RR_aA[i] / RR_a # jackknife weighting for bin and region
def do_subcross_count(model, mode='smu-post-recon-std',
use_shifted_randoms=True):
'''
cross-correlation between 1/N_sub^3 of a box and the whole box
can be pre/post-reconstruction data
'''
sim_name = model.mock.header['SimName']
model_name = model.model_name
redshift = model.mock.redshift
gt = model.mock.galaxy_table
x1, y1, z1 = gt['x'], gt['y'], gt['z'] # sample 1 is the full box volume
ND1 = model.mock.ND # number of galaxies in entire volume of periodic box
filedir = os.path.join(save_dir, sim_name,
'z{}-r{}'.format(redshift, model.r))
# split the (already shuffled) randoms into N copies, each of size NData
if use_shifted_randoms:
assert hasattr(model.mock, 'shifted_randoms')
sr_list = np.array_split(model.mock.shifted_randoms, random_multiplier)
for i, j, k in product(range(N_sub), repeat=3):
linind = i*N_sub**2 + j*N_sub + k # linearised index of subvolumes
print('r = {}, x-corr counting, subvol {}...'.format(model.r, linind))
x2, y2, z2 = subvol_mask(gt['x'], gt['y'], gt['z'], i, j, k, L, N_sub)
ND2 = x2.size # number of galaxies in the subvolume
DDnpy = DDsmu(0, N_threads, s_bins_counts, mu_max, n_mu_bins,
x1, y1, z1, X2=x2, Y2=y2, Z2=z2,
periodic=True, output_savg=True, verbose=False,
boxsize=L, c_api_timer=False)
DDnpy = rfn.append_fields(
DDnpy, ['DD'], [DDnpy['npairs'].astype(np.float64)/ND1/ND2],
usemask=False)
np.save(os.path.join(filedir, '{}-cross_{}-paircount-DD-{}.npy'
.format(model_name, linind, mode)),
DDnpy)
if use_shifted_randoms and hasattr(model.mock, 'shifted_randoms'):
# calculate D2Rbox by brute force sampling, where
# Rbox sample is homogeneous in volume of box
# force float32 (a.k.a. single precision float in C)
# to be consistent with galaxy table precision
# otherwise corrfunc raises an error
sr = model.mock.shifted_randoms # random shifted array
xr1, yr1, zr1 = sr[:, 0], sr[:, 1], sr[:, 2]
xr2, yr2, zr2 = subvol_mask(xr1, yr1, zr1, i, j, k, L, N_sub)
NR1, NR2 = xr1.size, xr2.size
DRnpy = DDsmu(0, N_threads, s_bins_counts, mu_max, n_mu_bins,
x1, y1, z1, X2=xr2, Y2=yr2, Z2=zr2,
periodic=True, verbose=False, output_savg=True,
boxsize=L, c_api_timer=False)
RDnpy = DDsmu(0, N_threads, s_bins_counts, mu_max, n_mu_bins,
xr1, yr1, zr1, X2=x2, Y2=y2, Z2=z2,
periodic=True, verbose=False, output_savg=True,
boxsize=L, c_api_timer=False)
DRnpy = rfn.append_fields(
DRnpy, ['DR'], [DRnpy['npairs'].astype(np.float64)/ND1/NR2],
usemask=False)
RDnpy = rfn.append_fields(
RDnpy, ['RD'], [RDnpy['npairs'].astype(np.float64)/NR1/ND2],
usemask=False)
for npy, tag in zip([DRnpy, RDnpy], ['DR', 'RD']):
np.save(os.path.join(
filedir,
'{}-cross_{}-paircount-{}-{}-sr.npy'
.format(model_name, linind, tag, mode)),
npy)
for n in range(random_multiplier): # for each split copy of R
sr = sr_list[n] # this is one of the N copies from R split
xr1, yr1, zr1 = sr[:, 0], sr[:, 1], sr[:, 2]
xr2, yr2, zr2 = subvol_mask(xr1, yr1, zr1, i, j, k, L, N_sub)
NR1, NR2 = xr1.size, xr2.size
RRnpy = DDsmu(0, N_threads, s_bins_counts, mu_max, n_mu_bins,
xr1, yr1, zr1, X2=xr2, Y2=yr2, Z2=zr2,
periodic=True, verbose=False, output_savg=True,
boxsize=L, c_api_timer=False)
RRnpy = rfn.append_fields(
RRnpy, ['RR'],
[RRnpy['npairs'].astype(np.float64)/NR1/NR2],
usemask=False)
np.save(os.path.join(
filedir, '{}-cross_{}_{}-paircount-RR-{}-sr.npy'
.format(model_name, linind, n, mode)),
RRnpy)
elif 'post' not in mode: # use_analytic_randoms
pass
else:
raise ValueError('Post-reconstruction cross-correlation requires'
' shifted randoms for mode: {}'.format(mode))
def main():
print("Go!")
nrealizations = 1
L = 750
#L = 450.0
N = 1000 # number of points in mock
cat_tag = 'cattest'
proj = 'gradient'
kwargs = {}
binwidth = 10
cf_tag = f"_{proj}_top_bw{binwidth}"
#proj = 'tophat'
#kwargs = {}
#binwidth = 5
#cf_tag = f"_{proj}_bw{binwidth}_anatest"
#proj = None
#kwargs = {}
#binwidth = 10
#cf_tag = f"_{proj}_bw{binwidth}"
rmin = 40.0
rmax = 60.0
r_edges = np.arange(rmin, rmax + binwidth, binwidth)
rmax = max(r_edges)
nrbins = len(r_edges) - 1
r_avg = 0.5 * (r_edges[1:] + r_edges[:-1])
result_dir = '../results/results_lognormal{}'.format(cat_tag)
if not os.path.exists(result_dir):
os.makedirs(result_dir)
proj_type, nprojbins, projfn = get_proj_parameters(proj,
r_edges=r_edges,
cf_tag=cf_tag,
**kwargs)
nthreads = 2
for Nr in range(nrealizations):
print(f"Realization {Nr}")
save_fn = f'{result_dir}/cf{cf_tag}_rlz{Nr}.npy'
# Generate cubic mock
x = np.random.rand(N) * float(L)
y = np.random.rand(N) * float(L)
z = np.random.rand(N) * float(L)
ones = np.ones(N)
if proj_type == 'gradient':
weights = np.array([ones, x, y, z])
else:
weights = ones
# Generate random catalog
randmult = 2
Nr = N * randmult
xr = np.random.rand(Nr) * float(L)
yr = np.random.rand(Nr) * float(L)
zr = np.random.rand(Nr) * float(L)
ones = np.ones(Nr)
if proj_type == 'gradient':
weights_r = np.array([ones, xr, yr, zr])
else:
weights_r = ones
print("Run DDsmu, {} basis".format(proj))
nmubins = 1
mumax = 1.0
periodic = True
verbose = True
weight_type = 'pair_product_gradient'
np.set_printoptions(precision=6)
dd_res, dd_proj, _ = DDsmu(1,
nthreads,
r_edges,
mumax,
nmubins,
x,
y,
z,
weights1=weights,
proj_type=proj_type,
nprojbins=nprojbins,
projfn=projfn,
periodic=periodic,
boxsize=L,
verbose=verbose,
weight_type=weight_type)
print("DD:", np.array(dd_proj))
dr_res, dr_proj, _ = DDsmu(0,
nthreads,
r_edges,
mumax,
nmubins,
x,
y,
z,
weights1=weights,
X2=xr,
Y2=yr,
Z2=zr,
weights2=weights_r,
proj_type=proj_type,
nprojbins=nprojbins,
projfn=projfn,
periodic=periodic,
boxsize=L,
verbose=verbose,
weight_type=weight_type)
print("DR:", np.array(dr_proj))
rr_res, rr_proj, qq_proj = DDsmu(1,
nthreads,
r_edges,
mumax,
nmubins,
xr,
yr,
zr,
weights1=weights_r,
proj_type=proj_type,
nprojbins=nprojbins,
projfn=projfn,
periodic=periodic,
boxsize=L,
verbose=verbose,
weight_type=weight_type)
print("RR:", np.array(rr_proj))
if proj_type is None:
print(dd_res)
continue
amps = compute_amps(nprojbins, N, N, Nr, Nr, dd_proj, dr_proj, dr_proj,
rr_proj, qq_proj)
print(amps)
#weights1 = np.array([1.0, 325., 325., 325.])
#weights2 = np.array([1.0, 325., 325., 325.])
weights1 = np.array([1.0, 750., 750., 750.])
weights2 = np.array([1.0, 750., 750., 750.])
rcont = np.linspace(rmin, rmax, 10)
#weights1=None
#weights2=None
#weight_type=None
print(amps, rcont, proj_type, r_edges, projfn, weights1, weights2,
weight_type)
xi = evaluate_xi(amps,
rcont,
proj_type,
rbins=r_edges,
projfn=projfn,
weights1=weights1,
weights2=weights2,
weight_type=weight_type)
# for now!
print(xi)
r = []
xi = []
extra_dict = {}
np.save(save_fn, [r, xi, amps, proj, extra_dict])
def run_estimator_numeric(self):
_, dd_proj, _ = DDsmu(1,
self.nthreads,
self.rbins,
self.mumax,
self.nmubins,
self.x,
self.y,
self.z,
proj_type=self.proj_type,
nprojbins=self.nprojbins,
projfn=self.projfn,
verbose=self.verbose,
boxsize=self.boxsize,
periodic=self.periodic)
_, dr_proj, _ = DDsmu(0,
self.nthreads,
self.rbins,
self.mumax,
self.nmubins,
self.x,
self.y,
self.z,
X2=self.x_rand,
Y2=self.y_rand,
Z2=self.z_rand,
proj_type=self.proj_type,
nprojbins=self.nprojbins,
projfn=self.projfn,
verbose=self.verbose,
boxsize=self.boxsize,
periodic=self.periodic)
_, rr_proj, qq_proj = DDsmu(1,
self.nthreads,
self.rbins,
self.mumax,
self.nmubins,
self.x_rand,
self.y_rand,
self.z_rand,
proj_type=self.proj_type,
nprojbins=self.nprojbins,
projfn=self.projfn,
verbose=self.verbose,
boxsize=self.boxsize,
periodic=self.periodic)
print("nd nr", self.nd, self.nr)
amps = compute_amps(self.nprojbins, self.nd, self.nd, self.nr, self.nr,
dd_proj, dr_proj, dr_proj, rr_proj, qq_proj)
print("AMPS:", amps)
xi_proj = evaluate_xi(self.nprojbins,
amps,
len(self.rcont),
self.rcont,
len(self.rbins) - 1,
self.rbins,
self.proj_type,
projfn=self.projfn)
return xi_proj, amps
RR_counts = RR[:]['npairs'] * RR[:]['weightavg']
if normed:
RR_counts /= np.sum(W)**2.
else:
# Compute RR counts for the periodic case (measuring mu from the Z-axis)
print("Using periodic input data")
from Corrfunc.theory.DDsmu import DDsmu
print("Computing pair counts")
RR = DDsmu(1,
nthreads,
binfile,
mu_max,
nmu_bins,
X,
Y,
Z,
weights1=W,
weight_type='pair_product',
periodic=False,
verbose=False)
# Weight by average particle weighting
RR_counts = RR[:]['npairs'] * RR[:]['weightavg']
if normed:
RR_counts /= np.sum(W)**2.
# Make sure output dir exists
if len(outdir) > 0:
os.makedirs(outdir, exist_ok=1)
outfile = os.path.join(outdir, "RR_counts_n%d_m%d_11.txt" % (nrbins, nmu_bins))
s_bins_counts = np.arange(0, 151, 1)
s_bins = np.arange(0, 155, 5)
mu_bins = np.arange(0, 1.05, 0.05)
data = np.fromfile('/home/dyt/store/recon_temp/gal_cat-{}.dat'.format(tag),
dtype=np.float32).reshape(-1, 4)
x = data[:, 1]
y = data[:, 2]
z = data[:, 3]
DDnpy = DDsmu(1,
30,
s_bins_counts,
1,
100,
x,
y,
z,
periodic=True,
verbose=True,
boxsize=1100,
output_savg=True,
c_api_timer=False)
DD, savg = rebin_smu_counts(DDnpy)
savg_vec = np.sum(savg * DD, axis=1) / np.sum(DD, axis=1)
NR = ND = x.size
DR_ar, _, RR_ar = analytic_random(ND, NR, ND, NR, s_bins, mu_bins, 1100**3)
xi_s_mu = ls_estimator(ND, NR, ND, NR, DD, DR_ar, DR_ar, RR_ar)
xi_0 = tpcf_multipole(xi_s_mu, mu_bins, order=0)
xi_2 = tpcf_multipole(xi_s_mu, mu_bins, order=2)
xi_0_txt = np.vstack([savg_vec, xi_0]).T
xi_2_txt = np.vstack([savg_vec, xi_2]).T
volume = float(boxsize**3)
sbins = rbins
nsbins = len(rbins)-1
nprojbins = nbins
qq_ana = qq_analytic(float(rmin), float(rmax), nd, volume, nprojbins, nsbins, sbins, proj_type)
print(qq_ana)
# ### 3D correlation function
# Here we will compute the 3D correlation function, xi(r). We will use DD(s, mu) with a single mu bin up to mu_max = 1, so this is equivalent to DD(r). Check:
# In[10]:
ddsmu_res, _, _ = DDsmu(1, nthreads, rbins, mumax, nmubins, x, y, z, boxsize=boxsize, periodic=True)
ddsmu = extract_counts(ddsmu_res)
print(ddsmu)
dd_res = DD(1, nthreads, rbins, x, y, z, boxsize=boxsize, periodic=True)
dd = extract_counts(dd_res, col=3)
print(dd)
# ### Perform xi(s, mu) continous estimation
# In[11]:
# projection
_, dd_proj, _ = DDsmu(1, nthreads, rbins, mumax, nmubins, x, y, z,
proj_type=proj_type, nprojbins=nbins,
def do_auto_count(model, do_DD=True, do_RR=True):
# do the pair counting in 1 Mpc bins
sim_name = model.mock.header['SimName']
L = model.mock.BoxSize
Lbox = model.mock.Lbox
redshift = model.mock.redshift
model_name = model.model_name
print(
'Auto-correlation pair counting for entire volume with {} threads...'.
format(n_threads))
x = model.mock.galaxy_table['x']
y = model.mock.galaxy_table['y']
z = model.mock.galaxy_table['z']
filedir = os.path.join(save_dir, sim_name,
'z{}-r{}'.format(redshift, model.r))
if not os.path.exists(filedir):
os.makedirs(filedir)
# c_api_timer needs to be set to false to make sure the DD variable
# and the npy file saved are exactly the same and can be recovered
# otherwise, with c_api_timer enabled, file is in "object" format,
# not proper datatypes, and will cause indexing issues
if do_DD:
DD = DDsmu(1,
n_threads,
s_bins_counts,
mu_max,
n_mu_bins,
x,
y,
z,
periodic=True,
verbose=False,
boxsize=L,
c_api_timer=False)
# save counting results as original structured array in npy format
np.save(
os.path.join(filedir,
'{}-auto-paircount-DD.npy'.format(model_name)), DD)
# calculate RR analytically for auto-correlation, in coarse bins
if do_RR:
ND = len(model.mock.galaxy_table)
_, RR = auto_analytic_random(ND, s_bins, mu_bins, Lbox.prod())
# save counting results as original structured array in npy format
np.savetxt(os.path.join(filedir,
'{}-auto-paircount-RR.txt'.format(model_name)),
RR,
fmt=txtfmt)
if not use_analytic_randoms or debug_mode:
ND = len(model.mock.galaxy_table)
NR = ND
xr = np.random.uniform(0, L, NR)
yr = np.random.uniform(0, L, NR)
zr = np.random.uniform(0, L, NR)
RR = DDsmu(1,
n_threads,
s_bins_counts,
mu_max,
n_mu_bins,
xr,
yr,
zr,
periodic=True,
verbose=False,
boxsize=L,
c_api_timer=False)
np.save(
os.path.join(filedir,
'{}-auto-paircount-RR.npy'.format(model_name)), RR)
def main():
savetag = ''
proj_type = 'tophat'
ncomponents = 9
r_edges = np.linspace(10., 100., ncomponents + 1)
proj_dict = {
'tophat': {
'ncomponents': ncomponents,
'proj_func': tophat_orig,
'proj_fn': None,
'args': [r_edges],
'kwargs': {}
}
}
proj = proj_dict[proj_type]
frac = 0.001
seed = 42
allx, ally, allz = read_catalog()
N = np.int(frac * len(allx))
print("N:", N)
np.random.seed(seed)
x = np.random.choice(allx, N, replace=False)
y = np.random.choice(ally, N, replace=False)
z = np.random.choice(allz, N, replace=False)
data = np.array([x, y, z]).T
fmt = '%10.10f'
### Brute force test
s = time.time()
print('brute force')
v_dd_correct, T_dd_correct = dd_bruteforce(data, proj['proj_func'],
proj['ncomponents'],
*proj['args'], **proj['kwargs'])
e = time.time()
print(v_dd_correct)
print(T_dd_correct)
print("brute force time:", e - s, 's')
s = time.time()
print('numpy trick')
v_dd_correct, T_dd_correct = dd_bruteforce_numpy(data, proj['proj_func'],
proj['ncomponents'],
*proj['args'],
**proj['kwargs'])
e = time.time()
print(v_dd_correct)
print(T_dd_correct)
print("numpy trick brute force time:", e - s, 's')
#np.save(f'../output/correct_full_{proj_type}.npy', [v_dd_correct, T_dd_correct, proj_type, proj])
#np.savetxt(f'../output/correct_vdd_{proj_type}.npy', v_dd_correct, fmt=fmt)
#np.savetxt(f'../output/correct_Tdd_{proj_type}.npy', T_dd_correct, fmt=fmt)
#print(v_dd_correct)
#print(T_dd_correct)
### Corrfunc/suave test
nthreads = 1
mumax = 1.0
nmubins = 1
_, v_dd, T_dd = DDsmu(1,
nthreads,
r_edges,
mumax,
nmubins,
x,
y,
z,
proj_type=proj_type,
ncomponents=proj['ncomponents'],
projfn=proj['proj_fn'],
periodic=False)
T_dd = T_dd.reshape(
(ncomponents, ncomponents)
) #make code output it like this?! or maybe i didn't because it makes it easier to pass directly to compute_amps, etc
print(v_dd)
print(T_dd)
else:
# Compute RR pair counts
for i, j in enumerate(J_regions):
# Compute pair counts between jackknife region and entire survey regions
filt = np.where(rJ_RR == j)
print("Computing RR pair counts for non-empty jackknife %d of %d" %
(i + 1, N_jack))
if len(filt[0]) > 0:
cross_RR = DDsmu(0,
nthreads,
binfile,
mu_max,
nmu_bins,
rX_RR,
rY_RR,
rZ_RR,
weights1=rW_RR,
weight_type='pair_product',
X2=rX_RR[filt],
Y2=rY_RR[filt],
Z2=rZ_RR[filt],
weights2=rW_RR[filt],
verbose=False,
periodic=True)
RR_counts[
i, :] += cross_RR[:]['npairs'] * cross_RR[:]['weightavg']
RR_counts[i, :] /= np.sum(
rW_RR)**2. #np.sum(rW_RR)*np.sum(rW_RR[filt])
print("Finished RR pair counts after %d seconds" % (time.time() - init))
# Now compute DR counts
DR_counts = np.zeros_like(RR_counts)
