def plot_bins():
#samplenums = [7,8,9,10,11]
#samplenums = [11, 10, 9, 8, 7]
samplenums = [11, 10, 9, 8, 7]
#samplenums = [7]
rps = []
wprps = []
labels = []
frac = 1
tag = '_square5k_noczcut'
#tag = '_square5k'
for samplenum in samplenums:
vals = None
fn = '../results/amps/amps_bin{}{}_top_frac{}.npy'.format(
samplenum, tag, frac)
basisfuncs = [estimator_chunks.tophat]
a, K, rpbins, pibinwidth, bin_arg, logwidth = np.load(fn)
x = run.bins_logavg(rpbins)
rp, wprp = run.calc_wprp(a, x, basisfuncs, K, rpbins, vals, pibinwidth,
bin_arg, logwidth)
rps += rp
wprps += wprp
labels.append('bin {}, {}'.format(samplenum, labels_mr[samplenum]))
fn = 'plots_2019-01-29/wprp_bins{}.png'.format(tag)
plotter.plot_wprp(rps, wprps, labels, saveto=fn)
def eval_amps_grid():
evalat = '_many'
tag = '_square1k'
basistag = '_topMrz0lingrid'
basisfuncs = [estimator_chunks.grid_Mrz]
fractag = '_frac1'
extratag = ''
fn = '../results/amps/amps{}{}{}.npy'.format(tag, basistag, fractag)
saveto = 'plots_2019-02-06/wprp{}{}{}{}.png'.format(
tag, basistag, evalat, extratag)
if evalat == '_midpoints':
vals = [-18.5, -19.5, -20.5, -21.5, -22.5]
labels = ['M_rz={} '.format(val) for val in vals]
elif evalat == '_many':
vals = np.arange(-18.5, -22.6, -0.1)
labelarr = [-18.5, -19.5, -20.5, -21.5, -22.5]
labels = [
'M_rz={} '.format(val) if round(val, 1) in labelarr else None
for val in vals
]
elif evalat == '':
vals = None
labels = ['bin{}'.format(samplenum)]
a, K, rpbins, pibinwidth, bin_arg, logwidth = np.load(fn)
x = run.bins_logavg(rpbins)
print ':::a:::'
print a
rps, wprps = run.calc_wprp(a, x, basisfuncs, K, rpbins, vals, pibinwidth,
bin_arg, logwidth)
print len(rps)
print len(labels)
print rps
print wprps
np.save(
'../results/wprps/wprp{}{}{}{}'.format(tag, basistag, fractag, evalat),
[rps, wprps, labels])
#fn = 'plots_2019-01-29/wprp_Mrzlin_square5k_midpoints.png'
#fn = 'plots_2019-01-29/wprp_Mrzall_square5k_-22.5to-18.5.png'
plotter.plot_wprp(rps, wprps, labels, saveto=saveto)
def run_bins(min_sep, max_sep, bin_size, wp, saveto=None):
#samplenums = [7, 8, 9, 10, 11, 12]
#samplenums = [14, 16, 18, 19, 20, 21]
samplenums = [10]
frac = 1
saveto = "../results/dr72_bin{}_frac{}_pibinmaxweights1rand0.1n1.npy".format(
samplenums[0], frac)
print saveto
cosmo = LambdaCDM(H0=70, Om0=0.25, Ob0=0.045, Ode0=0.75)
K = (np.log10(max_sep) - np.log10(min_sep)) / bin_size
rpbins = np.logspace(np.log10(min_sep), np.log10(max_sep), K + 1)
rpbins_avg = run.bins_logavg(rpbins)
rps = []
#pibinwidth = 1 #Mpc/h
wprps = []
labels = []
cols = []
for samplenum in samplenums:
if samplenum in labels_mr:
labels.append(labels_mr[samplenum])
else:
labels.append(samplenum)
pimax = pimaxs[samplenum]
pibinwidth = pimax
rp_avg, wprp = run_sample(samplenum,
min_sep,
max_sep,
rpbins,
pimax,
wp,
cosmo,
frac=frac,
pibinwidth=pibinwidth)
print samplenum
print wprp
rps.append(rp_avg)
wprps.append(wprp)
if samplenum in colors:
cols.append(colors[samplenum])
if saveto:
run.save_results(saveto, rps, wprps, labels)
def run_together(min_sep, max_sep, bin_size, K, pimax, wp):
#samplenums = [7, 8, 9, 10, 11, 12]
samplenums = [8, 9, 10]
data1, rand1 = combine_samples(samplenums)
data2 = data1.copy()
rand2 = rand1.copy()
print 'ndata={}, nrand={}'.format(len(data1.index), len(rand1.index))
rpbins = np.logspace(np.log10(min_sep), np.log10(max_sep), K + 1)
rpbins_avg = run.bins_logavg(rpbins)
logwidth = run.log_width(rpbins)
#basisfuncs = [estimator.top_z]
#basisfuncs = [estimator.top_Mr]
basisfuncs = [estimator.gauss_Mr]
#basisfuncs = [estimator.tophat]
K *= 3
#vals = [0.1, 0.15, 0.2, 0.25]
vals = [-22.5, -21.5, -20.5, -19.5, -18.5]
#vals = [-21., -20.5, -20, -19.5]
#labels = ["M_r={:.2f}".format(val) for val in vals]
labels = ['corrfunc all']
#labels = [0.15]
#vals = [None]
#labels = ['top']
cols = [
'purple', 'red', 'orange', 'green', 'blue', 'cyan', 'magenta', 'grey'
]
print 'Run'
start = time.time()
est_ls, wprp = run.run_corrfunc(data1, rand1, data2, rand2, rpbins, pimax)
rps = [rpbins_avg]
wprps = [wprp]
# rps, wprps = run.run(data1, rand1, data2, rand2, pimax, min_sep, max_sep, bin_size, basisfuncs,
# K, cosmo, wp, rpbins, vals, logrpbins_avg, logwidth)
end = time.time()
print 'Time for all, ndata={}: {}'.format(len(data1.index), end - start)
plotter.plot_wprp(rps, wprps, labels, colors=cols)
def eval_amps():
# evalat = '_many'
# tag = '_square5k'
# basistag = '_topMrz0linsumnoabs'
# #basistag = '_toplogMrz0linmean'
# #basistag = 'topMrzallmean'
# basisfuncs = [estimator_chunks.top_Mrz]
# fractag = '_frac1'
# #extratag = '_valx0.5'
# #extratag = '_doublecheck'
# savetag = '
# '
# evalat = '_midpoints'
# samplenum = 'bins'
# tag = '_square5k'
# fractag = '_frac1'
# basistag = '_grid'
# basisfuncs = [estimator_chunks.grid_Mrz]
# extratag = ''
# savetag = extratag
samplenum = 11
evalat = ''
#tag = '_bin{}_czcut'.format(samplenum)
tag = '_square5k'
basistag = '_toprobust'
basisfuncs = [estimator_chunks.tophat_robust]
#fractag = '_frac1rand0.1'
fractag = '_frac'
savetag = ''
extratag = ''
# samplenum = 7
# evalat = ''
# tag = '_bin{}_square1k'.format(samplenum)
# samplenum = ''
# basistag = '_top'
# basisfuncs = [estimator_chunks.tophat]
# fractag = '_frac1'
#extratag = ''
# savetag = ''
# evalat = '_midpoints'
# tag = '_square5k'
# fractag = '_frac1'
# #basistag = '_matchbins'
# #basistag = '_matchbinsandrp2'
# #basistag = '_matchbrighter'
# #basistag = '_matchdimmest'
# basistag = '_matchbins'
# basisfuncs = [estimator_chunks.match_bins]
# #extratag = '_binwidthdec'
# extratag = ''
# savetag = extratag
# samplenum = 0
if samplenum == 'bins':
bintag = '_bins'
else:
bintag = '_bin{}'.format(samplenum)
fn = '../results/amps/amps{}{}{}{}{}.npy'.format(bintag, tag, basistag,
fractag, extratag)
saveto = 'plots_2019-02-06/wprp{}{}{}{}{}.png'.format(
bintag, tag, basistag, evalat, savetag)
savetowp = '../results/wprps/wprp{}{}{}{}{}.png'.format(
bintag, tag, basistag, evalat, savetag)
#basisfuncs = [estimator_chunks.tophat]
#basisfuncs = [estimator_chunks.top_rand]
if evalat == '_midpoints':
vals = [-18.5, -19.5, -20.5, -21.5, -22.5]
#vals = [-18.3, -18.75, -19.5, -21]
#vals = [-18.5, -20, -21.5]
labels = ['M_rz={} '.format(val) for val in vals]
elif evalat == '_many':
vals = np.arange(-18.5, -22.6, -0.1)
labelarr = [-18.5, -19.5, -20.5, -21.5, -22.5]
labels = [
'M_rz={} '.format(val) if round(val, 1) in labelarr else None
for val in vals
]
elif evalat == '':
vals = None
labels = ['bin{}'.format(samplenum)]
a, K, rpbins, pibinwidth, bin_arg, logwidth = np.load(fn)
x = run.bins_logavg(rpbins)
print ':::a:::'
print a
rps, wprps = run.calc_wprp(a, x, basisfuncs, K, rpbins, vals, pibinwidth,
bin_arg, logwidth)
print len(rps)
print len(labels)
#print rps
#print wprps
np.save(savetowp, [rps, wprps, labels])
def run_est_grid(samplenum):
nproc = 24
frac = 1
tag = '_square5k'
#basistag = 'gridMrz0lin'
basisfuncs = [estimator_chunks.grid_Mrz]
basistag = '_gridmean'
#basistag = 'matchdimmest'
#basistag = '_multitest'
#basistag = '_matchbins'
#basisfuncs = [estimator_chunks.match_bins]
Kfac = 5
vals = [-18.5, -19.5, -20.5, -21.5, -22.5]
#savetag = '_binwidthdec'
if samplenum == 'bins':
bintag = '_bins'
else:
bintag = '_bin{}'.format(samplenum)
fractag = '_frac{}'.format(frac)
saveto = '../results/amps/amps{}{}{}{}.npy'.format(bintag, tag, basistag,
fractag)
#Separations should be given in Mpc/h
min_sep = 0.13
max_sep = 40. #Mpc/h
#K = 100
bin_size = 0.2
K = int((np.log10(max_sep) - np.log10(min_sep)) / bin_size)
print 'K:', K
print 'Kfac:', Kfac
print 'frac:', frac
rpbins = np.logspace(np.log10(min_sep), np.log10(max_sep), K + 1)
print 'rpbins:', rpbins
rpbins_avg = run.bins_logavg(rpbins)
logrpbins_avg = run.logbins_avg(rpbins)
logwidth = run.log_width(rpbins)
if samplenum == 'bins':
pimax = pimaxs[samplenum]
else:
pimax = pimaxs[int(samplenum)]
pibinwidth = int(pimax)
bin_arg = np.log10(rpbins)
wp = True
cosmo = LambdaCDM(H0=70, Om0=0.25, Ob0=0.045, Ode0=0.75)
###############
print 'Load samples'
fn = '../data/lss.dr72bright{}{}.dat'.format(samplenum, tag)
data1 = pd.read_csv(fn)
fn_rand = '../data/random-0.dr72bright{}{}.dat'.format(samplenum, tag)
rand1 = pd.read_csv(fn_rand)
data1 = run.add_info(data1)
rand1 = run.add_info(rand1)
#data1 = data1.sample(frac=frac)
#rand1 = rand1.sample(frac=frac)
data1 = data1[:int(frac * len(data1.index))]
rand1 = rand1[:int(frac * len(rand1.index))]
print len(data1), len(rand1)
data2 = data1
rand2 = rand1
##################
rps = []
wprps = []
K *= Kfac
print 'Running vecest'
rpsest, wprpsest, a = run.run_chunks(data1, rand1, data2, rand2, pimax,
min_sep, max_sep, basisfuncs, K,
cosmo, wp, rpbins, vals, pibinwidth,
nproc, bin_arg, logwidth)
rps += [rpbins_avg] * len(wprpsest)
wprps += wprpsest
print 'Saving to {}'.format(saveto)
np.save(saveto, [a, K, rpbins, pibinwidth, bin_arg, logwidth])
print 'Rp, wprps, a'
print rps
print wprps
print a
def run_est(samplenum):
nproc = 2
frac = 1
tag = '_square1k'
basistag = 'toprobust'
basisfuncs = [estimator_chunks.tophat_robust]
Kfac = 1
vals = None
# tag = '_square5k'
# basistag = 'topMrz0linsumnoabs'
# basisfuncs = [estimator_chunks.top_Mrz]
# Kfac = 2
# vals = [-18.5, -19.5, -20.5, -21.5, -22.5]
#tag = '_czcut'
#basistag = 'top'
#basisfuncs = [estimator_chunks.tophat]
#Kfac = 1
#vals = None
if samplenum == 'bins':
bintag = '_bins'
else:
bintag = '_bin{}'.format(samplenum)
fractag = '_frac'.format(frac)
#saveto = '../results/amps/amps{}{}{}{}.npy'.format(bintag, tag, basistag, fractag)
saveto = None
#Separations should be given in Mpc/h
min_sep = 0.13
max_sep = 40. #Mpc/h
#max_sep = 1. #Mpc/h
bin_size = 0.2
K = int((np.log10(max_sep) - np.log10(min_sep)) / bin_size)
rpbins = np.logspace(np.log10(min_sep), np.log10(max_sep), K + 1)
rpbins_avg = run.bins_logavg(rpbins)
logrpbins_avg = run.logbins_avg(rpbins)
logwidth = run.log_width(rpbins)
bin_arg = np.log10(rpbins)
#bin_arg = logrpbins_avg
#this will not work for non-int samplenum (aka 'bins')
pimax = pimaxs[int(samplenum)]
pibinwidth = int(pimax)
#bin_arg = logrpbins_avg
wp = True
cosmo = LambdaCDM(H0=70, Om0=0.25, Ob0=0.045, Ode0=0.75)
###############
print 'Load samples'
fn = '../data/lss.dr72bright{}{}.dat'.format(samplenum, tag)
data1 = pd.read_csv(fn)
fn_rand = '../data/random-0.dr72bright{}{}.dat'.format(samplenum, tag)
rand1 = pd.read_csv(fn_rand)
data1 = run.add_info(data1)
rand1 = run.add_info(rand1)
data1 = data1.sample(frac=frac)
#if samplenum>0:
# frac /= 10.
rand1 = rand1.sample(frac=frac)
#data1 = data1[:int(frac*len(data1.index))]
#rand1 = rand1[:int(frac*len(rand1.index))]
print len(data1), len(rand1)
data2 = data1
rand2 = rand1
##################
#vals = [np.mean(data1['M_rz'])]
#vals = np.linspace(min(data1['M_rz']), max(data1['M_rz']), 6)
#vals = [-18, -18.5, -19, -19.5, -20]
rps = []
wprps = []
# weights_data = data1['fgotten']
# weights_rand = rand1['fgotten']
# weights_data = None
# weights_rand = None
# print 'Corrfuncing'
# rpcf, wprpcf = run.run_corrfunc(data1, rand1, data2, rand2, rpbins, pimax, cosmo,
# weights_data=weights_data, weights_rand=weights_rand,
# pibinwidth=pibinwidth) #pimax because my est isnt binning in pi
# rps.append(rpbins_avg)
# wprps.append(wprpcf)
K *= Kfac
print 'Running vecest'
#rp, wprp = run.run(data1, rand1, data2, rand2, pimax, min_sep, max_sep,
# bin_size, basisfuncs, K, cosmo, wp, rpbins, vals, pibinwidth, bin_arg, logwidth)
rpsest, wprpsest, a = run.run_chunks(data1, rand1, data2, rand2, pimax,
min_sep, max_sep, basisfuncs, K,
cosmo, wp, rpbins, vals, pibinwidth,
nproc, bin_arg, logwidth)
rps += [rpbins_avg] * len(wprpsest)
wprps += wprpsest
if saveto:
print 'Saved to {}'.format(saveto)
np.save(saveto, [a, K, rpbins, pibinwidth, bin_arg, logwidth])
print rps
print wprps
def time_pairs():
times_cf = np.zeros(len(ndata))
times_pg = np.zeros(len(ndata))
times_pgz = np.zeros(len(ndata))
rps = []
wprps = []
nrand = []
nproc = 2
K = 10
pimax = 40. #Mpc/h
pibinwidth = pimax
min_sep = 0.1
max_sep = 10. #Mpc/h
basisfuncs = [estimator_chunks.tophat_robust]
#bin_sep = np.log(rmax / rmin) / float(K)
rpbins = np.logspace(np.log10(min_sep), np.log10(max_sep), K + 1)
rpbins_avg = run.bins_logavg(rpbins)
logrpbins_avg = run.logbins_avg(rpbins)
logwidth = run.log_width(rpbins)
bin_arg = np.log10(rpbins)
cosmo = LambdaCDM(H0=70, Om0=0.3, Ode0=0.7)
wp = True
for i in range(len(ndata)):
nd = ndata[i]
print i, ndata
data1fn = '../../lss/mangler/samples/a0.6452_0001.v5_ngc_ifield_ndata{}.rdzw'.format(
nd)
rand1fn = '../../lss/mangler/samples/a0.6452_rand20x.dr12d_cmass_ngc_ifield_ndata{}.rdz'.format(
nd)
data2fn = data1fn
rand2fn = rand1fn
data1 = pd.read_csv(data1fn)
rand1 = pd.read_csv(rand1fn)
data2 = pd.read_csv(data2fn)
rand2 = pd.read_csv(rand2fn)
nrand.append(len(rand1))
# should make so can take list
print 'Adding info to dataframes'
data1 = run.add_info(data1, zfile=None)
rand1 = run.add_info(rand1, zfile=None)
data2 = run.add_info(data2, zfile=None)
rand2 = run.add_info(rand2, zfile=None)
# start0 = time.time()
# # run.run_treecorr(data1, rand1, data2, rand2, rmin, rmax, bin_sep, pimax, wp)
# xi, d1d2pairs, d1r2pairs, d2r1pairs, r1r2pairs = run.pairs_treecorr(
# data1, rand1, data2, rand2, rmin, rmax, bin_sep, pimax, wp)
# end0 = time.time()
# print "Time treecorr pairs:", end0 - start0
# times_tcp[i] = end0 - start0
#
# start1 = time.time()
# #run.run_treecorr_orig(data1, rand1, data2, rand2, rmin, rmax, bin_sep, pimax, wp)
# end1 = time.time()
# print "Time treecorr:", end1 - start1
# times_tc[i] = end1 - start1
# start2 = time.time()
# d1d2pairs, d1r2pairs, d2r1pairs, r1r2pairs = pairs.pairs(data1, rand1, data2, rand2,
# rmax, cosmo, wp)
# end2 = time.time()
# print "Time pairs:", end2 - start2
# times_kd[i] = end2 - start2
start = time.time()
rp, wprp = run.run_corrfunc(data1,
rand1,
data2,
rand2,
rpbins,
pimax,
cosmo,
nproc=nproc,
pibinwidth=int(pibinwidth))
end = time.time()
print "Time corrfunc:", end - start
times_cf[i] = end - start
rps.append(logrpbins_avg)
wprps.append(wprp)
vals = None
start = time.time()
ddgen = pairgen.PairGen(data1, data2, max_sep, cosmo, wp)
drgen = pairgen.PairGen(data1, rand2, max_sep, cosmo, wp)
rdgen = pairgen.PairGen(data2, rand1, max_sep, cosmo, wp)
rrgen = pairgen.PairGen(rand1, rand2, max_sep, cosmo, wp)
a = estimator_chunks.est(ddgen, drgen, rdgen, rrgen, pimax, max_sep,
cosmo, basisfuncs, K, wp, nproc, bin_arg,
logwidth)
rp, wprp = run.calc_wprp(a, rpbins_avg, basisfuncs, K, rpbins, vals,
pibinwidth, bin_arg, logwidth)
rps.append(rp)
wprps.append(wprp)
end = time.time()
print "Time chunks:", end - start
times_pg[i] = end - start
start = time.time()
ddgen = pairgenz.PairGen(data1, data2, max_sep, cosmo, wp, pimax)
drgen = pairgenz.PairGen(data1, rand2, max_sep, cosmo, wp, pimax)
rdgen = pairgenz.PairGen(data2, rand1, max_sep, cosmo, wp, pimax)
rrgen = pairgenz.PairGen(rand1, rand2, max_sep, cosmo, wp, pimax)
a = estimator_chunks.est(ddgen, drgen, rdgen, rrgen, pimax, max_sep,
cosmo, basisfuncs, K, wp, nproc, bin_arg,
logwidth)
rp, wprp = run.calc_wprp(a, rpbins_avg, basisfuncs, K, rpbins, vals,
pibinwidth, bin_arg, logwidth)
rps.append(rp)
wprps.append(wprp)
end = time.time()
print "Time chunks:", end - start
times_pgz[i] = end - start
# time_arrs = [times_tc, times_kd]
# labels = ['treecorr', 'kdtree']
time_arrs = [times_cf, times_pg, times_pgz]
ndatas = [ndata] * len(time_arrs)
nrands = [nrand] * len(time_arrs)
labels = ['corrfunc', 'pairgen', 'pairgen zshells']
np.save(
'../results/times/times_zshells_n{}_nproc{}.npy'.format(
max(ndata), nproc),
[ndatas, nrands, time_arrs, labels, rps, wprps])