def test_single():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a single lens
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
nsource = 100000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
rng = np.random.RandomState(8675309)
x = (rng.random_sample(nsource)-0.5) * L
y = (rng.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * np.exp(-0.5*r2/r0**2)
g1 = -gammat * (x**2-y**2)/r2
g2 = -gammat * (2.*x*y)/r2
lens_cat = treecorr.Catalog(x=[0], y=[0], k=[kappa], x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ',kg.meanlogr - np.log(kg.meanr))
np.testing.assert_allclose(kg.meanlogr, np.log(kg.meanr), atol=1.e-3)
r = kg.meanr
true_kgt = kappa * gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_kgt)
print('ratio = ',kg.xi / true_kgt)
print('diff = ',kg.xi - true_kgt)
print('max diff = ',max(abs(kg.xi - true_kgt)))
np.testing.assert_allclose(kg.xi, true_kgt, rtol=1.e-2)
np.testing.assert_allclose(kg.xi_im, 0, atol=1.e-4)
# Check that we get the same result using the corr2 function:
lens_cat.write(os.path.join('data','kg_single_lens.dat'))
source_cat.write(os.path.join('data','kg_single_source.dat'))
config = treecorr.read_config('configs/kg_single.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg_single.out'), names=True,
skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-4)
def test_pairwise2():
# Test the same profile, but with the pairwise calcualtion:
nsource = 100000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
rng = np.random.RandomState(8675309)
x = (rng.random_sample(nsource)-0.5) * L
y = (rng.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * np.exp(-0.5*r2/r0**2)
g1 = -gammat * (x**2-y**2)/r2
g2 = -gammat * (2.*x*y)/r2
dx = (rng.random_sample(nsource)-0.5) * L
dx = (rng.random_sample(nsource)-0.5) * L
k = kappa * np.ones(nsource)
lens_cat = treecorr.Catalog(x=dx, y=dx, k=k, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x+dx, y=y+dx, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1, pairwise=True)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_kgt = kappa * gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_kgt)
print('ratio = ',kg.xi / true_kgt)
print('diff = ',kg.xi - true_kgt)
print('max diff = ',max(abs(kg.xi - true_kgt)))
np.testing.assert_allclose(kg.xi, true_kgt, rtol=1.e-2)
np.testing.assert_allclose(kg.xi_im, 0, atol=1.e-4)
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data','kg_pairwise_lens.dat'))
source_cat.write(os.path.join('data','kg_pairwise_source.dat'))
config = treecorr.read_config('configs/kg_pairwise.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg_pairwise.out'), names=True,
skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-4)
def test_varxi():
# Test that varxi is correct (or close) based on actual variance of many runs.
# Signal doesn't matter much. Use the one from test_gg.
gamma0 = 0.05
kappa0 = 0.03
r0 = 10.
L = 50. * r0
rng = np.random.RandomState(8675309)
# Note: to get a good estimate of var(xi), you need a lot of runs. The number of
# runs matters much more than the number of galaxies for getting this to pass.
if __name__ == '__main__':
ngal = 1000
nruns = 50000
tol_factor = 1
else:
ngal = 100
nruns = 5000
tol_factor = 5
all_kgs = []
for run in range(nruns):
# In addition to the shape noise below, there is shot noise from the random x,y positions.
x = (rng.random_sample(ngal) - 0.5) * L
y = (rng.random_sample(ngal) - 0.5) * L
# Varied weights are hard, but at least check that non-unit weights work correctly.
w = np.ones_like(x) * 5
r2 = (x**2 + y**2) / r0**2
g1 = -gamma0 * np.exp(-r2 / 2.) * (x**2 - y**2) / r0**2
g2 = -gamma0 * np.exp(-r2 / 2.) * (2. * x * y) / r0**2
k = kappa0 * np.exp(-r2 / 2.)
# This time, add some shape noise (different each run).
g1 += rng.normal(0, 0.3, size=ngal)
g2 += rng.normal(0, 0.3, size=ngal)
k += rng.normal(0, 0.1, size=ngal)
cat = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, k=k)
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=10., max_sep=100.)
kg.process(cat, cat)
all_kgs.append(kg)
mean_xi = np.mean([kg.xi for kg in all_kgs], axis=0)
var_xi = np.var([kg.xi for kg in all_kgs], axis=0)
mean_varxi = np.mean([kg.varxi for kg in all_kgs], axis=0)
print('mean_xi = ', mean_xi)
print('mean_varxi = ', mean_varxi)
print('var_xi = ', var_xi)
print('ratio = ', var_xi / mean_varxi)
print('max relerr for xi = ', np.max(np.abs(
(var_xi - mean_varxi) / var_xi)))
np.testing.assert_allclose(mean_varxi, var_xi, rtol=0.02 * tol_factor)
def corr2(config, logger=None):
"""Run the full two-point correlation function code based on the parameters in the
given config dict.
The function print_corr2_params() will output information about the valid parameters
that are expected to be in the config dict.
Optionally a logger parameter maybe given, in which case it is used for logging.
If not given, the logging will be based on the verbose and log_file parameters.
:param config: The configuration dict which defines what to do.
:param logger: If desired, a logger object for logging. (default: None, in which case
one will be built according to the config dict's verbose level.)
"""
# Setup logger based on config verbose value
if logger is None:
logger = treecorr.config.setup_logger(
treecorr.config.get(config,'verbose',int,1),
config.get('log_file',None))
# Check that config doesn't have any extra parameters.
# (Such values are probably typos.)
# Also convert the given parameters to the correct type, etc.
config = treecorr.config.check_config(config, corr2_valid_params, corr2_aliases, logger)
import pprint
logger.debug('Using configuration dict:\n%s',pprint.pformat(config))
if ( 'output_dots' not in config
and config.get('log_file',None) is None
and config['verbose'] >= 2 ):
config['output_dots'] = True
# Set the number of threads
num_threads = config.get('num_threads',None)
logger.debug('From config dict, num_threads = %s',num_threads)
treecorr.set_omp_threads(num_threads, logger)
# Read in the input files. Each of these is a list.
cat1 = treecorr.read_catalogs(config, 'file_name', 'file_list', 0, logger)
if len(cat1) == 0:
raise AttributeError("Either file_name or file_list is required")
cat2 = treecorr.read_catalogs(config, 'file_name2', 'file_list2', 1, logger)
rand1 = treecorr.read_catalogs(config, 'rand_file_name', 'rand_file_list', 0, logger)
rand2 = treecorr.read_catalogs(config, 'rand_file_name2', 'rand_file_list2', 1, logger)
if len(cat2) == 0 and len(rand2) > 0:
raise AttributeError("rand_file_name2 is invalid without file_name2")
logger.info("Done reading input catalogs")
# Do GG correlation function if necessary
if 'gg_file_name' in config or 'm2_file_name' in config:
logger.warning("Performing GG calculations...")
gg = treecorr.GGCorrelation(config,logger)
gg.process(cat1,cat2)
logger.info("Done GG calculations.")
if 'gg_file_name' in config:
gg.write(config['gg_file_name'])
logger.warning("Wrote GG correlation to %s",config['gg_file_name'])
if 'm2_file_name' in config:
gg.writeMapSq(config['m2_file_name'], m2_uform=config['m2_uform'])
logger.warning("Wrote Mapsq values to %s",config['m2_file_name'])
# Do NG correlation function if necessary
if 'ng_file_name' in config or 'nm_file_name' in config or 'norm_file_name' in config:
if len(cat2) == 0:
raise AttributeError("file_name2 is required for ng correlation")
logger.warning("Performing NG calculations...")
ng = treecorr.NGCorrelation(config,logger)
ng.process(cat1,cat2)
logger.info("Done NG calculation.")
# The default ng_statistic is compensated _iff_ rand files are given.
rg = None
if len(rand1) == 0:
if config.get('ng_statistic',None) == 'compensated':
raise AttributeError("rand_files is required for ng_statistic = compensated")
elif config.get('ng_statistic','compensated') == 'compensated':
rg = treecorr.NGCorrelation(config,logger)
rg.process(rand1,cat2)
logger.info("Done RG calculation.")
if 'ng_file_name' in config:
ng.write(config['ng_file_name'], rg)
logger.warning("Wrote NG correlation to %s",config['ng_file_name'])
if 'nm_file_name' in config:
ng.writeNMap(config['nm_file_name'], rg, m2_uform=config['m2_uform'])
logger.warning("Wrote NMap values to %s",config['nm_file_name'])
if 'norm_file_name' in config:
gg = treecorr.GGCorrelation(config,logger)
gg.process(cat2)
logger.info("Done GG calculation for norm")
dd = treecorr.NNCorrelation(config,logger)
dd.process(cat1)
logger.info("Done DD calculation for norm")
rr = treecorr.NNCorrelation(config,logger)
rr.process(rand1)
logger.info("Done RR calculation for norm")
dr = None
if config['nn_statistic'] == 'compensated':
dr = treecorr.NNCorrelation(config,logger)
dr.process(cat1,rand1)
logger.info("Done DR calculation for norm")
ng.writeNorm(config['norm_file_name'],gg,dd,rr,dr,rg,m2_uform=config['m2_uform'])
logger.warning("Wrote Norm values to %s",config['norm_file_name'])
# Do NN correlation function if necessary
if 'nn_file_name' in config:
if len(rand1) == 0:
raise AttributeError("rand_file_name is required for NN correlation")
if len(cat2) > 0 and len(rand2) == 0:
raise AttributeError("rand_file_name2 is required for NN cross-correlation")
logger.warning("Performing DD calculations...")
dd = treecorr.NNCorrelation(config,logger)
dd.process(cat1,cat2)
logger.info("Done DD calculations.")
dr = None
rd = None
if len(cat2) == 0:
logger.warning("Performing RR calculations...")
rr = treecorr.NNCorrelation(config,logger)
rr.process(rand1)
logger.info("Done RR calculations.")
if config['nn_statistic'] == 'compensated':
logger.warning("Performing DR calculations...")
dr = treecorr.NNCorrelation(config,logger)
dr.process(cat1,rand1)
logger.info("Done DR calculations.")
else:
logger.warning("Performing RR calculations...")
rr = treecorr.NNCorrelation(config,logger)
rr.process(rand1,rand2)
logger.info("Done RR calculations.")
if config['nn_statistic'] == 'compensated':
logger.warning("Performing DR calculations...")
dr = treecorr.NNCorrelation(config,logger)
dr.process(cat1,rand2)
logger.info("Done DR calculations.")
rd = treecorr.NNCorrelation(config,logger)
rd.process(rand1,cat2)
logger.info("Done RD calculations.")
dd.write(config['nn_file_name'],rr,dr,rd)
logger.warning("Wrote NN correlation to %s",config['nn_file_name'])
# Do KK correlation function if necessary
if 'kk_file_name' in config:
logger.warning("Performing KK calculations...")
kk = treecorr.KKCorrelation(config,logger)
kk.process(cat1,cat2)
logger.info("Done KK calculations.")
kk.write(config['kk_file_name'])
logger.warning("Wrote KK correlation to %s",config['kk_file_name'])
# Do NG correlation function if necessary
if 'nk_file_name' in config:
if len(cat2) == 0:
raise AttributeError("file_name2 is required for nk correlation")
logger.warning("Performing NK calculations...")
nk = treecorr.NKCorrelation(config,logger)
nk.process(cat1,cat2)
logger.info("Done NK calculation.")
rk = None
if len(rand1) == 0:
if config.get('nk_statistic',None) == 'compensated':
raise AttributeError("rand_files is required for nk_statistic = compensated")
elif config.get('nk_statistic','compensated') == 'compensated':
rk = treecorr.NKCorrelation(config,logger)
rk.process(rand1,cat2)
logger.info("Done RK calculation.")
nk.write(config['nk_file_name'], rk)
logger.warning("Wrote NK correlation to %s",config['nk_file_name'])
# Do KG correlation function if necessary
if 'kg_file_name' in config:
if len(cat2) == 0:
raise AttributeError("file_name2 is required for kg correlation")
logger.warning("Performing KG calculations...")
kg = treecorr.KGCorrelation(config,logger)
kg.process(cat1,cat2)
logger.info("Done KG calculation.")
kg.write(config['kg_file_name'])
logger.warning("Wrote KG correlation to %s",config['kg_file_name'])
m1config = {'ra_col': 'RA',
'dec_col': 'DEC',
'k_col': 'm1',
'w_col': 'w',
'ra_units': 'deg',
'dec_units': 'deg'}
nside = 2048
logging.info(f"Calculating for NSIDE = {nside}")
rcscat = treecorr.Catalog("rcslens.fits", rcsconfig)
szcat = treecorr.Catalog(f"szmaps_masked{maskno}_{nside}.fits", szconfig)
m1cat = treecorr.Catalog("rcslens.fits", m1config)
kg = treecorr.KGCorrelation(corrconfig,logger=logging.getLogger())
kg.process(szcat, rcscat) # Calculate the cross-correlation
kg.write(f"crosscorr{maskno}_{nside}.result")
nk = treecorr.NKCorrelation(corrconfig,logger=logging.getLogger())
nk.process(szcat, m1cat)
nk.write(f"calib{maskno}_{nside}.result")
ny = treecorr.NKCorrelation(corrconfig,logger=logging.getLogger())
ny.process(rcscat, szcat)
ny.write(f"ycorr{maskno}_{nside}.result")
ng = treecorr.NGCorrelation(corrconfig,logger=logging.getLogger())
ng.process(rcscat, rcscat)
ng.write(f"shearcorr{maskno}_{nside}.result")
def test_kg():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a bunch of foreground lenses.
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
# For each lens, we divide this by a random kappa value assigned to that lens, so
# the final kg output shoudl be just gamma_t.
nlens = 1000
nsource = 30000
r0 = 10.
L = 50. * r0
gamma0 = 0.05
rng = np.random.RandomState(8675309)
xl = (rng.random_sample(nlens)-0.5) * L
yl = (rng.random_sample(nlens)-0.5) * L
kl = rng.normal(0.23, 0.05, (nlens,) )
xs = (rng.random_sample(nsource)-0.5) * L
ys = (rng.random_sample(nsource)-0.5) * L
g1 = np.zeros( (nsource,) )
g2 = np.zeros( (nsource,) )
for x,y,k in zip(xl,yl,kl):
dx = xs-x
dy = ys-y
r2 = dx**2 + dy**2
gammat = gamma0 * np.exp(-0.5*r2/r0**2) / k
g1 += -gammat * (dx**2-dy**2)/r2
g2 += -gammat * (2.*dx*dy)/r2
lens_cat = treecorr.Catalog(x=xl, y=yl, k=kl, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=xs, y=ys, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_gt = gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_gt)
print('ratio = ',kg.xi / true_gt)
print('diff = ',kg.xi - true_gt)
print('max diff = ',max(abs(kg.xi - true_gt)))
np.testing.assert_allclose(kg.xi, true_gt, rtol=0.1)
np.testing.assert_allclose(kg.xi_im, 0., atol=1.e-2)
# Check that we get the same result using the corr2 function:
lens_cat.write(os.path.join('data','kg_lens.dat'))
source_cat.write(os.path.join('data','kg_source.dat'))
config = treecorr.read_config('configs/kg.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg.out'), names=True, skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-2)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check the fits write option
out_file_name1 = os.path.join('output','kg_out1.fits')
kg.write(out_file_name1)
data = fitsio.read(out_file_name1)
np.testing.assert_almost_equal(data['r_nom'], np.exp(kg.logr))
np.testing.assert_almost_equal(data['meanr'], kg.meanr)
np.testing.assert_almost_equal(data['meanlogr'], kg.meanlogr)
np.testing.assert_almost_equal(data['kgamT'], kg.xi)
np.testing.assert_almost_equal(data['kgamX'], kg.xi_im)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(kg.varxi))
np.testing.assert_almost_equal(data['weight'], kg.weight)
np.testing.assert_almost_equal(data['npairs'], kg.npairs)
# Check the read function
kg2 = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin')
kg2.read(out_file_name1)
np.testing.assert_almost_equal(kg2.logr, kg.logr)
np.testing.assert_almost_equal(kg2.meanr, kg.meanr)
np.testing.assert_almost_equal(kg2.meanlogr, kg.meanlogr)
np.testing.assert_almost_equal(kg2.xi, kg.xi)
np.testing.assert_almost_equal(kg2.xi_im, kg.xi_im)
np.testing.assert_almost_equal(kg2.varxi, kg.varxi)
np.testing.assert_almost_equal(kg2.weight, kg.weight)
np.testing.assert_almost_equal(kg2.npairs, kg.npairs)
assert kg2.coords == kg.coords
assert kg2.metric == kg.metric
assert kg2.sep_units == kg.sep_units
assert kg2.bin_type == kg.bin_type
def test_pairwise():
# Test the pairwise option.
ngal = 1000
s = 10.
rng = np.random.RandomState(8675309)
x1 = rng.normal(0,s, (ngal,) )
y1 = rng.normal(0,s, (ngal,) )
w1 = rng.random_sample(ngal)
k1 = rng.normal(5,1, (ngal,) )
x2 = rng.normal(0,s, (ngal,) )
y2 = rng.normal(0,s, (ngal,) )
w2 = rng.random_sample(ngal)
g12 = rng.normal(0,0.2, (ngal,) )
g22 = rng.normal(0,0.2, (ngal,) )
w1 = np.ones_like(w1)
w2 = np.ones_like(w2)
cat1 = treecorr.Catalog(x=x1, y=y1, w=w1, k=k1)
cat2 = treecorr.Catalog(x=x2, y=y2, w=w2, g1=g12, g2=g22)
min_sep = 5.
max_sep = 50.
nbins = 10
bin_size = np.log(max_sep/min_sep) / nbins
kg = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins)
kg.process_pairwise(cat1, cat2)
kg.finalize(cat1.vark, cat2.varg)
true_npairs = np.zeros(nbins, dtype=int)
true_weight = np.zeros(nbins, dtype=float)
true_xi = np.zeros(nbins, dtype=complex)
rsq = (x1-x2)**2 + (y1-y2)**2
r = np.sqrt(rsq)
logr = np.log(r)
expmialpha = ((x1-x2) - 1j*(y1-y2)) / r
ww = w1 * w2
xi = -ww * k1 * (g12 + 1j*g22) * expmialpha**2
index = np.floor(np.log(r/min_sep) / bin_size).astype(int)
mask = (index >= 0) & (index < nbins)
np.add.at(true_npairs, index[mask], 1)
np.add.at(true_weight, index[mask], ww[mask])
np.add.at(true_xi, index[mask], xi[mask])
true_xi /= true_weight
np.testing.assert_array_equal(kg.npairs, true_npairs)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-4, atol=1.e-8)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-4, atol=1.e-8)
# If cats have names, then the logger will mention them.
# Also, test running with optional args.
cat1.name = "first"
cat2.name = "second"
with CaptureLog() as cl:
kg.logger = cl.logger
kg.process_pairwise(cat1, cat2, metric='Euclidean', num_threads=2)
assert "for cats first, second" in cl.output
def test_direct():
# If the catalogs are small enough, we can do a direct calculation to see if comes out right.
# This should exactly match the treecorr result if brute=True.
ngal = 200
s = 10.
rng = np.random.RandomState(8675309)
x1 = rng.normal(0,s, (ngal,) )
y1 = rng.normal(0,s, (ngal,) )
w1 = rng.random_sample(ngal)
k1 = rng.normal(5,1, (ngal,) )
x2 = rng.normal(0,s, (ngal,) )
y2 = rng.normal(0,s, (ngal,) )
w2 = rng.random_sample(ngal)
g12 = rng.normal(0,0.2, (ngal,) )
g22 = rng.normal(0,0.2, (ngal,) )
cat1 = treecorr.Catalog(x=x1, y=y1, w=w1, k=k1)
cat2 = treecorr.Catalog(x=x2, y=y2, w=w2, g1=g12, g2=g22)
min_sep = 1.
max_sep = 50.
nbins = 50
bin_size = np.log(max_sep/min_sep) / nbins
kg = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, brute=True)
kg.process(cat1, cat2)
true_npairs = np.zeros(nbins, dtype=int)
true_weight = np.zeros(nbins, dtype=float)
true_xi = np.zeros(nbins, dtype=complex)
for i in range(ngal):
# It's hard to do all the pairs at once with numpy operations (although maybe possible).
# But we can at least do all the pairs for each entry in cat1 at once with arrays.
rsq = (x1[i]-x2)**2 + (y1[i]-y2)**2
r = np.sqrt(rsq)
logr = np.log(r)
expmialpha = ((x1[i]-x2) - 1j*(y1[i]-y2)) / r
ww = w1[i] * w2
xi = -ww * k1[i] * (g12 + 1j*g22) * expmialpha**2
index = np.floor(np.log(r/min_sep) / bin_size).astype(int)
mask = (index >= 0) & (index < nbins)
np.add.at(true_npairs, index[mask], 1)
np.add.at(true_weight, index[mask], ww[mask])
np.add.at(true_xi, index[mask], xi[mask])
true_xi /= true_weight
print('true_npairs = ',true_npairs)
print('diff = ',kg.npairs - true_npairs)
np.testing.assert_array_equal(kg.npairs, true_npairs)
print('true_weight = ',true_weight)
print('diff = ',kg.weight - true_weight)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
print('true_xi = ',true_xi)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-4, atol=1.e-8)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-4, atol=1.e-8)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check that running via the corr2 script works correctly.
config = treecorr.config.read_config('configs/kg_direct.yaml')
cat1.write(config['file_name'])
cat2.write(config['file_name2'])
treecorr.corr2(config)
data = fitsio.read(config['kg_file_name'])
np.testing.assert_allclose(data['r_nom'], kg.rnom)
np.testing.assert_allclose(data['npairs'], kg.npairs)
np.testing.assert_allclose(data['weight'], kg.weight)
np.testing.assert_allclose(data['kgamT'], kg.xi, rtol=1.e-3)
np.testing.assert_allclose(data['kgamX'], kg.xi_im, rtol=1.e-3)
# Invalid with only one file_name
del config['file_name2']
with assert_raises(TypeError):
treecorr.corr2(config)
# Repeat with binslop = 0, since code is different for bin_slop=0 and brute=True.
# And don't do any top-level recursion so we actually test not going to the leaves.
kg = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_slop=0,
max_top=0)
kg.process(cat1, cat2)
np.testing.assert_array_equal(kg.npairs, true_npairs)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-3, atol=1.e-3)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-3, atol=1.e-3)
# Check a few basic operations with a KGCorrelation object.
do_pickle(kg)
kg2 = kg.copy()
kg2 += kg
np.testing.assert_allclose(kg2.npairs, 2*kg.npairs)
np.testing.assert_allclose(kg2.weight, 2*kg.weight)
np.testing.assert_allclose(kg2.meanr, 2*kg.meanr)
np.testing.assert_allclose(kg2.meanlogr, 2*kg.meanlogr)
np.testing.assert_allclose(kg2.xi, 2*kg.xi)
np.testing.assert_allclose(kg2.xi_im, 2*kg.xi_im)
kg2.clear()
kg2 += kg
np.testing.assert_allclose(kg2.npairs, kg.npairs)
np.testing.assert_allclose(kg2.weight, kg.weight)
np.testing.assert_allclose(kg2.meanr, kg.meanr)
np.testing.assert_allclose(kg2.meanlogr, kg.meanlogr)
np.testing.assert_allclose(kg2.xi, kg.xi)
np.testing.assert_allclose(kg2.xi_im, kg.xi_im)
ascii_name = 'output/kg_ascii.txt'
kg.write(ascii_name, precision=16)
kg3 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins)
kg3.read(ascii_name)
np.testing.assert_allclose(kg3.npairs, kg.npairs)
np.testing.assert_allclose(kg3.weight, kg.weight)
np.testing.assert_allclose(kg3.meanr, kg.meanr)
np.testing.assert_allclose(kg3.meanlogr, kg.meanlogr)
np.testing.assert_allclose(kg3.xi, kg.xi)
np.testing.assert_allclose(kg3.xi_im, kg.xi_im)
fits_name = 'output/kg_fits.fits'
kg.write(fits_name)
kg4 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins)
kg4.read(fits_name)
np.testing.assert_allclose(kg4.npairs, kg.npairs)
np.testing.assert_allclose(kg4.weight, kg.weight)
np.testing.assert_allclose(kg4.meanr, kg.meanr)
np.testing.assert_allclose(kg4.meanlogr, kg.meanlogr)
np.testing.assert_allclose(kg4.xi, kg.xi)
np.testing.assert_allclose(kg4.xi_im, kg.xi_im)
with assert_raises(TypeError):
kg2 += config
kg4 = treecorr.KGCorrelation(min_sep=min_sep/2, max_sep=max_sep, nbins=nbins)
with assert_raises(ValueError):
kg2 += kg4
kg5 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep*2, nbins=nbins)
with assert_raises(ValueError):
kg2 += kg5
kg6 = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins*2)
with assert_raises(ValueError):
kg2 += kg6
def test_direct_spherical():
# Repeat in spherical coords
ngal = 100
s = 10.
rng = np.random.RandomState(8675309)
x1 = rng.normal(0,s, (ngal,) )
y1 = rng.normal(0,s, (ngal,) ) + 200 # Put everything at large y, so small angle on sky
z1 = rng.normal(0,s, (ngal,) )
w1 = rng.random_sample(ngal)
k1 = rng.normal(5,1, (ngal,) )
x2 = rng.normal(0,s, (ngal,) )
y2 = rng.normal(0,s, (ngal,) ) + 200
z2 = rng.normal(0,s, (ngal,) )
w2 = rng.random_sample(ngal)
g12 = rng.normal(0,0.2, (ngal,) )
g22 = rng.normal(0,0.2, (ngal,) )
ra1, dec1 = coord.CelestialCoord.xyz_to_radec(x1,y1,z1)
ra2, dec2 = coord.CelestialCoord.xyz_to_radec(x2,y2,z2)
cat1 = treecorr.Catalog(ra=ra1, dec=dec1, ra_units='rad', dec_units='rad', w=w1, k=k1)
cat2 = treecorr.Catalog(ra=ra2, dec=dec2, ra_units='rad', dec_units='rad', w=w2, g1=g12, g2=g22)
min_sep = 1.
max_sep = 10.
nbins = 50
bin_size = np.log(max_sep/min_sep) / nbins
kg = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins,
sep_units='deg', brute=True)
kg.process(cat1, cat2)
r1 = np.sqrt(x1**2 + y1**2 + z1**2)
r2 = np.sqrt(x2**2 + y2**2 + z2**2)
x1 /= r1; y1 /= r1; z1 /= r1
x2 /= r2; y2 /= r2; z2 /= r2
north_pole = coord.CelestialCoord(0*coord.radians, 90*coord.degrees)
true_npairs = np.zeros(nbins, dtype=int)
true_weight = np.zeros(nbins, dtype=float)
true_xi = np.zeros(nbins, dtype=complex)
c1 = [coord.CelestialCoord(r*coord.radians, d*coord.radians) for (r,d) in zip(ra1, dec1)]
c2 = [coord.CelestialCoord(r*coord.radians, d*coord.radians) for (r,d) in zip(ra2, dec2)]
for i in range(ngal):
for j in range(ngal):
rsq = (x1[i]-x2[j])**2 + (y1[i]-y2[j])**2 + (z1[i]-z2[j])**2
r = np.sqrt(rsq)
r *= coord.radians / coord.degrees
logr = np.log(r)
index = np.floor(np.log(r/min_sep) / bin_size).astype(int)
if index < 0 or index >= nbins:
continue
# Rotate shears to coordinates where line connecting is horizontal.
# Original orientation is where north is up.
theta2 = 90*coord.degrees - c2[j].angleBetween(c1[i], north_pole)
expm2theta2 = np.cos(2*theta2) - 1j * np.sin(2*theta2)
g2 = g12[j] + 1j * g22[j]
g2 *= expm2theta2
ww = w1[i] * w2[j]
xi = -ww * k1[i] * g2
true_npairs[index] += 1
true_weight[index] += ww
true_xi[index] += xi
true_xi /= true_weight
print('true_npairs = ',true_npairs)
print('diff = ',kg.npairs - true_npairs)
np.testing.assert_array_equal(kg.npairs, true_npairs)
print('true_weight = ',true_weight)
print('diff = ',kg.weight - true_weight)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
print('true_xi = ',true_xi)
print('kg.xi = ',kg.xi)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-4, atol=1.e-8)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-4, atol=1.e-8)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check that running via the corr2 script works correctly.
config = treecorr.config.read_config('configs/kg_direct_spherical.yaml')
cat1.write(config['file_name'])
cat2.write(config['file_name2'])
treecorr.corr2(config)
data = fitsio.read(config['kg_file_name'])
np.testing.assert_allclose(data['r_nom'], kg.rnom)
np.testing.assert_allclose(data['npairs'], kg.npairs)
np.testing.assert_allclose(data['weight'], kg.weight)
np.testing.assert_allclose(data['kgamT'], kg.xi, rtol=1.e-3)
np.testing.assert_allclose(data['kgamX'], kg.xi_im, rtol=1.e-3)
# Repeat with binslop = 0
# And don't do any top-level recursion so we actually test not going to the leaves.
kg = treecorr.KGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins,
sep_units='deg', bin_slop=0, max_top=0)
kg.process(cat1, cat2)
np.testing.assert_array_equal(kg.npairs, true_npairs)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-3, atol=1.e-3)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-3, atol=1.e-3)
def test_pairwise():
# Test the same profile, but with the pairwise calcualtion:
nsource = 1000000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
numpy.random.seed(8675309)
x = (numpy.random.random_sample(nsource) - 0.5) * L
y = (numpy.random.random_sample(nsource) - 0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * numpy.exp(-0.5 * r2 / r0**2)
g1 = -gammat * (x**2 - y**2) / r2
g2 = -gammat * (2. * x * y) / r2
dx = (numpy.random.random_sample(nsource) - 0.5) * L
dx = (numpy.random.random_sample(nsource) - 0.5) * L
k = kappa * numpy.ones(nsource)
lens_cat = treecorr.Catalog(x=dx,
y=dx,
k=k,
x_units='arcmin',
y_units='arcmin')
source_cat = treecorr.Catalog(x=x + dx,
y=y + dx,
g1=g1,
g2=g2,
x_units='arcmin',
y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=25.,
sep_units='arcmin',
verbose=1,
pairwise=True)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_kgt = kappa * gamma0 * numpy.exp(-0.5 * r**2 / r0**2)
print('kg.xi = ', kg.xi)
print('kg.xi_im = ', kg.xi_im)
print('true_gammat = ', true_kgt)
print('ratio = ', kg.xi / true_kgt)
print('diff = ', kg.xi - true_kgt)
print('max diff = ', max(abs(kg.xi - true_kgt)))
assert max(abs(kg.xi - true_kgt)) < 4.e-4
assert max(abs(kg.xi_im)) < 3.e-5
# Check that we get the same result using the corr2 executable:
if __name__ == '__main__':
lens_cat.write(os.path.join('data', 'kg_pairwise_lens.dat'))
source_cat.write(os.path.join('data', 'kg_pairwise_source.dat'))
import subprocess
corr2_exe = get_script_name('corr2')
p = subprocess.Popen([corr2_exe, "kg_pairwise.params"])
p.communicate()
corr2_output = numpy.genfromtxt(os.path.join('output',
'kg_pairwise.out'),
names=True)
print('kg.xi = ', kg.xi)
print('from corr2 output = ', corr2_output['kgamT'])
print('ratio = ', corr2_output['kgamT'] / kg.xi)
print('diff = ', corr2_output['kgamT'] - kg.xi)
numpy.testing.assert_almost_equal(corr2_output['kgamT'] / kg.xi,
1.,
decimal=3)
print('xi_im from corr2 output = ', corr2_output['kgamX'])
assert max(abs(corr2_output['kgamX'])) < 3.e-5
def test_single():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a single lens
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
nsource = 1000000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
numpy.random.seed(8675309)
x = (numpy.random.random_sample(nsource) - 0.5) * L
y = (numpy.random.random_sample(nsource) - 0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * numpy.exp(-0.5 * r2 / r0**2)
g1 = -gammat * (x**2 - y**2) / r2
g2 = -gammat * (2. * x * y) / r2
lens_cat = treecorr.Catalog(x=[0],
y=[0],
k=[kappa],
x_units='arcmin',
y_units='arcmin')
source_cat = treecorr.Catalog(x=x,
y=y,
g1=g1,
g2=g2,
x_units='arcmin',
y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=25.,
sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ', kg.meanlogr - numpy.log(kg.meanr))
numpy.testing.assert_almost_equal(kg.meanlogr,
numpy.log(kg.meanr),
decimal=3)
r = kg.meanr
true_kgt = kappa * gamma0 * numpy.exp(-0.5 * r**2 / r0**2)
print('kg.xi = ', kg.xi)
print('kg.xi_im = ', kg.xi_im)
print('true_gammat = ', true_kgt)
print('ratio = ', kg.xi / true_kgt)
print('diff = ', kg.xi - true_kgt)
print('max diff = ', max(abs(kg.xi - true_kgt)))
assert max(abs(kg.xi - true_kgt)) < 4.e-4
assert max(abs(kg.xi_im)) < 3.e-5
# Check that we get the same result using the corr2 executable:
if __name__ == '__main__':
lens_cat.write(os.path.join('data', 'kg_single_lens.dat'))
source_cat.write(os.path.join('data', 'kg_single_source.dat'))
import subprocess
corr2_exe = get_script_name('corr2')
p = subprocess.Popen([corr2_exe, "kg_single.params"])
p.communicate()
corr2_output = numpy.genfromtxt(os.path.join('output',
'kg_single.out'),
names=True)
print('kg.xi = ', kg.xi)
print('from corr2 output = ', corr2_output['kgamT'])
print('ratio = ', corr2_output['kgamT'] / kg.xi)
print('diff = ', corr2_output['kgamT'] - kg.xi)
numpy.testing.assert_almost_equal(corr2_output['kgamT'] / kg.xi,
1.,
decimal=3)
print('xi_im from corr2 output = ', corr2_output['kgamX'])
assert max(abs(corr2_output['kgamX'])) < 3.e-5
def test_kg():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a bunch of foreground lenses.
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
nlens = 1000
nsource = 100000
gamma0 = 0.05
r0 = 10.
L = 50. * r0
numpy.random.seed(8675309)
xl = (numpy.random.random_sample(nlens) - 0.5) * L
yl = (numpy.random.random_sample(nlens) - 0.5) * L
xs = (numpy.random.random_sample(nsource) - 0.5) * L
ys = (numpy.random.random_sample(nsource) - 0.5) * L
g1 = numpy.zeros((nsource, ))
g2 = numpy.zeros((nsource, ))
kl = numpy.random.normal(0.23, 0.05, (nlens, ))
for x, y, k in zip(xl, yl, kl):
dx = xs - x
dy = ys - y
r2 = dx**2 + dy**2
gammat = gamma0 * numpy.exp(-0.5 * r2 / r0**2) / k
g1 += -gammat * (dx**2 - dy**2) / r2
g2 += -gammat * (2. * dx * dy) / r2
lens_cat = treecorr.Catalog(x=xl,
y=yl,
k=kl,
x_units='arcmin',
y_units='arcmin')
source_cat = treecorr.Catalog(x=xs,
y=ys,
g1=g1,
g2=g2,
x_units='arcmin',
y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=25.,
sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_gt = gamma0 * numpy.exp(-0.5 * r**2 / r0**2)
print('kg.xi = ', kg.xi)
print('kg.xi_im = ', kg.xi_im)
print('true_gammat = ', true_gt)
print('ratio = ', kg.xi / true_gt)
print('diff = ', kg.xi - true_gt)
print('max diff = ', max(abs(kg.xi - true_gt)))
assert max(abs(kg.xi - true_gt)) < 4.e-3
assert max(abs(kg.xi_im)) < 4.e-3
# Check that we get the same result using the corr2 executable:
if __name__ == '__main__':
lens_cat.write(os.path.join('data', 'kg_lens.dat'))
source_cat.write(os.path.join('data', 'kg_source.dat'))
import subprocess
corr2_exe = get_script_name('corr2')
p = subprocess.Popen([corr2_exe, "kg.params"])
p.communicate()
corr2_output = numpy.genfromtxt(os.path.join('output', 'kg.out'),
names=True)
print('kg.xi = ', kg.xi)
print('from corr2 output = ', corr2_output['kgamT'])
print('ratio = ', corr2_output['kgamT'] / kg.xi)
print('diff = ', corr2_output['kgamT'] - kg.xi)
numpy.testing.assert_almost_equal(corr2_output['kgamT'] / kg.xi,
1.,
decimal=3)
print('xi_im from corr2 output = ', corr2_output['kgamX'])
assert max(abs(corr2_output['kgamX'])) < 4.e-3
# Check the fits write option
out_file_name1 = os.path.join('output', 'kg_out1.fits')
kg.write(out_file_name1)
try:
import fitsio
data = fitsio.read(out_file_name1)
numpy.testing.assert_almost_equal(data['R_nom'], numpy.exp(kg.logr))
numpy.testing.assert_almost_equal(data['meanR'], kg.meanr)
numpy.testing.assert_almost_equal(data['meanlogR'], kg.meanlogr)
numpy.testing.assert_almost_equal(data['kgamT'], kg.xi)
numpy.testing.assert_almost_equal(data['kgamX'], kg.xi_im)
numpy.testing.assert_almost_equal(data['sigma'], numpy.sqrt(kg.varxi))
numpy.testing.assert_almost_equal(data['weight'], kg.weight)
numpy.testing.assert_almost_equal(data['npairs'], kg.npairs)
except ImportError:
print('Unable to import fitsio. Skipping fits tests.')
# Check the read function
kg2 = treecorr.KGCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=25.,
sep_units='arcmin')
kg2.read(out_file_name1)
numpy.testing.assert_almost_equal(kg2.logr, kg.logr)
numpy.testing.assert_almost_equal(kg2.meanr, kg.meanr)
numpy.testing.assert_almost_equal(kg2.meanlogr, kg.meanlogr)
numpy.testing.assert_almost_equal(kg2.xi, kg.xi)
numpy.testing.assert_almost_equal(kg2.xi_im, kg.xi_im)
numpy.testing.assert_almost_equal(kg2.varxi, kg.varxi)
numpy.testing.assert_almost_equal(kg2.weight, kg.weight)
numpy.testing.assert_almost_equal(kg2.npairs, kg.npairs)
def xi_2pt(cata,
catb=None,
k=None,
ga=None,
gb=None,
corr='GG',
maska=None,
maskb=None,
wa=None,
wb=None,
ran=True,
mock=False,
erron=True,
jkmask=None,
label0='',
plot=False):
"""
This is a flexible convenience wrapper for interaction with treecorr to work on CatalogStore objects. Some basic examples are given in corr_tests() of the main testsuite.py. g1, g2 correctly by c1, c2 if ellipticities and cat.bs is true. Correction by sensitivity, 1+m applied if cat.bs=True. Weighting applied if cat.wt is true. Other config properties for treecorr stored in CatalogStore object. See catalog.py or config.py. Not all correlation types fully integrated or tested. For example, only one kappa value is currently possible. Will be updated in future as useful.
Use:
:cata, catb: CatalogStore - Must supply both cata, catb (can be same reference) if NG or NK correlation. Otherwise catb is optional.
:k: str - Array name in cata, catb to use for kappa correlation.
:ga, gb: str - Array names for g1, g2 treecorr inputs. If None assume e1, e2.
:corr: str - Type of correlation for treecorr.
:maska, maskb: [bool] - Masking array to apply to input catalogs.
:wa, wb: [float] - Additional weights to apply after cat.w is used. Combined as e.g., w=sqrt(cat.w*wa).
:ran: bool - Use randoms in correlation calculation. If True, assumes cat.ran_ra, cat.ran_dec exist.
:mock: bool - If mock catalog from sims. Used when calculating covariances from sims, not currently migrated from SV code.
:erron: bool - Calculate jackknife or sim cov errors. If False, uses treecorr error outputs. Not currently migrated from SV code. When implemented requires cat.calc_err in ('jk', 'mock').
:jkmask: [bool] - For jk, mock cov calculation loop over regions/sims.
:label0: str - Additional (optional) label string used in some outputs.
:plot: bool - Plot output?
Output (len cat.tbins):
:theta: [float] - Treecorr np.exp(meanlogr)
:out: ([float]x4) - Output of signal e.g., (xi+,xi-,xi+im,x-im). For correlations with only one xi output, (xi,0.,xi_im,0.).
:err: ([float]x4) - Same but for sqrt(var).
:chi2: ([float]x4) - Same but for chi^2 if using jk or sim covariance.
"""
maska = catalog.CatalogMethods.check_mask(cata.coadd, maska)
jkmask = catalog.CatalogMethods.check_mask(cata.coadd, jkmask)
maska0 = maska & jkmask
if wa is None:
wa = np.ones(len(cata.coadd))
e1, e2, w, ms = lin.linear_methods.get_lin_e_w_ms(cata,
xi=True,
mock=mock,
mask=maska0,
w1=wa)
if catb is None:
if corr not in ['GG', 'NN', 'KK']:
raise UseError(
'Must supply both cata,catb for NG,NK correlations.')
if ga is not None:
e1 = getattr(cata, ga + '1')[maska]
e2 = getattr(cata, ga + '2')[maska]
else:
ga = 'e'
if catb is None:
gb = ga
if (corr == 'GG') | ((catb != None) & (corr == 'KG')):
catxa = treecorr.Catalog(g1=e1,
g2=e2,
w=w,
ra=cata.ra[maska0],
dec=cata.dec[maska0],
ra_units='deg',
dec_units='deg')
catma = treecorr.Catalog(k=ms,
w=w,
ra=cata.ra[maska0],
dec=cata.dec[maska0],
ra_units='deg',
dec_units='deg')
elif (corr == 'NN') | ((catb != None) & (corr in ['NG', 'NK'])):
catxa = treecorr.Catalog(w=w,
ra=cata.ra[maska0],
dec=cata.dec[maska0],
ra_units='deg',
dec_units='deg')
if ran:
catra = treecorr.Catalog(w=w,
ra=cata.ran_ra[maska0],
dec=cata.ran_dec[maska0],
ra_units='deg',
dec_units='deg')
elif corr == 'KK':
if k is None:
raise UseError('Must specify k for KK correlation.')
if k not in dir(cata):
raise UseError('Unknown k field specified.')
catxa = treecorr.Catalog(k=getattr(cata, k)[maska0],
w=w,
ra=cata.ra[maska0],
dec=cata.dec[maska0],
ra_units='deg',
dec_units='deg')
if catb is not None:
maskb = catalog.CatalogMethods.check_mask(catb.coadd, maskb)
if wb is None:
wb = np.ones(len(catb.coadd))
e1, e2, w, ms = lin.linear_methods.get_lin_e_w_ms(catb,
xi=True,
mock=mock,
mask=maskb,
w1=wb)
if gb is not None:
e1 = getattr(cata, gb + '1')[maskb]
e2 = getattr(cata, gb + '2')[maskb]
else:
gb = 'e'
if corr in ['GG', 'NG', 'KG']:
catxb = treecorr.Catalog(g1=e1,
g2=e2,
w=w,
ra=catb.ra[maskb],
dec=catb.dec[maskb],
ra_units='deg',
dec_units='deg')
catmb = treecorr.Catalog(k=ms,
w=w,
ra=catb.ra[maskb],
dec=catb.dec[maskb],
ra_units='deg',
dec_units='deg')
elif corr == 'NN':
catxb = treecorr.Catalog(w=w,
ra=catb.ra[maskb],
dec=catb.dec[maskb],
ra_units='deg',
dec_units='deg')
if ran:
catrb = treecorr.Catalog(w=w,
ra=catb.ran_ra[maskb],
dec=catb.ran_dec[maskb],
ra_units='deg',
dec_units='deg')
elif corr in ['KK', 'NK']:
if k is None:
raise UseError('Must specify k for KK correlation.')
if k not in dir(catb):
raise UseError('Unknown k field specified.')
catxb = treecorr.Catalog(k=getattr(catb, k)[maskb],
w=w,
ra=catb.ra[maskb],
dec=catb.dec[maskb],
ra_units='deg',
dec_units='deg')
xim = None
xip_im = None
xim_im = None
ximerr = None
xiperr_im = None
ximerr_im = None
if corr == 'GG':
gg = treecorr.GGCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
kk = treecorr.KKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
if catb is None:
gg.process(catxa)
kk.process(catma)
else:
gg.process(catxa, catxb)
kk.process(catma, catmb)
xip = gg.xip / kk.xi
xim = gg.xim / kk.xi
xiperr = ximerr = np.sqrt(gg.varxi)
xip_im = gg.xip_im / kk.xi
xim_im = gg.xim_im / kk.xi
theta = np.exp(gg.meanlogr)
elif corr == 'NN':
nn = treecorr.NNCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
if ran:
nr = treecorr.NNCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
rr = treecorr.NNCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
if catb is None:
nn.process(catxa)
xip = nn.npairs
xiperr = np.sqrt(nn.npairs)
if ran:
nr.process(catxa, catra)
rr.process(catra)
xip, xiperr = nn.calculateXi(rr, nr)
xiperr = np.sqrt(xiperr)
else:
rn = treecorr.NNCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
nn.process(catxa, catxb)
xip = nn.npairs
xiperr = np.sqrt(nn.npairs)
if ran:
nr.process(catxa, catrb)
nr.process(catra, catxb)
rr.process(catra, catrb)
xip, xiperr = nn.calculateXi(rr, nr, rn)
xiperr = np.sqrt(xiperr)
theta = np.exp(nn.meanlogr)
elif corr == 'KK':
kk = treecorr.KKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
if catb is None:
kk.process(catxa)
else:
kk.process(catxa, catxb)
xip = kk.xi
xiperr = np.sqrt(kk.varxi)
theta = np.exp(kk.meanlogr)
elif corr == 'KG':
kg = treecorr.KGCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
kk = treecorr.KKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
kg.process(catxa, catxb)
kk.process(catxa, catmb)
xip = kg.xi / kk.xi
xiperr = np.sqrt(kg.varxi)
xip_im = kg.xi_im / kk.xi
theta = np.exp(kg.meanlogr)
elif corr == 'NG':
ng = treecorr.NGCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
nk = treecorr.NKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
ng.process(catxa, catxb)
nk.process(catxa, catmb)
xip = ng.xi / nk.xi
xiperr = np.sqrt(ng.varxi)
xip_im = ng.xi_im / nk.xi
if ran:
rg = treecorr.NGCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
rk = treecorr.NKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
rg.process(catra, catxb)
rk.process(catra, catmb)
xip, xip_im, xiperr = ng.calculateXi(rg)
tmpa, tmp = nk.calculateXi(rk)
if np.sum(tmpa) == 0:
tmpa = np.ones(len(xip))
xip /= tmpa
xiperr = np.sqrt(xiperr)
xip_im /= tmpa
theta = np.exp(ng.meanlogr)
elif corr == 'NK':
nk = treecorr.NKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
nk.process(catxa, catxb)
xip = nk.xi
xiperr = np.sqrt(nk.varxi)
if ran:
rk = treecorr.NKCorrelation(nbins=cata.tbins,
min_sep=cata.sep[0],
max_sep=cata.sep[1],
sep_units='arcmin',
bin_slop=cata.slop,
verbose=0)
rk.process(catra, catxb)
xip, xiperr = nk.calculateXi(rk)
xiperr = np.sqrt(xiperr)
theta = np.exp(nk.meanlogr)
out = [xip, xim, xip_im, xim_im]
err = [xiperr, ximerr, xiperr, ximerr]
chi2 = [0., 0., 0., 0.]
if erron:
kwargs = {
'catb': catb,
'k': k,
'corr': corr,
'maska': maska,
'maskb': maskb,
'wa': wa,
'wb': wb,
'ran': ran
}
if catb is None:
if corr in ['KK', 'NK', 'KG']:
label = 'xi_2pt_' + cata.name + '_' + k + '_' + corr + '_' + label0
else:
label = 'xi_2pt_' + cata.name + '_' + corr + '_' + label0
else:
if corr in ['KK', 'NK', 'KG']:
label = 'xi_2pt_' + cata.name + '-' + catb.name + '_' + k + '_' + corr + '_' + label0
else:
label = 'xi_2pt_' + cata.name + '-' + catb.name + '_' + corr + '_' + label0
if cata.calc_err == 'jk':
err, chi2 = jackknife_methods.jk(cata, xi_2pt.xi_2pt,
[xip, xim, xip_im, xim_im],
label, **kwargs)
elif cata.calc_err == 'mock':
ggperr, ggmerr, chi2p, chi2m, ceerr, cberr, cechi2, cbchi2 = BCC_Methods.jk_iter_xi(
cat,
ggp,
ggm,
ce,
cb,
mask,
w,
cosebi=cosebi,
parallel=parallel)
if plot:
fig.plot_methods.fig_create_xi(cata, corr, theta, out, err, k, ga,
gb)
return theta, out, err, chi2
def test_negw():
# Test having some weights be < 0.
# Daniel Gruen had a use case where some weights were negative, and it used to die
# with an assert error in the C++ layer.
ngal = 200
s = 10.
rng = np.random.RandomState(8675309)
x1 = rng.normal(0, s, (ngal, ))
y1 = rng.normal(0, s, (ngal, ))
w1 = rng.uniform(-0.2, 1.0, (ngal, ))
k1 = rng.normal(5, 1, (ngal, ))
x2 = rng.normal(0, s, (ngal, ))
y2 = rng.normal(0, s, (ngal, ))
w2 = rng.uniform(-3.0, 4.0, (ngal, ))
g12 = rng.normal(0, 0.2, (ngal, ))
g22 = rng.normal(0, 0.2, (ngal, ))
cat1 = treecorr.Catalog(x=x1, y=y1, w=w1, k=k1)
cat2 = treecorr.Catalog(x=x2, y=y2, w=w2, g1=g12, g2=g22)
min_sep = 1.
max_sep = 50.
nbins = 50
bin_size = np.log(max_sep / min_sep) / nbins
kg = treecorr.KGCorrelation(min_sep=min_sep,
max_sep=max_sep,
nbins=nbins,
brute=True)
kg.process(cat1, cat2)
true_npairs = np.zeros(nbins, dtype=int)
true_weight = np.zeros(nbins, dtype=float)
true_xi = np.zeros(nbins, dtype=complex)
for i in range(ngal):
# It's hard to do all the pairs at once with numpy operations (although maybe possible).
# But we can at least do all the pairs for each entry in cat1 at once with arrays.
rsq = (x1[i] - x2)**2 + (y1[i] - y2)**2
r = np.sqrt(rsq)
expmialpha = ((x1[i] - x2) - 1j * (y1[i] - y2)) / r
ww = w1[i] * w2
xi = -ww * k1[i] * (g12 + 1j * g22) * expmialpha**2
index = np.floor(np.log(r / min_sep) / bin_size).astype(int)
mask = (index >= 0) & (index < nbins)
np.add.at(true_npairs, index[mask], 1)
np.add.at(true_weight, index[mask], ww[mask])
np.add.at(true_xi, index[mask], xi[mask])
true_xi /= true_weight
print('true_npairs = ', true_npairs)
print('diff = ', kg.npairs - true_npairs)
np.testing.assert_array_equal(kg.npairs, true_npairs)
print('true_weight = ', true_weight)
print('diff = ', kg.weight - true_weight)
np.testing.assert_allclose(kg.weight, true_weight, rtol=1.e-5, atol=1.e-8)
print('true_xi = ', true_xi)
print('kg.xi = ', kg.xi)
print('kg.xi_im = ', kg.xi_im)
np.testing.assert_allclose(kg.xi, true_xi.real, rtol=1.e-4, atol=1.e-8)
np.testing.assert_allclose(kg.xi_im, true_xi.imag, rtol=1.e-4, atol=1.e-8)
def test_twod_singlebin():
# Test the singleBin function for TwoD bintype.
# N random points in 2 dimensions
rng = np.random.RandomState(8675309)
N = 5000
x = rng.uniform(-20, 20, N)
y = rng.uniform(-20, 20, N)
g1 = rng.uniform(-0.2, 0.2, N)
g2 = rng.uniform(-0.2, 0.2, N)
cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, k=10.*np.ones_like(x))
max_sep = 21.
nbins = 5 # Use very chunky bins, so more pairs of non-leaf cells can fall in single bin.
# First use brute force for reference
gg0 = treecorr.GGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', brute=True)
t0 = time.time()
gg0.process(cat)
t1 = time.time()
print('t for bs=0 = ',t1-t0)
# Now do bin_slop = 0
gg1 = treecorr.GGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0)
t0 = time.time()
gg1.process(cat)
t1 = time.time()
print('t for bs=1.e-10 = ',t1-t0)
print('max abs diff xip = ',np.max(np.abs(gg1.xip - gg0.xip)))
print('max abs diff xim = ',np.max(np.abs(gg1.xim - gg0.xim)))
np.testing.assert_array_equal(gg1.npairs, gg0.npairs)
np.testing.assert_allclose(gg1.xip, gg0.xip, atol=1.e-10)
np.testing.assert_allclose(gg1.xim, gg0.xim, atol=1.e-5)
# Now do bin_slop = 0.1
gg2 = treecorr.GGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0.1,
max_top=4)
t0 = time.time()
gg2.process(cat)
t1 = time.time()
print('t for bs=0.1 = ',t1-t0)
print('max abs diff npairs = ',np.max(np.abs(gg2.npairs - gg0.npairs)))
print('max rel diff npairs = ',np.max(np.abs(gg2.npairs - gg0.npairs)/np.abs(gg0.npairs)))
print('max abs diff xip = ',np.max(np.abs(gg2.xip - gg0.xip)))
print('max abs diff xim = ',np.max(np.abs(gg2.xim - gg0.xim)))
np.testing.assert_allclose(gg2.npairs, gg0.npairs, rtol=3.e-3)
np.testing.assert_allclose(gg2.xip, gg0.xip, atol=1.e-4)
np.testing.assert_allclose(gg2.xim, gg0.xim, atol=1.e-4)
# Repeat with NG and KG so we can test those routines too.
ng0 = treecorr.NGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', brute=True)
t0 = time.time()
ng0.process(cat, cat)
t1 = time.time()
print('t for bs=0 = ',t1-t0)
ng1 = treecorr.NGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0)
t0 = time.time()
ng1.process(cat, cat)
t1 = time.time()
print('t for bs=1.e-10 = ',t1-t0)
print('max abs diff ng.xi = ',np.max(np.abs(ng1.xi - ng0.xi)))
np.testing.assert_array_equal(ng1.npairs, ng0.npairs)
np.testing.assert_allclose(ng1.xi, ng0.xi, atol=2.e-4)
ng2 = treecorr.NGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0.1)
t0 = time.time()
ng2.process(cat, cat)
t1 = time.time()
print('t for bs=0.1 = ',t1-t0)
print('max abs diff npairs = ',np.max(np.abs(ng2.npairs - ng0.npairs)))
print('max rel diff npairs = ',np.max(np.abs(ng2.npairs - ng0.npairs)/np.abs(ng0.npairs)))
print('max abs diff ng.xi = ',np.max(np.abs(ng2.xi - ng0.xi)))
np.testing.assert_allclose(ng2.npairs, ng0.npairs, rtol=1.e-2)
np.testing.assert_allclose(ng2.xi, ng0.xi, atol=5.e-4)
kg1 = treecorr.KGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0)
t0 = time.time()
kg1.process(cat, cat)
t1 = time.time()
print('t for bs=1.e-10 = ',t1-t0)
print('ng0.xi = ',ng0.xi)
print('ng1.xi = ',ng1.xi)
print('kg1.xi = ',kg1.xi)
print('max abs diff kg.xi = ',np.max(np.abs(kg1.xi - 10.*ng0.xi)))
np.testing.assert_array_equal(kg1.npairs, ng0.npairs)
np.testing.assert_allclose(kg1.xi, 10.*ng0.xi, atol=2.e-3)
np.testing.assert_array_equal(kg1.npairs, ng1.npairs)
np.testing.assert_allclose(kg1.xi, 10.*ng1.xi, atol=1.e-10)
kg2 = treecorr.KGCorrelation(max_sep=max_sep, nbins=nbins, bin_type='TwoD', bin_slop=0.1)
t0 = time.time()
kg2.process(cat, cat)
t1 = time.time()
print('t for bs=0.1 = ',t1-t0)
print('max abs diff npairs = ',np.max(np.abs(kg2.npairs - ng0.npairs)))
print('max rel diff npairs = ',np.max(np.abs(kg2.npairs - ng0.npairs)/np.abs(ng0.npairs)))
print('max abs diff kg.xi = ',np.max(np.abs(kg2.xi - 10.*ng0.xi)))
np.testing.assert_allclose(kg2.npairs, ng0.npairs, rtol=1.e-2)
np.testing.assert_allclose(kg2.xi, 10.*ng0.xi, atol=5.e-3)
np.testing.assert_array_equal(kg2.npairs, ng2.npairs)
np.testing.assert_allclose(kg2.xi, 10.*ng2.xi, atol=1.e-10)
# Finalize by applying the total weight
ng.finalize(varg)
# The resulting ESD profile
Rnom, gamma_t, gamma_x, gamma_error, Nsrc = \
[ng.rnom, ng.xi, ng.xi_im, np.sqrt(ng.varxi), ng.npairs]
"""
if 'pc' in Runit:
### Calculate the ESD profile
config = {'min_sep': Rarcmin, 'max_sep': Rarcmax, 'nbins': Nbins,\
'metric': 'Rlens', 'min_rpar': 0, 'verbose': 0}
kg = treecorr.KGCorrelation(config)
print('Rbins (pc):', Rarcmin, Rarcmax, Nbins)
# Define the source redshift bins
nZbins = 20
Zlims = np.linspace(srcZmin, srcZmax, nZbins+1)
Zbins = Zlims[0:-1] + np.diff(Zlims)/2.
Dcbins = (cosmo.comoving_distance(Zbins).to('pc')).value
print('Zbins:', Zlims, 'dZ:', (np.amax(Zlims)-np.amin(Zlims))/nZbins)
Zmasks = [(Zlims[b] < srcZ) & (srcZ <= Zlims[b+1]) for b in np.arange(nZbins)]
Ngals = [np.sum(Zmasks[b]) for b in np.arange(nZbins)]
print('Ngalaxies:', Ngals, np.sum(Ngals), len(srcZ))
