def test_fits():
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data','Aardvark.fit')
config = treecorr.read_config('Aardvark.yaml')
config['verbose'] = 1
# Just test a few random particular values
cat1 = treecorr.Catalog(file_name, config)
np.testing.assert_equal(len(cat1.ra), 390935)
np.testing.assert_equal(cat1.nobj, 390935)
np.testing.assert_almost_equal(cat1.ra[0], 56.4195 * (pi/180.))
np.testing.assert_almost_equal(cat1.ra[390934], 78.4782 * (pi/180.))
np.testing.assert_almost_equal(cat1.dec[290333], 83.1579 * (pi/180.))
np.testing.assert_almost_equal(cat1.g1[46392], 0.0005066675)
np.testing.assert_almost_equal(cat1.g2[46392], -0.0001006742)
np.testing.assert_almost_equal(cat1.k[46392], -0.0008628797)
# The catalog doesn't have x, y, or w, but test that functionality as well.
del config['ra_col']
del config['dec_col']
config['x_col'] = 'RA'
config['y_col'] = 'DEC'
config['w_col'] = 'MU'
config['flag_col'] = 'INDEX'
config['ignore_flag'] = 64
cat2 = treecorr.Catalog(file_name, config)
np.testing.assert_almost_equal(cat2.x[390934], 78.4782, decimal=4)
np.testing.assert_almost_equal(cat2.y[290333], 83.1579, decimal=4)
np.testing.assert_almost_equal(cat2.w[46392], 0.) # index = 1200379
np.testing.assert_almost_equal(cat2.w[46393], 0.9995946) # index = 1200386
# Test using a limited set of rows
config['first_row'] = 101
config['last_row'] = 50000
cat3 = treecorr.Catalog(file_name, config)
np.testing.assert_equal(len(cat3.x), 49900)
np.testing.assert_equal(cat3.ntot, 49900)
np.testing.assert_equal(cat3.nobj, sum(cat3.w != 0))
np.testing.assert_equal(cat3.sumw, sum(cat3.w))
np.testing.assert_equal(cat3.sumw, sum(cat2.w[100:50000]))
np.testing.assert_almost_equal(cat3.g1[46292], 0.0005066675)
np.testing.assert_almost_equal(cat3.g2[46292], -0.0001006742)
np.testing.assert_almost_equal(cat3.k[46292], -0.0008628797)
cat4 = treecorr.read_catalogs(config, key='file_name', is_rand=True)[0]
np.testing.assert_equal(len(cat4.x), 49900)
np.testing.assert_equal(cat4.ntot, 49900)
np.testing.assert_equal(cat4.nobj, sum(cat4.w != 0))
np.testing.assert_equal(cat4.sumw, sum(cat4.w))
np.testing.assert_equal(cat4.sumw, sum(cat2.w[100:50000]))
assert cat4.g1 is None
assert cat4.g2 is None
assert cat4.k is None
def test_single():
# Use kappa(r) = kappa0 exp(-r^2/2r0^2) (1-r^2/2r0^2) around a single lens
nsource = 100000
kappa0 = 0.05
r0 = 10.
L = 5. * r0
np.random.seed(8675309)
x = (np.random.random_sample(nsource)-0.5) * L
y = (np.random.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
k = kappa0 * np.exp(-0.5*r2/r0**2) * (1.-0.5*r2/r0**2)
lens_cat = treecorr.Catalog(x=[0], y=[0], x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x, y=y, k=k, x_units='arcmin', y_units='arcmin')
nk = treecorr.NKCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=1)
nk.process(lens_cat, source_cat)
r = nk.meanr
true_k = kappa0 * np.exp(-0.5*r**2/r0**2) * (1.-0.5*r**2/r0**2)
print('nk.xi = ',nk.xi)
print('true_kappa = ',true_k)
print('ratio = ',nk.xi / true_k)
print('diff = ',nk.xi - true_k)
print('max diff = ',max(abs(nk.xi - true_k)))
# Note: there is a zero crossing, so need to include atol as well as rtol
np.testing.assert_allclose(nk.xi, true_k, rtol=1.e-2, atol=1.e-4)
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data','nk_single_lens.dat'))
source_cat.write(os.path.join('data','nk_single_source.dat'))
config = treecorr.read_config('configs/nk_single.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','nk_single.out'), names=True,
skip_header=1)
print('nk.xi = ',nk.xi)
print('from corr2 output = ',corr2_output['kappa'])
print('ratio = ',corr2_output['kappa']/nk.xi)
print('diff = ',corr2_output['kappa']-nk.xi)
np.testing.assert_allclose(corr2_output['kappa'], nk.xi, rtol=1.e-3)
# There is special handling for single-row catalogs when using np.genfromtxt rather
# than pandas. So mock it up to make sure we test it.
if sys.version_info < (3,): return # mock only available on python 3
from unittest import mock
with mock.patch.dict(sys.modules, {'pandas':None}):
with CaptureLog() as cl:
treecorr.corr2(config, logger=cl.logger)
assert "Unable to import pandas" in cl.output
corr2_output = np.genfromtxt(os.path.join('output','nk_single.out'), names=True,
skip_header=1)
np.testing.assert_allclose(corr2_output['kappa'], nk.xi, rtol=1.e-3)
def test_single():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a single lens
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
nsource = 100000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
rng = np.random.RandomState(8675309)
x = (rng.random_sample(nsource)-0.5) * L
y = (rng.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * np.exp(-0.5*r2/r0**2)
g1 = -gammat * (x**2-y**2)/r2
g2 = -gammat * (2.*x*y)/r2
lens_cat = treecorr.Catalog(x=[0], y=[0], k=[kappa], x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ',kg.meanlogr - np.log(kg.meanr))
np.testing.assert_allclose(kg.meanlogr, np.log(kg.meanr), atol=1.e-3)
r = kg.meanr
true_kgt = kappa * gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_kgt)
print('ratio = ',kg.xi / true_kgt)
print('diff = ',kg.xi - true_kgt)
print('max diff = ',max(abs(kg.xi - true_kgt)))
np.testing.assert_allclose(kg.xi, true_kgt, rtol=1.e-2)
np.testing.assert_allclose(kg.xi_im, 0, atol=1.e-4)
# Check that we get the same result using the corr2 function:
lens_cat.write(os.path.join('data','kg_single_lens.dat'))
source_cat.write(os.path.join('data','kg_single_source.dat'))
config = treecorr.read_config('configs/kg_single.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg_single.out'), names=True,
skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-4)
def test_single():
# Use kappa(r) = kappa0 exp(-r^2/2r0^2) (1-r^2/2r0^2) around a single lens
nsource = 100000
kappa0 = 0.05
r0 = 10.
L = 5. * r0
rng = np.random.RandomState(8675309)
x = (rng.random_sample(nsource)-0.5) * L
y = (rng.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
k = kappa0 * np.exp(-0.5*r2/r0**2) * (1.-0.5*r2/r0**2)
lens_cat = treecorr.Catalog(x=[0], y=[0], x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x, y=y, k=k, x_units='arcmin', y_units='arcmin')
nk = treecorr.NKCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=1)
nk.process(lens_cat, source_cat)
r = nk.meanr
true_k = kappa0 * np.exp(-0.5*r**2/r0**2) * (1.-0.5*r**2/r0**2)
print('nk.xi = ',nk.xi)
print('true_kappa = ',true_k)
print('ratio = ',nk.xi / true_k)
print('diff = ',nk.xi - true_k)
print('max diff = ',max(abs(nk.xi - true_k)))
# Note: there is a zero crossing, so need to include atol as well as rtol
np.testing.assert_allclose(nk.xi, true_k, rtol=1.e-2, atol=1.e-4)
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data','nk_single_lens.dat'))
source_cat.write(os.path.join('data','nk_single_source.dat'))
config = treecorr.read_config('configs/nk_single.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','nk_single.out'), names=True,
skip_header=1)
print('nk.xi = ',nk.xi)
print('from corr2 output = ',corr2_output['kappa'])
print('ratio = ',corr2_output['kappa']/nk.xi)
print('diff = ',corr2_output['kappa']-nk.xi)
np.testing.assert_allclose(corr2_output['kappa'], nk.xi, rtol=1.e-3)
# There is special handling for single-row catalogs when using np.genfromtxt rather
# than pandas. So mock it up to make sure we test it.
if sys.version_info < (3,): return # mock only available on python 3
from unittest import mock
with mock.patch.dict(sys.modules, {'pandas':None}):
with CaptureLog() as cl:
treecorr.corr2(config, logger=cl.logger)
assert "Unable to import pandas" in cl.output
corr2_output = np.genfromtxt(os.path.join('output','nk_single.out'), names=True,
skip_header=1)
np.testing.assert_allclose(corr2_output['kappa'], nk.xi, rtol=1.e-3)
def test_pairwise2():
# Test the same profile, but with the pairwise calcualtion:
nsource = 100000
gamma0 = 0.05
kappa = 0.23
r0 = 10.
L = 5. * r0
rng = np.random.RandomState(8675309)
x = (rng.random_sample(nsource)-0.5) * L
y = (rng.random_sample(nsource)-0.5) * L
r2 = (x**2 + y**2)
gammat = gamma0 * np.exp(-0.5*r2/r0**2)
g1 = -gammat * (x**2-y**2)/r2
g2 = -gammat * (2.*x*y)/r2
dx = (rng.random_sample(nsource)-0.5) * L
dx = (rng.random_sample(nsource)-0.5) * L
k = kappa * np.ones(nsource)
lens_cat = treecorr.Catalog(x=dx, y=dx, k=k, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=x+dx, y=y+dx, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1, pairwise=True)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_kgt = kappa * gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_kgt)
print('ratio = ',kg.xi / true_kgt)
print('diff = ',kg.xi - true_kgt)
print('max diff = ',max(abs(kg.xi - true_kgt)))
np.testing.assert_allclose(kg.xi, true_kgt, rtol=1.e-2)
np.testing.assert_allclose(kg.xi_im, 0, atol=1.e-4)
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data','kg_pairwise_lens.dat'))
source_cat.write(os.path.join('data','kg_pairwise_source.dat'))
config = treecorr.read_config('configs/kg_pairwise.yaml')
config['verbose'] = 0
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg_pairwise.out'), names=True,
skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-4)
def main(): nmocks = 1 nshells = 1 config_name = '/home/anderson/3x2pt/config.yaml' config = treecorr.read_config(config_name, file_type='yaml') compute_AllMock_NNCorrelation(nmocks, nshells, config) return
def main():
nshells = 5
config_name = '/home/anderson/3x2pt/config.yaml'
config = treecorr.read_config(config_name, file_type='yaml')
compute_AllBin_NNCorrelation(nshells, config)
compute_AllPair_NGCorrelation(nshells, config)
compute_AllPair_GGCorrelation(nshells, config)
return
def test_merge():
"""Test merging two config dicts.
"""
# First a simple case with no conflicts
config1 = treecorr.read_config('Aardvark.yaml')
kwargs = {'cat_precision': 10}
valid_params = treecorr.Catalog._valid_params
config2 = treecorr.config.merge_config(config1, kwargs, valid_params)
assert config2['cat_precision'] == 10
assert config2['ra_col'] == 'RA'
assert config2['verbose'] == 2
# config is allowed to have invalid parameters
assert 'gg_file_name' in config1
assert 'gg_file_name' not in config2
# If either is None, then return subset of other that is valid
config2 = treecorr.config.merge_config(config1.copy(), None, valid_params)
for key in config2:
assert key in valid_params
if key in config1:
assert config2[key] == config1[key] or config2[key] in config1[key]
else:
assert config2[key] == valid_params[key][2]
assert 'gg_file_name' not in config2
config2 = treecorr.config.merge_config(None, kwargs.copy(), valid_params)
for key in config2:
assert key in valid_params
if key in kwargs:
assert config2[key] == kwargs[key] or config2[key] in kwargs[key]
else:
assert config2[key] == valid_params[key][2]
# If conflicts, kwargs takes precedence
kwargs['ra_col'] = 'alpha2000'
config2 = treecorr.config.merge_config(config1, kwargs,
treecorr.Catalog._valid_params)
assert config2['ra_col'] == 'alpha2000'
# If kwargs has invalid parameters, exception is raised
kwargs = {'cat_prec': 10}
with assert_raises(TypeError):
treecorr.config.merge_config(config1, kwargs,
treecorr.Catalog._valid_params)
def test_merge():
"""Test merging two config dicts.
"""
# First a simple case with no conflicts
config1 = treecorr.read_config('Aardvark.yaml')
kwargs = { 'cat_precision' : 10 }
valid_params = treecorr.Catalog._valid_params
config2 = treecorr.config.merge_config(config1, kwargs, valid_params)
assert config2['cat_precision'] == 10
assert config2['ra_col'] == 'RA'
assert config2['verbose'] == 2
# config is allowed to have invalid parameters
assert 'gg_file_name' in config1
assert 'gg_file_name' not in config2
# If either is None, then return subset of other that is valid
config2 = treecorr.config.merge_config(config1.copy(), None, valid_params)
for key in config2:
assert key in valid_params
if key in config1:
assert config2[key] == config1[key] or config2[key] in config1[key]
else:
assert config2[key] == valid_params[key][2]
assert 'gg_file_name' not in config2
config2 = treecorr.config.merge_config(None, kwargs.copy(), valid_params)
for key in config2:
assert key in valid_params
if key in kwargs:
assert config2[key] == kwargs[key] or config2[key] in kwargs[key]
else:
assert config2[key] == valid_params[key][2]
# If conflicts, kwargs takes precedence
kwargs['ra_col'] = 'alpha2000'
config2 = treecorr.config.merge_config(config1, kwargs, treecorr.Catalog._valid_params)
assert config2['ra_col'] == 'alpha2000'
# If kwargs has invalid parameters, exception is raised
kwargs = { 'cat_prec' : 10 }
with assert_raises(TypeError):
treecorr.config.merge_config(config1, kwargs, treecorr.Catalog._valid_params)
def test_parse_variables():
"""Test parse_variables functionality
"""
config = treecorr.read_config('configs/nn.yaml')
# parse_variables is used by corr2 executable to add or change items in config
# with extra command line arguments
assert 'file_name2' not in config
treecorr.config.parse_variable(config, 'file_name2 = data/nn_data.dat')
assert config['file_name2'] == 'data/nn_data.dat'
treecorr.config.parse_variable(config, 'file_name2=data/nn_data2.dat')
assert config['file_name2'] == 'data/nn_data2.dat'
# It's also used by params parsing, so removes trailing comments
treecorr.config.parse_variable(
config, 'file_name2=data/nn_data3.dat # The second file')
assert config['file_name2'] == 'data/nn_data3.dat'
# Extra whitespace is ignored
treecorr.config.parse_variable(
config, 'file_name2 = \t\tdata/nn_data4.dat ')
assert config['file_name2'] == 'data/nn_data4.dat'
# Invalid if no = sign.
with assert_raises(ValueError):
treecorr.config.parse_variable(config, 'file_name2:data/nn_data2.dat')
# Can specify lists with [], () or {}
treecorr.config.parse_variable(config, 'file_name2 = [f1, f2, f3]')
assert config['file_name2'] == ['f1', 'f2', 'f3']
treecorr.config.parse_variable(config,
'file_name2 = ( g1\t, g2\t, g3\t)')
assert config['file_name2'] == ['g1', 'g2', 'g3']
treecorr.config.parse_variable(config, 'file_name2 = {h1,h2,h3}')
assert config['file_name2'] == ['h1', 'h2', 'h3']
# In config file, can also separate by whitespace
treecorr.config.parse_variable(config, 'file_name2 = f1 g2 h3')
assert config['file_name2'] == ['f1', 'g2', 'h3']
# If starts with [, needs trailing ] or error.
with assert_raises(ValueError):
treecorr.config.parse_variable(config, 'file_name2 = [h1, h2, h3')
def test_fits():
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data','Aardvark.fit')
config = treecorr.read_config('Aardvark.yaml')
config['verbose'] = 1
# Just test a few random particular values
cat1 = treecorr.Catalog(file_name, config)
numpy.testing.assert_equal(len(cat1.ra), 390935)
numpy.testing.assert_equal(cat1.nobj, 390935)
numpy.testing.assert_almost_equal(cat1.ra[0], 56.4195 * (pi/180.))
numpy.testing.assert_almost_equal(cat1.ra[390934], 78.4782 * (pi/180.))
numpy.testing.assert_almost_equal(cat1.dec[290333], 83.1579 * (pi/180.))
numpy.testing.assert_almost_equal(cat1.g1[46392], 0.0005066675)
numpy.testing.assert_almost_equal(cat1.g2[46392], -0.0001006742)
numpy.testing.assert_almost_equal(cat1.k[46392], -0.0008628797)
# The catalog doesn't have x, y, or w, but test that functionality as well.
del config['ra_col']
del config['dec_col']
config['x_col'] = 'RA'
config['y_col'] = 'DEC'
config['w_col'] = 'MU'
config['flag_col'] = 'INDEX'
config['ignore_flag'] = 64
cat2 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat2.x[390934], 78.4782, decimal=4)
numpy.testing.assert_almost_equal(cat2.y[290333], 83.1579, decimal=4)
numpy.testing.assert_almost_equal(cat2.w[46392], 0.) # index = 1200379
numpy.testing.assert_almost_equal(cat2.w[46393], 0.9995946) # index = 1200386
# Test using a limited set of rows
config['first_row'] = 101
config['last_row'] = 50000
cat3 = treecorr.Catalog(file_name, config)
numpy.testing.assert_equal(len(cat3.x), 49900)
numpy.testing.assert_equal(cat3.ntot, 49900)
numpy.testing.assert_equal(cat3.nobj, sum(cat3.w != 0))
numpy.testing.assert_equal(cat3.sumw, sum(cat3.w))
numpy.testing.assert_equal(cat3.sumw, sum(cat2.w[100:50000]))
numpy.testing.assert_almost_equal(cat3.g1[46292], 0.0005066675)
numpy.testing.assert_almost_equal(cat3.g2[46292], -0.0001006742)
numpy.testing.assert_almost_equal(cat3.k[46292], -0.0008628797)
def test_get():
"""Test getting a parameter from a config dict
"""
config1 = treecorr.read_config('configs/kg.yaml')
assert treecorr.config.get(config1, 'x_col', int) == 1
assert treecorr.config.get(config1, 'x_col', str) == '1'
assert treecorr.config.get(config1, 'x_col') == '1'
assert treecorr.config.get(config1, 'x_col', int, 2) == 1
assert treecorr.config.get(config1, 'ra_col', int) == None
assert treecorr.config.get(config1, 'ra_col', int, 2) == 2
config1['flip_g1'] = True
assert treecorr.config.get(config1, 'flip_g1', bool) == True
assert treecorr.config.get(config1, 'flip_g1', bool, False) == True
assert treecorr.config.get(config1, 'flip_g2', bool, False) == False
assert treecorr.config.get(config1, 'flip_g2', bool) == None
assert treecorr.config.get_from_list(config1, 'k_col', 0, int) == 3
assert treecorr.config.get_from_list(config1, 'k_col', 0, str) == '3'
assert treecorr.config.get_from_list(config1, 'k_col', 0) == '3'
assert treecorr.config.get_from_list(config1, 'k_col', 0, int, 2) == 3
assert treecorr.config.get_from_list(config1, 'k_col', 1, int) == 0
assert treecorr.config.get_from_list(config1, 'k_col', 1, int, 2) == 0
assert treecorr.config.get_from_list(config1, 'ra_col', 1, int, 2) == 2
assert treecorr.config.get_from_list(config1, 'ra_col', 1, int) == None
config1['flip_g1'] = [True, False]
assert treecorr.config.get_from_list(config1, 'flip_g1', 0, bool) == True
assert treecorr.config.get_from_list(config1, 'flip_g1', 1, bool) == False
assert treecorr.config.get_from_list(config1, 'flip_g1', 0, bool,
False) == True
assert treecorr.config.get_from_list(config1, 'flip_g2', 1, bool) == None
assert treecorr.config.get_from_list(config1, 'flip_g2', 1, bool,
False) == False
assert treecorr.config.get_from_list(config1, 'flip_g2', 2, bool,
False) == False
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'k_col', 2, int)
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'flip_g1', 2, bool)
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'flip_g1', 2, bool, False)
def test_fits():
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data', 'Aardvark.fit')
config = treecorr.read_config('Aardvark.params')
# Just test a few random particular values
cat1 = treecorr.Catalog(file_name, config)
numpy.testing.assert_equal(len(cat1.ra), 390935)
numpy.testing.assert_equal(cat1.nobj, 390935)
numpy.testing.assert_almost_equal(cat1.ra[0], 56.4195 * (pi / 180.))
numpy.testing.assert_almost_equal(cat1.ra[390934], 78.4782 * (pi / 180.))
numpy.testing.assert_almost_equal(cat1.dec[290333], 83.1579 * (pi / 180.))
numpy.testing.assert_almost_equal(cat1.g1[46392], 0.0005066675)
numpy.testing.assert_almost_equal(cat1.g2[46392], -0.0001006742)
numpy.testing.assert_almost_equal(cat1.k[46392], -0.0008628797)
# The catalog doesn't have x, y, or w, but test that functionality as well.
del config['ra_col']
del config['dec_col']
config['x_col'] = 'RA'
config['y_col'] = 'DEC'
config['w_col'] = 'MU'
config['flag_col'] = 'INDEX'
config['ignore_flag'] = 64
cat2 = treecorr.Catalog(file_name, config)
numpy.testing.assert_almost_equal(cat2.x[390934], 78.4782, decimal=4)
numpy.testing.assert_almost_equal(cat2.y[290333], 83.1579, decimal=4)
numpy.testing.assert_almost_equal(cat2.w[46392], 0.) # index = 1200379
numpy.testing.assert_almost_equal(cat2.w[46393],
0.9995946) # index = 1200386
# Test using a limited set of rows
config['first_row'] = 101
config['last_row'] = 50000
cat3 = treecorr.Catalog(file_name, config)
numpy.testing.assert_equal(len(cat3.x), 49900)
numpy.testing.assert_equal(cat3.ntot, 49900)
numpy.testing.assert_equal(cat3.nobj, sum(cat3.w != 0))
numpy.testing.assert_equal(cat3.sumw, sum(cat3.w))
numpy.testing.assert_equal(cat3.sumw, sum(cat2.w[100:50000]))
numpy.testing.assert_almost_equal(cat3.g1[46292], 0.0005066675)
numpy.testing.assert_almost_equal(cat3.g2[46292], -0.0001006742)
numpy.testing.assert_almost_equal(cat3.k[46292], -0.0008628797)
def test_parse_variables():
"""Test parse_variables functionality
"""
config = treecorr.read_config('configs/nn.yaml')
# parse_variables is used by corr2 executable to add or change items in config
# with extra command line arguments
assert 'file_name2' not in config
treecorr.config.parse_variable(config, 'file_name2 = data/nn_data.dat')
assert config['file_name2'] == 'data/nn_data.dat'
treecorr.config.parse_variable(config, 'file_name2=data/nn_data2.dat')
assert config['file_name2'] == 'data/nn_data2.dat'
# It's also used by params parsing, so removes trailing comments
treecorr.config.parse_variable(config, 'file_name2=data/nn_data3.dat # The second file')
assert config['file_name2'] == 'data/nn_data3.dat'
# Extra whitespace is ignored
treecorr.config.parse_variable(config, 'file_name2 = \t\tdata/nn_data4.dat ')
assert config['file_name2'] == 'data/nn_data4.dat'
# Invalid if no = sign.
with assert_raises(ValueError):
treecorr.config.parse_variable(config, 'file_name2:data/nn_data2.dat')
# Can specify lists with [], () or {}
treecorr.config.parse_variable(config, 'file_name2 = [f1, f2, f3]')
assert config['file_name2'] == ['f1', 'f2', 'f3']
treecorr.config.parse_variable(config, 'file_name2 = ( g1\t, g2\t, g3\t)')
assert config['file_name2'] == ['g1', 'g2', 'g3']
treecorr.config.parse_variable(config, 'file_name2 = {h1,h2,h3}')
assert config['file_name2'] == ['h1', 'h2', 'h3']
# In config file, can also separate by whitespace
treecorr.config.parse_variable(config, 'file_name2 = f1 g2 h3')
assert config['file_name2'] == ['f1', 'g2', 'h3']
# If starts with [, needs trailing ] or error.
with assert_raises(ValueError):
treecorr.config.parse_variable(config, 'file_name2 = [h1, h2, h3')
def test_get():
"""Test getting a parameter from a config dict
"""
config1 = treecorr.read_config('configs/kg.yaml')
assert treecorr.config.get(config1, 'x_col', int) == 1
assert treecorr.config.get(config1, 'x_col', str) == '1'
assert treecorr.config.get(config1, 'x_col') == '1'
assert treecorr.config.get(config1, 'x_col', int, 2) == 1
assert treecorr.config.get(config1, 'ra_col', int) == None
assert treecorr.config.get(config1, 'ra_col', int, 2) == 2
config1['flip_g1'] = True
assert treecorr.config.get(config1, 'flip_g1', bool) == True
assert treecorr.config.get(config1, 'flip_g1', bool, False) == True
assert treecorr.config.get(config1, 'flip_g2', bool, False) == False
assert treecorr.config.get(config1, 'flip_g2', bool) == None
assert treecorr.config.get_from_list(config1, 'k_col', 0, int) == 3
assert treecorr.config.get_from_list(config1, 'k_col', 0, str) == '3'
assert treecorr.config.get_from_list(config1, 'k_col', 0) == '3'
assert treecorr.config.get_from_list(config1, 'k_col', 0, int, 2) == 3
assert treecorr.config.get_from_list(config1, 'k_col', 1, int) == 0
assert treecorr.config.get_from_list(config1, 'k_col', 1, int, 2) == 0
assert treecorr.config.get_from_list(config1, 'ra_col', 1, int, 2) == 2
assert treecorr.config.get_from_list(config1, 'ra_col', 1, int) == None
config1['flip_g1'] = [True, False]
assert treecorr.config.get_from_list(config1, 'flip_g1', 0, bool) == True
assert treecorr.config.get_from_list(config1, 'flip_g1', 1, bool) == False
assert treecorr.config.get_from_list(config1, 'flip_g1', 0, bool, False) == True
assert treecorr.config.get_from_list(config1, 'flip_g2', 1, bool) == None
assert treecorr.config.get_from_list(config1, 'flip_g2', 1, bool, False) == False
assert treecorr.config.get_from_list(config1, 'flip_g2', 2, bool, False) == False
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'k_col', 2, int)
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'flip_g1', 2, bool)
with assert_raises(IndexError):
treecorr.config.get_from_list(config1, 'flip_g1', 2, bool, False)
def test_kk():
# cf. http://adsabs.harvard.edu/abs/2002A%26A...389..729S for the basic formulae I use here.
#
# Use kappa(r) = A exp(-r^2/2s^2)
#
# The Fourier transform is: kappa~(k) = 2 pi A s^2 exp(-s^2 k^2/2) / L^2
# P(k) = (1/2pi) <|kappa~(k)|^2> = 2 pi A^2 (s/L)^4 exp(-s^2 k^2)
# xi(r) = (1/2pi) int( dk k P(k) J0(kr) )
# = pi A^2 (s/L)^2 exp(-r^2/2s^2/4)
# Note: I'm not sure I handled the L factors correctly, but the units at the end need
# to be kappa^2, so it needs to be (s/L)^2.
s = 10.
if __name__ == '__main__':
ngal = 1000000
L = 30. * s # Not infinity, so this introduces some error. Our integrals were to infinity.
tol_factor = 1
else:
ngal = 100000
L = 30. * s
tol_factor = 2
A = 0.05
rng = np.random.RandomState(8675309)
x = (rng.random_sample(ngal)-0.5) * L
y = (rng.random_sample(ngal)-0.5) * L
r2 = (x**2 + y**2)/s**2
kappa = A * np.exp(-r2/2.)
cat = treecorr.Catalog(x=x, y=y, k=kappa, x_units='arcmin', y_units='arcmin')
kk = treecorr.KKCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kk.process(cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ',kk.meanlogr - np.log(kk.meanr))
np.testing.assert_allclose(kk.meanlogr, np.log(kk.meanr), atol=1.e-3)
r = kk.meanr
true_xi = np.pi * A**2 * (s/L)**2 * np.exp(-0.25*r**2/s**2)
print('kk.xi = ',kk.xi)
print('true_xi = ',true_xi)
print('ratio = ',kk.xi / true_xi)
print('diff = ',kk.xi - true_xi)
print('max diff = ',max(abs(kk.xi - true_xi)))
print('max rel diff = ',max(abs((kk.xi - true_xi)/true_xi)))
np.testing.assert_allclose(kk.xi, true_xi, rtol=0.1*tol_factor)
# It should also work as a cross-correlation of this cat with itself
kk.process(cat,cat)
np.testing.assert_allclose(kk.meanlogr, np.log(kk.meanr), atol=1.e-3)
np.testing.assert_allclose(kk.xi, true_xi, rtol=0.1*tol_factor)
# Check that we get the same result using the corr2 function
cat.write(os.path.join('data','kk.dat'))
config = treecorr.read_config('configs/kk.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kk.out'), names=True, skip_header=1)
print('kk.xi = ',kk.xi)
print('from corr2 output = ',corr2_output['xi'])
print('ratio = ',corr2_output['xi']/kk.xi)
print('diff = ',corr2_output['xi']-kk.xi)
np.testing.assert_allclose(corr2_output['xi'], kk.xi, rtol=1.e-3)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check the fits write option
out_file_name = os.path.join('output','kk_out.fits')
kk.write(out_file_name)
data = fitsio.read(out_file_name)
np.testing.assert_almost_equal(data['r_nom'], np.exp(kk.logr))
np.testing.assert_almost_equal(data['meanr'], kk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], kk.meanlogr)
np.testing.assert_almost_equal(data['xi'], kk.xi)
np.testing.assert_almost_equal(data['sigma_xi'], np.sqrt(kk.varxi))
np.testing.assert_almost_equal(data['weight'], kk.weight)
np.testing.assert_almost_equal(data['npairs'], kk.npairs)
# Check the read function
kk2 = treecorr.KKCorrelation(bin_size=0.1, min_sep=1., max_sep=100., sep_units='arcmin')
kk2.read(out_file_name)
np.testing.assert_almost_equal(kk2.logr, kk.logr)
np.testing.assert_almost_equal(kk2.meanr, kk.meanr)
np.testing.assert_almost_equal(kk2.meanlogr, kk.meanlogr)
np.testing.assert_almost_equal(kk2.xi, kk.xi)
np.testing.assert_almost_equal(kk2.varxi, kk.varxi)
np.testing.assert_almost_equal(kk2.weight, kk.weight)
np.testing.assert_almost_equal(kk2.npairs, kk.npairs)
assert kk2.coords == kk.coords
assert kk2.metric == kk.metric
assert kk2.sep_units == kk.sep_units
assert kk2.bin_type == kk.bin_type
def test_kg():
# Use gamma_t(r) = gamma0 exp(-r^2/2r0^2) around a bunch of foreground lenses.
# i.e. gamma(r) = -gamma0 exp(-r^2/2r0^2) (x+iy)^2/r^2
# For each lens, we divide this by a random kappa value assigned to that lens, so
# the final kg output shoudl be just gamma_t.
nlens = 1000
nsource = 30000
r0 = 10.
L = 50. * r0
gamma0 = 0.05
rng = np.random.RandomState(8675309)
xl = (rng.random_sample(nlens)-0.5) * L
yl = (rng.random_sample(nlens)-0.5) * L
kl = rng.normal(0.23, 0.05, (nlens,) )
xs = (rng.random_sample(nsource)-0.5) * L
ys = (rng.random_sample(nsource)-0.5) * L
g1 = np.zeros( (nsource,) )
g2 = np.zeros( (nsource,) )
for x,y,k in zip(xl,yl,kl):
dx = xs-x
dy = ys-y
r2 = dx**2 + dy**2
gammat = gamma0 * np.exp(-0.5*r2/r0**2) / k
g1 += -gammat * (dx**2-dy**2)/r2
g2 += -gammat * (2.*dx*dy)/r2
lens_cat = treecorr.Catalog(x=xl, y=yl, k=kl, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=xs, y=ys, g1=g1, g2=g2, x_units='arcmin', y_units='arcmin')
kg = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kg.process(lens_cat, source_cat)
r = kg.meanr
true_gt = gamma0 * np.exp(-0.5*r**2/r0**2)
print('kg.xi = ',kg.xi)
print('kg.xi_im = ',kg.xi_im)
print('true_gammat = ',true_gt)
print('ratio = ',kg.xi / true_gt)
print('diff = ',kg.xi - true_gt)
print('max diff = ',max(abs(kg.xi - true_gt)))
np.testing.assert_allclose(kg.xi, true_gt, rtol=0.1)
np.testing.assert_allclose(kg.xi_im, 0., atol=1.e-2)
# Check that we get the same result using the corr2 function:
lens_cat.write(os.path.join('data','kg_lens.dat'))
source_cat.write(os.path.join('data','kg_source.dat'))
config = treecorr.read_config('configs/kg.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kg.out'), names=True, skip_header=1)
print('kg.xi = ',kg.xi)
print('from corr2 output = ',corr2_output['kgamT'])
print('ratio = ',corr2_output['kgamT']/kg.xi)
print('diff = ',corr2_output['kgamT']-kg.xi)
np.testing.assert_allclose(corr2_output['kgamT'], kg.xi, rtol=1.e-3)
print('xi_im from corr2 output = ',corr2_output['kgamX'])
np.testing.assert_allclose(corr2_output['kgamX'], 0., atol=1.e-2)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check the fits write option
out_file_name1 = os.path.join('output','kg_out1.fits')
kg.write(out_file_name1)
data = fitsio.read(out_file_name1)
np.testing.assert_almost_equal(data['r_nom'], np.exp(kg.logr))
np.testing.assert_almost_equal(data['meanr'], kg.meanr)
np.testing.assert_almost_equal(data['meanlogr'], kg.meanlogr)
np.testing.assert_almost_equal(data['kgamT'], kg.xi)
np.testing.assert_almost_equal(data['kgamX'], kg.xi_im)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(kg.varxi))
np.testing.assert_almost_equal(data['weight'], kg.weight)
np.testing.assert_almost_equal(data['npairs'], kg.npairs)
# Check the read function
kg2 = treecorr.KGCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin')
kg2.read(out_file_name1)
np.testing.assert_almost_equal(kg2.logr, kg.logr)
np.testing.assert_almost_equal(kg2.meanr, kg.meanr)
np.testing.assert_almost_equal(kg2.meanlogr, kg.meanlogr)
np.testing.assert_almost_equal(kg2.xi, kg.xi)
np.testing.assert_almost_equal(kg2.xi_im, kg.xi_im)
np.testing.assert_almost_equal(kg2.varxi, kg.varxi)
np.testing.assert_almost_equal(kg2.weight, kg.weight)
np.testing.assert_almost_equal(kg2.npairs, kg.npairs)
assert kg2.coords == kg.coords
assert kg2.metric == kg.metric
assert kg2.sep_units == kg.sep_units
assert kg2.bin_type == kg.bin_type
def test_nk():
# Use kappa(r) = kappa0 exp(-r^2/2r0^2) (1-r^2/2r0^2) around many lenses.
nlens = 1000
nsource = 100000
kappa0 = 0.05
r0 = 10.
L = 100. * r0
rng = np.random.RandomState(8675309)
xl = (rng.random_sample(nlens)-0.5) * L
yl = (rng.random_sample(nlens)-0.5) * L
xs = (rng.random_sample(nsource)-0.5) * L
ys = (rng.random_sample(nsource)-0.5) * L
k = np.zeros( (nsource,) )
for x,y in zip(xl,yl):
dx = xs-x
dy = ys-y
r2 = dx**2 + dy**2
k += kappa0 * np.exp(-0.5*r2/r0**2) * (1.-0.5*r2/r0**2)
lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=xs, y=ys, k=k, x_units='arcmin', y_units='arcmin')
nk = treecorr.NKCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
nk.process(lens_cat, source_cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ',nk.meanlogr - np.log(nk.meanr))
np.testing.assert_allclose(nk.meanlogr, np.log(nk.meanr), atol=1.e-3)
r = nk.meanr
true_k = kappa0 * np.exp(-0.5*r**2/r0**2) * (1.-0.5*r**2/r0**2)
print('nk.xi = ',nk.xi)
print('true_kappa = ',true_k)
print('ratio = ',nk.xi / true_k)
print('diff = ',nk.xi - true_k)
print('max diff = ',max(abs(nk.xi - true_k)))
np.testing.assert_allclose(nk.xi, true_k, rtol=0.1, atol=2.e-3)
nrand = nlens * 13
xr = (rng.random_sample(nrand)-0.5) * L
yr = (rng.random_sample(nrand)-0.5) * L
rand_cat = treecorr.Catalog(x=xr, y=yr, x_units='arcmin', y_units='arcmin')
rk = treecorr.NKCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
rk.process(rand_cat, source_cat)
print('rk.xi = ',rk.xi)
xi, varxi = nk.calculateXi(rk)
print('compensated xi = ',xi)
print('true_kappa = ',true_k)
print('ratio = ',xi / true_k)
print('diff = ',xi - true_k)
print('max diff = ',max(abs(xi - true_k)))
# It turns out this doesn't come out much better. I think the imprecision is mostly just due
# to the smallish number of lenses, not to edge effects
np.testing.assert_allclose(nk.xi, true_k, rtol=0.05, atol=1.e-3)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data','nk_lens.fits'))
source_cat.write(os.path.join('data','nk_source.fits'))
rand_cat.write(os.path.join('data','nk_rand.fits'))
config = treecorr.read_config('configs/nk.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','nk.out'), names=True, skip_header=1)
print('nk.xi = ',nk.xi)
print('xi = ',xi)
print('from corr2 output = ',corr2_output['kappa'])
print('ratio = ',corr2_output['kappa']/xi)
print('diff = ',corr2_output['kappa']-xi)
np.testing.assert_allclose(corr2_output['kappa'], xi, rtol=1.e-3)
# In the corr2 context, you can turn off the compensated bit, even if there are randoms
# (e.g. maybe you only want randoms for some nn calculation, but not nk.)
config['nk_statistic'] = 'simple'
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','nk.out'), names=True, skip_header=1)
xi_simple, _ = nk.calculateXi()
np.testing.assert_allclose(corr2_output['kappa'], xi_simple, rtol=1.e-3)
# Check the fits write option
out_file_name1 = os.path.join('output','nk_out1.fits')
nk.write(out_file_name1)
data = fitsio.read(out_file_name1)
np.testing.assert_almost_equal(data['r_nom'], np.exp(nk.logr))
np.testing.assert_almost_equal(data['meanr'], nk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], nk.meanlogr)
np.testing.assert_almost_equal(data['kappa'], nk.xi)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(nk.varxi))
np.testing.assert_almost_equal(data['weight'], nk.weight)
np.testing.assert_almost_equal(data['npairs'], nk.npairs)
out_file_name2 = os.path.join('output','nk_out2.fits')
nk.write(out_file_name2, rk)
data = fitsio.read(out_file_name2)
np.testing.assert_almost_equal(data['r_nom'], np.exp(nk.logr))
np.testing.assert_almost_equal(data['meanr'], nk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], nk.meanlogr)
np.testing.assert_almost_equal(data['kappa'], xi)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(varxi))
np.testing.assert_almost_equal(data['weight'], nk.weight)
np.testing.assert_almost_equal(data['npairs'], nk.npairs)
# Check the read function
nk2 = treecorr.NKCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin')
nk2.read(out_file_name1)
np.testing.assert_almost_equal(nk2.logr, nk.logr)
np.testing.assert_almost_equal(nk2.meanr, nk.meanr)
np.testing.assert_almost_equal(nk2.meanlogr, nk.meanlogr)
np.testing.assert_almost_equal(nk2.xi, nk.xi)
np.testing.assert_almost_equal(nk2.varxi, nk.varxi)
np.testing.assert_almost_equal(nk2.weight, nk.weight)
np.testing.assert_almost_equal(nk2.npairs, nk.npairs)
assert nk2.coords == nk.coords
assert nk2.metric == nk.metric
assert nk2.sep_units == nk.sep_units
assert nk2.bin_type == nk.bin_type
def test_kk():
# cf. http://adsabs.harvard.edu/abs/2002A%26A...389..729S for the basic formulae I use here.
#
# Use kappa(r) = A exp(-r^2/2s^2)
#
# The Fourier transform is: kappa~(k) = 2 pi A s^2 exp(-s^2 k^2/2) / L^2
# P(k) = (1/2pi) <|kappa~(k)|^2> = 2 pi A^2 (s/L)^4 exp(-s^2 k^2)
# xi(r) = (1/2pi) int( dk k P(k) J0(kr) )
# = pi A^2 (s/L)^2 exp(-r^2/2s^2/4)
# Note: I'm not sure I handled the L factors correctly, but the units at the end need
# to be kappa^2, so it needs to be (s/L)^2.
s = 10.
if __name__ == '__main__':
ngal = 1000000
L = 30. * s # Not infinity, so this introduces some error. Our integrals were to infinity.
tol_factor = 1
else:
ngal = 100000
L = 30. * s
tol_factor = 2
A = 0.05
rng = np.random.RandomState(8675309)
x = (rng.random_sample(ngal)-0.5) * L
y = (rng.random_sample(ngal)-0.5) * L
r2 = (x**2 + y**2)/s**2
kappa = A * np.exp(-r2/2.)
cat = treecorr.Catalog(x=x, y=y, k=kappa, x_units='arcmin', y_units='arcmin')
kk = treecorr.KKCorrelation(bin_size=0.1, min_sep=1., max_sep=20., sep_units='arcmin',
verbose=1)
kk.process(cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ',kk.meanlogr - np.log(kk.meanr))
np.testing.assert_allclose(kk.meanlogr, np.log(kk.meanr), atol=1.e-3)
r = kk.meanr
true_xi = np.pi * A**2 * (s/L)**2 * np.exp(-0.25*r**2/s**2)
print('kk.xi = ',kk.xi)
print('true_xi = ',true_xi)
print('ratio = ',kk.xi / true_xi)
print('diff = ',kk.xi - true_xi)
print('max diff = ',max(abs(kk.xi - true_xi)))
print('max rel diff = ',max(abs((kk.xi - true_xi)/true_xi)))
np.testing.assert_allclose(kk.xi, true_xi, rtol=0.1*tol_factor)
# It should also work as a cross-correlation of this cat with itself
kk.process(cat,cat)
np.testing.assert_allclose(kk.meanlogr, np.log(kk.meanr), atol=1.e-3)
np.testing.assert_allclose(kk.xi, true_xi, rtol=0.1*tol_factor)
# Check that we get the same result using the corr2 function
cat.write(os.path.join('data','kk.dat'))
config = treecorr.read_config('configs/kk.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output','kk.out'), names=True, skip_header=1)
print('kk.xi = ',kk.xi)
print('from corr2 output = ',corr2_output['xi'])
print('ratio = ',corr2_output['xi']/kk.xi)
print('diff = ',corr2_output['xi']-kk.xi)
np.testing.assert_allclose(corr2_output['xi'], kk.xi, rtol=1.e-3)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check the fits write option
out_file_name = os.path.join('output','kk_out.fits')
kk.write(out_file_name)
data = fitsio.read(out_file_name)
np.testing.assert_almost_equal(data['r_nom'], np.exp(kk.logr))
np.testing.assert_almost_equal(data['meanr'], kk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], kk.meanlogr)
np.testing.assert_almost_equal(data['xi'], kk.xi)
np.testing.assert_almost_equal(data['sigma_xi'], np.sqrt(kk.varxi))
np.testing.assert_almost_equal(data['weight'], kk.weight)
np.testing.assert_almost_equal(data['npairs'], kk.npairs)
# Check the read function
kk2 = treecorr.KKCorrelation(bin_size=0.1, min_sep=1., max_sep=100., sep_units='arcmin')
kk2.read(out_file_name)
np.testing.assert_almost_equal(kk2.logr, kk.logr)
np.testing.assert_almost_equal(kk2.meanr, kk.meanr)
np.testing.assert_almost_equal(kk2.meanlogr, kk.meanlogr)
np.testing.assert_almost_equal(kk2.xi, kk.xi)
np.testing.assert_almost_equal(kk2.varxi, kk.varxi)
np.testing.assert_almost_equal(kk2.weight, kk.weight)
np.testing.assert_almost_equal(kk2.npairs, kk.npairs)
assert kk2.coords == kk.coords
assert kk2.metric == kk.metric
assert kk2.sep_units == kk.sep_units
assert kk2.bin_type == kk.bin_type
def test_aardvark():
# Eric Suchyta did a brute force calculation of the Aardvark catalog, so it is useful to
# compare the output from my code with that.
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data', 'Aardvark.fit')
config = treecorr.read_config('Aardvark.params')
cat1 = treecorr.Catalog(file_name, config)
gg = treecorr.GGCorrelation(config)
gg.process(cat1)
direct_file_name = os.path.join('data', 'Aardvark.direct')
direct_data = numpy.genfromtxt(direct_file_name)
direct_xip = direct_data[:, 3]
direct_xim = direct_data[:, 4]
#print('gg.xip = ',gg.xip)
#print('direct.xip = ',direct_xip)
xip_err = gg.xip - direct_xip
print('xip_err = ', xip_err)
print('max = ', max(abs(xip_err)))
assert max(abs(xip_err)) < 2.e-7
print('xip_im = ', gg.xip_im)
print('max = ', max(abs(gg.xip_im)))
assert max(abs(gg.xip_im)) < 3.e-7
xim_err = gg.xim - direct_xim
print('xim_err = ', xim_err)
print('max = ', max(abs(xim_err)))
assert max(abs(xim_err)) < 1.e-7
print('xim_im = ', gg.xim_im)
print('max = ', max(abs(gg.xim_im)))
assert max(abs(gg.xim_im)) < 1.e-7
# However, after some back and forth about the calculation, we concluded that Eric hadn't
# done the spherical trig correctly to get the shears relative to the great circle joining
# the two positions. So let's compare with my own brute force calculation (i.e. using
# bin_slop = 0):
# This also has the advantage that the radial bins are done the same way -- uniformly
# spaced in log of the chord distance, rather than the great circle distance.
bs0_file_name = os.path.join('data', 'Aardvark.bs0')
bs0_data = numpy.genfromtxt(bs0_file_name)
bs0_xip = bs0_data[:, 2]
bs0_xim = bs0_data[:, 3]
#print('gg.xip = ',gg.xip)
#print('bs0.xip = ',bs0_xip)
xip_err = gg.xip - bs0_xip
print('xip_err = ', xip_err)
print('max = ', max(abs(xip_err)))
assert max(abs(xip_err)) < 1.e-7
xim_err = gg.xim - bs0_xim
print('xim_err = ', xim_err)
print('max = ', max(abs(xim_err)))
assert max(abs(xim_err)) < 5.e-8
# Check that we get the same result using the corr2 executable:
# Note: This is the only test of the corr2 executable that we do with nosetests.
# The other similar tests are blocked out with: if __name__ == '__main__':
import subprocess
corr2_exe = get_script_name('corr2')
p = subprocess.Popen([corr2_exe, "Aardvark.params"])
p.communicate()
corr2_output = numpy.genfromtxt(os.path.join('output', 'Aardvark.out'),
names=True)
print('gg.xip = ', gg.xip)
print('from corr2 output = ', corr2_output['xip'])
print('ratio = ', corr2_output['xip'] / gg.xip)
print('diff = ', corr2_output['xip'] - gg.xip)
numpy.testing.assert_almost_equal(corr2_output['xip'] / gg.xip,
1.,
decimal=3)
print('gg.xim = ', gg.xim)
print('from corr2 output = ', corr2_output['xim'])
print('ratio = ', corr2_output['xim'] / gg.xim)
print('diff = ', corr2_output['xim'] - gg.xim)
numpy.testing.assert_almost_equal(corr2_output['xim'] / gg.xim,
1.,
decimal=3)
print('xip_im from corr2 output = ', corr2_output['xip_im'])
print('max err = ', max(abs(corr2_output['xip_im'])))
assert max(abs(corr2_output['xip_im'])) < 3.e-7
print('xim_im from corr2 output = ', corr2_output['xim_im'])
print('max err = ', max(abs(corr2_output['xim_im'])))
assert max(abs(corr2_output['xim_im'])) < 1.e-7
# As bin_slop decreases, the agreement should get even better.
# This test is slow, so only do it if running test_gg.py directly.
if __name__ == '__main__':
config['bin_slop'] = 0.2
gg = treecorr.GGCorrelation(config)
gg.process(cat1)
#print('gg.xip = ',gg.xip)
#print('bs0.xip = ',bs0_xip)
xip_err = gg.xip - bs0_xip
print('xip_err = ', xip_err)
print('max = ', max(abs(xip_err)))
assert max(abs(xip_err)) < 1.e-8
xim_err = gg.xim - bs0_xim
print('xim_err = ', xim_err)
print('max = ', max(abs(xim_err)))
assert max(abs(xim_err)) < 1.e-8
def test_nk():
# Use kappa(r) = kappa0 exp(-r^2/2r0^2) (1-r^2/2r0^2) around many lenses.
nlens = 1000
nsource = 100000
kappa0 = 0.05
r0 = 10.
L = 100. * r0
rng = np.random.RandomState(8675309)
xl = (rng.random_sample(nlens) - 0.5) * L
yl = (rng.random_sample(nlens) - 0.5) * L
xs = (rng.random_sample(nsource) - 0.5) * L
ys = (rng.random_sample(nsource) - 0.5) * L
k = np.zeros((nsource, ))
for x, y in zip(xl, yl):
dx = xs - x
dy = ys - y
r2 = dx**2 + dy**2
k += kappa0 * np.exp(-0.5 * r2 / r0**2) * (1. - 0.5 * r2 / r0**2)
lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin')
source_cat = treecorr.Catalog(x=xs,
y=ys,
k=k,
x_units='arcmin',
y_units='arcmin')
nk = treecorr.NKCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=20.,
sep_units='arcmin',
verbose=1)
nk.process(lens_cat, source_cat)
# log(<R>) != <logR>, but it should be close:
print('meanlogr - log(meanr) = ', nk.meanlogr - np.log(nk.meanr))
np.testing.assert_allclose(nk.meanlogr, np.log(nk.meanr), atol=1.e-3)
r = nk.meanr
true_k = kappa0 * np.exp(-0.5 * r**2 / r0**2) * (1. - 0.5 * r**2 / r0**2)
print('nk.xi = ', nk.xi)
print('true_kappa = ', true_k)
print('ratio = ', nk.xi / true_k)
print('diff = ', nk.xi - true_k)
print('max diff = ', max(abs(nk.xi - true_k)))
np.testing.assert_allclose(nk.xi, true_k, rtol=0.1, atol=2.e-3)
nrand = nlens * 13
xr = (rng.random_sample(nrand) - 0.5) * L
yr = (rng.random_sample(nrand) - 0.5) * L
rand_cat = treecorr.Catalog(x=xr, y=yr, x_units='arcmin', y_units='arcmin')
rk = treecorr.NKCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=20.,
sep_units='arcmin',
verbose=1)
rk.process(rand_cat, source_cat)
print('rk.xi = ', rk.xi)
xi, varxi = nk.calculateXi(rk)
print('compensated xi = ', xi)
print('true_kappa = ', true_k)
print('ratio = ', xi / true_k)
print('diff = ', xi - true_k)
print('max diff = ', max(abs(xi - true_k)))
# It turns out this doesn't come out much better. I think the imprecision is mostly just due
# to the smallish number of lenses, not to edge effects
np.testing.assert_allclose(nk.xi, true_k, rtol=0.05, atol=1.e-3)
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
# Check that we get the same result using the corr2 function
lens_cat.write(os.path.join('data', 'nk_lens.fits'))
source_cat.write(os.path.join('data', 'nk_source.fits'))
rand_cat.write(os.path.join('data', 'nk_rand.fits'))
config = treecorr.read_config('configs/nk.yaml')
config['verbose'] = 0
config['precision'] = 8
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output', 'nk.out'),
names=True,
skip_header=1)
print('nk.xi = ', nk.xi)
print('xi = ', xi)
print('from corr2 output = ', corr2_output['kappa'])
print('ratio = ', corr2_output['kappa'] / xi)
print('diff = ', corr2_output['kappa'] - xi)
np.testing.assert_allclose(corr2_output['kappa'], xi, rtol=1.e-3)
# In the corr2 context, you can turn off the compensated bit, even if there are randoms
# (e.g. maybe you only want randoms for some nn calculation, but not nk.)
config['nk_statistic'] = 'simple'
treecorr.corr2(config)
corr2_output = np.genfromtxt(os.path.join('output', 'nk.out'),
names=True,
skip_header=1)
xi_simple, _ = nk.calculateXi()
np.testing.assert_equal(xi_simple, nk.xi)
np.testing.assert_allclose(corr2_output['kappa'], xi_simple, rtol=1.e-3)
# Check the fits write option
out_file_name1 = os.path.join('output', 'nk_out1.fits')
nk.write(out_file_name1)
data = fitsio.read(out_file_name1)
np.testing.assert_almost_equal(data['r_nom'], np.exp(nk.logr))
np.testing.assert_almost_equal(data['meanr'], nk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], nk.meanlogr)
np.testing.assert_almost_equal(data['kappa'], nk.xi)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(nk.varxi))
np.testing.assert_almost_equal(data['weight'], nk.weight)
np.testing.assert_almost_equal(data['npairs'], nk.npairs)
out_file_name2 = os.path.join('output', 'nk_out2.fits')
nk.write(out_file_name2, rk)
data = fitsio.read(out_file_name2)
np.testing.assert_almost_equal(data['r_nom'], np.exp(nk.logr))
np.testing.assert_almost_equal(data['meanr'], nk.meanr)
np.testing.assert_almost_equal(data['meanlogr'], nk.meanlogr)
np.testing.assert_almost_equal(data['kappa'], xi)
np.testing.assert_almost_equal(data['sigma'], np.sqrt(varxi))
np.testing.assert_almost_equal(data['weight'], nk.weight)
np.testing.assert_almost_equal(data['npairs'], nk.npairs)
# Check the read function
nk2 = treecorr.NKCorrelation(bin_size=0.1,
min_sep=1.,
max_sep=20.,
sep_units='arcmin')
nk2.read(out_file_name2)
np.testing.assert_almost_equal(nk2.logr, nk.logr)
np.testing.assert_almost_equal(nk2.meanr, nk.meanr)
np.testing.assert_almost_equal(nk2.meanlogr, nk.meanlogr)
np.testing.assert_almost_equal(nk2.xi, nk.xi)
np.testing.assert_almost_equal(nk2.varxi, nk.varxi)
np.testing.assert_almost_equal(nk2.weight, nk.weight)
np.testing.assert_almost_equal(nk2.npairs, nk.npairs)
assert nk2.coords == nk.coords
assert nk2.metric == nk.metric
assert nk2.sep_units == nk.sep_units
assert nk2.bin_type == nk.bin_type
def test_aardvark():
# Eric Suchyta did a brute force calculation of the Aardvark catalog, so it is useful to
# compare the output from my code with that.
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data','Aardvark.fit')
config = treecorr.read_config('Aardvark.yaml')
cat1 = treecorr.Catalog(file_name, config)
gg = treecorr.GGCorrelation(config)
gg.process(cat1)
direct_file_name = os.path.join('data','Aardvark.direct')
direct_data = numpy.genfromtxt(direct_file_name)
direct_xip = direct_data[:,3]
direct_xim = direct_data[:,4]
#print('gg.xip = ',gg.xip)
#print('direct.xip = ',direct_xip)
xip_err = gg.xip - direct_xip
print('xip_err = ',xip_err)
print('max = ',max(abs(xip_err)))
assert max(abs(xip_err)) < 2.e-7
print('xip_im = ',gg.xip_im)
print('max = ',max(abs(gg.xip_im)))
assert max(abs(gg.xip_im)) < 3.e-7
xim_err = gg.xim - direct_xim
print('xim_err = ',xim_err)
print('max = ',max(abs(xim_err)))
assert max(abs(xim_err)) < 1.e-7
print('xim_im = ',gg.xim_im)
print('max = ',max(abs(gg.xim_im)))
assert max(abs(gg.xim_im)) < 1.e-7
# However, after some back and forth about the calculation, we concluded that Eric hadn't
# done the spherical trig correctly to get the shears relative to the great circle joining
# the two positions. So let's compare with my own brute force calculation (i.e. using
# bin_slop = 0):
# This also has the advantage that the radial bins are done the same way -- uniformly
# spaced in log of the chord distance, rather than the great circle distance.
bs0_file_name = os.path.join('data','Aardvark.bs0')
bs0_data = numpy.genfromtxt(bs0_file_name)
bs0_xip = bs0_data[:,2]
bs0_xim = bs0_data[:,3]
#print('gg.xip = ',gg.xip)
#print('bs0.xip = ',bs0_xip)
xip_err = gg.xip - bs0_xip
print('xip_err = ',xip_err)
print('max = ',max(abs(xip_err)))
assert max(abs(xip_err)) < 1.e-7
xim_err = gg.xim - bs0_xim
print('xim_err = ',xim_err)
print('max = ',max(abs(xim_err)))
assert max(abs(xim_err)) < 5.e-8
# Check that we get the same result using the corr2 executable:
# Note: This is the only test of the corr2 executable that we do with nosetests.
# The other similar tests are blocked out with: if __name__ == '__main__':
import subprocess
corr2_exe = get_script_name('corr2')
p = subprocess.Popen( [corr2_exe,"Aardvark.yaml"] )
p.communicate()
corr2_output = numpy.genfromtxt(os.path.join('output','Aardvark.out'), names=True)
print('gg.xip = ',gg.xip)
print('from corr2 output = ',corr2_output['xip'])
print('ratio = ',corr2_output['xip']/gg.xip)
print('diff = ',corr2_output['xip']-gg.xip)
numpy.testing.assert_almost_equal(corr2_output['xip']/gg.xip, 1., decimal=3)
print('gg.xim = ',gg.xim)
print('from corr2 output = ',corr2_output['xim'])
print('ratio = ',corr2_output['xim']/gg.xim)
print('diff = ',corr2_output['xim']-gg.xim)
numpy.testing.assert_almost_equal(corr2_output['xim']/gg.xim, 1., decimal=3)
print('xip_im from corr2 output = ',corr2_output['xip_im'])
print('max err = ',max(abs(corr2_output['xip_im'])))
assert max(abs(corr2_output['xip_im'])) < 3.e-7
print('xim_im from corr2 output = ',corr2_output['xim_im'])
print('max err = ',max(abs(corr2_output['xim_im'])))
assert max(abs(corr2_output['xim_im'])) < 1.e-7
# As bin_slop decreases, the agreement should get even better.
# This test is slow, so only do it if running test_gg.py directly.
if __name__ == '__main__':
config['bin_slop'] = 0.2
gg = treecorr.GGCorrelation(config)
gg.process(cat1)
#print('gg.xip = ',gg.xip)
#print('bs0.xip = ',bs0_xip)
xip_err = gg.xip - bs0_xip
print('xip_err = ',xip_err)
print('max = ',max(abs(xip_err)))
assert max(abs(xip_err)) < 1.e-8
xim_err = gg.xim - bs0_xim
print('xim_err = ',xim_err)
print('max = ',max(abs(xim_err)))
assert max(abs(xim_err)) < 1.e-8
out_keyword = keyword1
elif cycle == 1:
# Do clipped X clipped
keyword1 = 'SS%s.rCLIP_%ssigma'%(SS,sigma)
keyword2 = keyword1
out_keyword = keyword1
elif cycle == 2:
# Do unclipped X clipped
keyword1 = 'ORIG'
keyword2 = 'SS%s.rCLIP_%ssigma'%(SS,sigma)
out_keyword = keyword1 + '_X_' + keyword2
input_file1 = '%s/Correlation_Function/%s/%s.%s.ThetaX_ThetaY_e1_e2_w.Std.asc' %(overall_DIR, DIRname1, combined_name1, keyword1)
input_file2 = '%s/Correlation_Function/%s/%s.%s.ThetaX_ThetaY_e1_e2_w.Std.asc' %(overall_DIR, DIRname2, combined_name2, keyword2)
output_file = '%s/Tree_Correlation_Function/%s/ThBins%s/%s.%s.CorrFun.asc' %(overall_DIR, DIRname, ThBins, combined_name1, out_keyword)
Assemble_TreeCorr_ConfigFile(input_file1, input_file2, output_file, metric, flip_g2, bin_slop, min_sep, max_sep, ThBins, cn)
config_file='%s/Tree_Correlation_Function/config_files/config_treecorr%s.yaml' %(overall_DIR,cn)
config = treecorr.read_config(config_file)
t1 = time.time()
treecorr.corr2(config)
t2=time.time()
print( "TreeCorr time for %s is %.1f s" %(output_file, (t2-t1)) )
def test_check():
"""Test checking the validity of config values.
"""
# First a simple case with no conflicts
config1 = treecorr.read_config('configs/kg.yaml')
valid_params = treecorr.corr2_valid_params
config2 = treecorr.config.check_config(config1.copy(), valid_params)
# Just check a few values
assert config2['x_col'] == '1'
assert config2['k_col'] == ['3', '0']
assert config2['verbose'] == 1
assert config2['kg_file_name'] == 'output/kg.out'
# Will also have other parameters filled from the valid_params dict
for key in config2:
assert key in valid_params
if key in config1:
if isinstance(config1[key], list):
assert [str(v) for v in config2[key]
] == [str(v) for v in config1[key]]
else:
assert config2[key] == config1[key] or str(
config2[key]) == str(config1[key])
else:
assert config2[key] == valid_params[key][2]
# Check list of bool
config1['flip_g1'] = [True, 0]
config2 = treecorr.config.check_config(config1.copy(), valid_params)
assert config2['flip_g1'] == [True, False]
# Longer names are allowed
config1['x_units'] = 'arcminutes'
config1['y_units'] = 'arcminute'
config2 = treecorr.config.check_config(config1.copy(), valid_params)
assert config2['x_units'] == 'arcmin'
assert config2['y_units'] == 'arcmin'
# Also other aliases, but you need to list them explicitly.
config1['reverse_g1'] = True
with assert_raises(TypeError):
treecorr.config.check_config(config1.copy(), valid_params)
config2 = treecorr.config.check_config(config1.copy(),
valid_params,
aliases={'reverse_g1': 'flip_g1'})
assert config2['flip_g1'] == True
assert 'reverse_g1' not in config2
del config1['reverse_g1']
# Invalid values raise errors
config1['verbose'] = -1
with assert_raises(ValueError):
treecorr.config.check_config(config1.copy(), valid_params)
config1['verbose'] = 1
config1['metric'] = 'hyperbolic'
with assert_raises(ValueError):
treecorr.config.check_config(config1.copy(), valid_params)
del config1['metric']
# With a logger, aliases emit a warning.
config1['n2_file_name'] = 'output/n2.out'
with CaptureLog() as cl:
config2 = treecorr.config.check_config(
config1.copy(),
valid_params,
logger=cl.logger,
aliases={'n2_file_name': 'nn_file_name'})
assert "The parameter n2_file_name is deprecated." in cl.output
assert "You should use nn_file_name instead." in cl.output
# corr2 has a list of standard aliases
# It is currently empty, but let's mock it up to test the functionality.
if sys.version_info < (3, ): return # mock only available on python 3
from unittest import mock
with mock.patch('treecorr.corr2_aliases',
{'n2_file_name': 'nn_file_name'}):
config2 = treecorr.config.check_config(config1.copy(),
valid_params,
aliases=treecorr.corr2_aliases)
assert 'n2_file_name' not in config2
assert config2['nn_file_name'] == 'output/n2.out'
del config1['n2_file_name']
def test_check():
"""Test checking the validity of config values.
"""
# First a simple case with no conflicts
config1 = treecorr.read_config('configs/kg.yaml')
valid_params = treecorr.corr2_valid_params
config2 = treecorr.config.check_config(config1.copy(), valid_params)
# Just check a few values
assert config2['x_col'] == '1'
assert config2['k_col'] == ['3', '0']
assert config2['verbose'] == 1
assert config2['kg_file_name'] == 'output/kg.out'
# Will also have other parameters filled from the valid_params dict
for key in config2:
assert key in valid_params
if key in config1:
if isinstance(config1[key], list):
assert [str(v) for v in config2[key]] == [str(v) for v in config1[key]]
else:
assert config2[key] == config1[key] or str(config2[key]) == str(config1[key])
else:
assert config2[key] == valid_params[key][2]
# Check list of bool
config1['flip_g1'] = [True, 0]
config2 = treecorr.config.check_config(config1.copy(), valid_params)
assert config2['flip_g1'] == [True, False]
# Longer names are allowed
config1['x_units'] = 'arcminutes'
config1['y_units'] = 'arcminute'
config2 = treecorr.config.check_config(config1.copy(), valid_params)
assert config2['x_units'] == 'arcmin'
assert config2['y_units'] == 'arcmin'
# Also other aliases, but you need to list them explicitly.
config1['reverse_g1'] = True
with assert_raises(TypeError):
treecorr.config.check_config(config1.copy(), valid_params)
config2 = treecorr.config.check_config(config1.copy(), valid_params,
aliases={'reverse_g1' : 'flip_g1'})
assert config2['flip_g1'] == True
assert 'reverse_g1' not in config2
del config1['reverse_g1']
# Invalid values raise errors
config1['verbose'] = -1
with assert_raises(ValueError):
treecorr.config.check_config(config1.copy(), valid_params)
config1['verbose'] = 1
config1['metric'] = 'hyperbolic'
with assert_raises(ValueError):
treecorr.config.check_config(config1.copy(), valid_params)
del config1['metric']
# With a logger, aliases emit a warning.
config1['n2_file_name'] = 'output/n2.out'
with CaptureLog() as cl:
config2 = treecorr.config.check_config(config1.copy(), valid_params, logger=cl.logger,
aliases={'n2_file_name' : 'nn_file_name'})
assert "The parameter n2_file_name is deprecated." in cl.output
assert "You should use nn_file_name instead." in cl.output
# corr2 has a list of standard aliases
# It is currently empty, but let's mock it up to test the functionality.
if sys.version_info < (3,): return # mock only available on python 3
from unittest import mock
with mock.patch('treecorr.corr2_aliases', {'n2_file_name' : 'nn_file_name'}):
config2 = treecorr.config.check_config(config1.copy(), valid_params,
aliases=treecorr.corr2_aliases)
assert 'n2_file_name' not in config2
assert config2['nn_file_name'] == 'output/n2.out'
del config1['n2_file_name']
import numpy as np
import healpy as hp
import matplotlib.pyplot as plt
import pylab
import os
import sys
import pyfits
import glob
import treecorr
## Full im3shape_v7_r
config = treecorr.read_config('/Users/drgk/DES/SV_tests/split_xi/sample.params_shearhsear_im3shape')
config['file_name'] = '/Users/drgk/DES/SV_tests/athena_cats/im3shape_v7_r_shears_03_z_13.dat'
config['gg_file_name'] = '/Users/drgk/DES/SV_tests/split_xi/gg_galshear_im3shape_v7_r.out'
treecorr.corr2(config)
config = treecorr.read_config('/Users/drgk/DES/SV_tests/split_xi/sample.params_kappakappa_im3shape')
config['file_name'] = '/Users/drgk/DES/SV_tests/athena_cats/im3shape_v7_r_m_03_z_13.dat'
config['kk_file_name'] = '/Users/drgk/DES/SV_tests/split_xi/kk_galshear_im3shape_v7_r.out'
treecorr.corr2(config)
####### AIRMASS ##############
## im3shape_v7_r Airmass Upper
config = treecorr.read_config('/Users/drgk/DES/SV_tests/split_xi/sample.params_shearhsear_im3shape')
config['file_name'] = '/Users/drgk/DES/SV_tests/athena_cats/im3shape_v7_r_shears_airmass_r_upper_nzweighted.dat'
config['gg_file_name'] = '/Users/drgk/DES/SV_tests/split_xi/gg_galshear_im3shape_v7_r_airmass_r_upper_nzweighted.out'
treecorr.corr2(config)
config = treecorr.read_config('/Users/drgk/DES/SV_tests/split_xi/sample.params_kappakappa_im3shape')
config['file_name'] = '/Users/drgk/DES/SV_tests/athena_cats/im3shape_v7_r_m_airmass_r_upper_nzweighted.dat'
def run_treecorr(config_file, tomo_file, cat_file, output_file):
config = treecorr.read_config(config_file)
tomo_data = fitsio.read(tomo_file)
shear_data = fitsio.read(cat_file)
tomo_header = fitsio.read_header(tomo_file, 1)
nbin = tomo_header.get("NBIN")
gg = treecorr.GGCorrelation(config)
theta = []
xip = []
xim = []
bin1 = []
bin2 = []
angbin = []
print("Not doing the proper ngmix weighting in this test!")
for b1 in range(nbin):
w1 = np.where(tomo_data['BIN'] == b1 + 1)
d1 = shear_data[w1]
cat1 = treecorr.Catalog(g1=d1['E_1'],
g2=d1['E_1'],
w=d1['W'],
ra=d1['RA'],
dec=d1['DEC'],
ra_units='deg',
dec_units='deg')
for b2 in range(nbin):
if b2 < b1:
continue
w2 = np.where(tomo_data['BIN'] == b2 + 1)
d2 = shear_data[w2]
cat2 = treecorr.Catalog(g1=d2['E_1'],
g2=d2['E_1'],
w=d2['W'],
ra=d2['RA'],
dec=d2['DEC'],
ra_units='deg',
dec_units='deg')
gg.process(cat1, cat2)
ntheta = len(gg.meanr)
theta.append(gg.meanr)
xip.append(gg.xip)
xim.append(gg.xim)
bin1.append(np.repeat(b1, ntheta))
bin2.append(np.repeat(b2, ntheta))
angbin.append(np.arange(ntheta, dtype=int))
theta = np.concatenate(theta)
xip = np.concatenate(xip)
xim = np.concatenate(xim)
bin1 = np.concatenate(bin1)
bin2 = np.concatenate(bin2)
angbin = np.concatenate(angbin)
tp = twopoint.Types.galaxy_shear_plus_real
tm = twopoint.Types.galaxy_shear_minus_real
XIP = twopoint.SpectrumMeasurement("xip", (bin1, bin2), (tp, tp),
("source", "source"),
"SAMPLE",
angbin,
xip,
angle=theta,
angle_unit="arcmin")
XIM = twopoint.SpectrumMeasurement("xim", (bin1, bin2), (tm, tm),
("source", "source"),
"SAMPLE",
angbin,
xim,
angle=theta,
angle_unit="arcmin")
output = twopoint.TwoPointFile([XIP, XIM], [], [], None)
output.to_fits(output_file, clobber=True)
def test_fits():
try:
import fitsio
except ImportError:
print('Skipping FITS tests, since fitsio is not installed')
return
get_from_wiki('Aardvark.fit')
file_name = os.path.join('data','Aardvark.fit')
config = treecorr.read_config('Aardvark.yaml')
config['verbose'] = 1
config['kk_file_name'] = 'kk.fits'
config['gg_file_name'] = 'gg.fits'
# Just test a few random particular values
cat1 = treecorr.Catalog(file_name, config)
np.testing.assert_equal(len(cat1.ra), 390935)
np.testing.assert_equal(cat1.nobj, 390935)
np.testing.assert_almost_equal(cat1.ra[0], 56.4195 * (pi/180.))
np.testing.assert_almost_equal(cat1.ra[390934], 78.4782 * (pi/180.))
np.testing.assert_almost_equal(cat1.dec[290333], 83.1579 * (pi/180.))
np.testing.assert_almost_equal(cat1.g1[46392], 0.0005066675)
np.testing.assert_almost_equal(cat1.g2[46392], -0.0001006742)
np.testing.assert_almost_equal(cat1.k[46392], -0.0008628797)
assert_raises(ValueError, treecorr.Catalog, file_name, config, ra_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, dec_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, r_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, w_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, wpos_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, flag_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g1_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g2_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, k_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, ra_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, dec_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, x_col='x')
assert_raises(ValueError, treecorr.Catalog, file_name, config, y_col='y')
assert_raises(ValueError, treecorr.Catalog, file_name, config, z_col='z')
assert_raises(ValueError, treecorr.Catalog, file_name, config, ra_col='0', dec_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g1_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g2_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, k_col='0')
assert_raises(TypeError, treecorr.Catalog, file_name, config, x_units='arcmin')
assert_raises(TypeError, treecorr.Catalog, file_name, config, y_units='arcmin')
del config['ra_units']
assert_raises(TypeError, treecorr.Catalog, file_name, config)
del config['dec_units']
assert_raises(TypeError, treecorr.Catalog, file_name, config, ra_units='deg')
# The catalog doesn't have x, y, or w, but test that functionality as well.
del config['ra_col']
del config['dec_col']
config['x_col'] = 'RA'
config['y_col'] = 'DEC'
config['w_col'] = 'MU'
config['flag_col'] = 'INDEX'
config['ignore_flag'] = 64
cat2 = treecorr.Catalog(file_name, config)
np.testing.assert_almost_equal(cat2.x[390934], 78.4782, decimal=4)
np.testing.assert_almost_equal(cat2.y[290333], 83.1579, decimal=4)
np.testing.assert_almost_equal(cat2.w[46392], 0.) # index = 1200379
np.testing.assert_almost_equal(cat2.w[46393], 0.9995946) # index = 1200386
assert_raises(ValueError, treecorr.Catalog, file_name, config, x_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, y_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, z_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, ra_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, dec_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, r_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, w_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, wpos_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, flag_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g1_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g2_col='invalid')
assert_raises(ValueError, treecorr.Catalog, file_name, config, k_col='invalid')
# Test using a limited set of rows
config['first_row'] = 101
config['last_row'] = 50000
cat3 = treecorr.Catalog(file_name, config)
np.testing.assert_equal(len(cat3.x), 49900)
np.testing.assert_equal(cat3.ntot, 49900)
np.testing.assert_equal(cat3.nobj, sum(cat3.w != 0))
np.testing.assert_equal(cat3.sumw, sum(cat3.w))
np.testing.assert_equal(cat3.sumw, sum(cat2.w[100:50000]))
np.testing.assert_almost_equal(cat3.g1[46292], 0.0005066675)
np.testing.assert_almost_equal(cat3.g2[46292], -0.0001006742)
np.testing.assert_almost_equal(cat3.k[46292], -0.0008628797)
cat4 = treecorr.read_catalogs(config, key='file_name', is_rand=True)[0]
np.testing.assert_equal(len(cat4.x), 49900)
np.testing.assert_equal(cat4.ntot, 49900)
np.testing.assert_equal(cat4.nobj, sum(cat4.w != 0))
np.testing.assert_equal(cat4.sumw, sum(cat4.w))
np.testing.assert_equal(cat4.sumw, sum(cat2.w[100:50000]))
assert cat4.g1 is None
assert cat4.g2 is None
assert cat4.k is None
do_pickle(cat1)
do_pickle(cat2)
do_pickle(cat3)
do_pickle(cat4)
assert_raises(ValueError, treecorr.Catalog, file_name, config, first_row=-10)
assert_raises(ValueError, treecorr.Catalog, file_name, config, first_row=0)
assert_raises(ValueError, treecorr.Catalog, file_name, config, first_row=60000)
assert_raises(ValueError, treecorr.Catalog, file_name, config, first_row=50001)
assert_raises(TypeError, treecorr.read_catalogs, config)
assert_raises(TypeError, treecorr.read_catalogs, config, key='file_name', list_key='file_name')
# If gg output not given, it is still invalid to only have one or the other of g1,g2.
del config['gg_file_name']
assert_raises(ValueError, treecorr.Catalog, file_name, config, g1_col='0')
assert_raises(ValueError, treecorr.Catalog, file_name, config, g2_col='0')
def compute_w(dataf, randf, config, estimator='PW1', compensated=1, nbins_rpar=30, random_oversampling=10., verbosity=1, largePi=0, **kwargs): """ dataf: paths to galaxy samples randf: paths to random points corresponding to galaxy samples config: path to config file, or dict specifying file types; column names/numbers, etc. a la TreeCorr configuration estimator: 'PW1', 'PW2', 'AS' or 'wgg' to specify correlation & estimator nbins_rpar: number of line-of-sight bins for 3D correlation function -- specify limits in config arg """ assert estimator in ['PW1', 'PW2', 'AS', 'wgg'], "for IA: estimator must be 'PW1/2' (pair_weighted 1=RDs, 2=RRs norm) or 'AS' (average shear), for clustering: 'wgg'" assert hasattr(dataf, '__iter__'), "dataf must be list/tuple of 2x paths; density, shapes for IA, or density1, density2 for clustering (these can be the same!)" assert hasattr(randf, '__iter__'), "randf must be list/tuple of 2x paths for randoms corresponding to each of dataf (these can be the same!)" nbins_rpar = int(nbins_rpar) random_oversampling = float(random_oversampling) verbosity = int(verbosity) largePi = int(largePi) if type(config) == str: config = treecorr.read_config(config) if config['file_type'] == 'ASCII': config['ra_col'] = int(config['ra_col']) config['dec_col'] = int(config['dec_col']) config['r_col'] = int(config['r_col']) config['g1_col'] = int(config['g1_col']) config['g2_col'] = int(config['g2_col']) config['verbose'] = verbosity if not largePi: Pi = np.linspace(config['min_rpar'], config['max_rpar'], nbins_rpar + 1) else: dPi = (config['max_rpar'] - config['min_rpar']) / nbins_rpar Pi = np.arange(config['min_rpar']*1.5, config['max_rpar']*1.5 + dPi, step=dPi, dtype=float) if estimator in ['PW1', 'PW2', 'AS']: corr = 'ng' gt_3D = np.zeros([len(Pi)-1, config['nbins']]) gx_3D = np.zeros([len(Pi)-1, config['nbins']]) varg_3D = np.zeros([len(Pi)-1, config['nbins']]) npair_3D = np.zeros([len(Pi)-1, config['nbins']]) elif estimator == 'wgg': corr = 'nn' wgg_3D = np.zeros([len(Pi)-1, config['nbins']]) else: raise ValueError, "unsupported estimator choice" config_r = config.copy() config_r['flip_g1'] = False config_r['flip_g2'] = False data1 = treecorr.Catalog(dataf[0], config) # 1 = density/lenses data2 = treecorr.Catalog(dataf[1], config) # 2 = shapes rand1 = treecorr.Catalog(randf[0], config_r, is_rand=1) rand2 = treecorr.Catalog(randf[1], config_r, is_rand=1) f1 = data1.ntot * random_oversampling / float(rand1.ntot) f2 = data2.ntot * random_oversampling / float(rand2.ntot) rand1.w = np.array(np.random.rand(rand1.ntot) < f1, dtype=float) rand2.w = np.array(np.random.rand(rand2.ntot) < f2, dtype=float) varg = treecorr.calculateVarG(data2) for p in tqdm(range(len(Pi)-1), ascii=True, desc='Correlating'): if largePi & any(abs(Pi[p:p+2]) < config['max_rpar']): continue conf_pi = config.copy() conf_pi['min_rpar'] = Pi[p] conf_pi['max_rpar'] = Pi[p+1] if corr == 'ng': ng = treecorr.NGCorrelation(conf_pi) rg = treecorr.NGCorrelation(conf_pi) ng.process_cross(data1, data2) rg.process_cross(rand1, data2) if estimator == 'PW1': # RDs norm f = data1.ntot / rand1.w.sum() norm1 = rg.weight * f norm2 = rg.weight if estimator == 'PW2': # RRs norm RRs = get_RRs(rand1, rand2, conf_pi, **kwargs) f1 = data1.ntot / rand1.w.sum() f2 = data2.ntot / rand2.w.sum() norm1 = RRs * f1 * f2 norm2 = RRs * f2 elif estimator == 'AS': # DDs norm norm1 = ng.weight norm2 = rg.weight if int(compensated): gt_3D[p] += (ng.xi / norm1) - (rg.xi / norm2) gx_3D[p] += (ng.xi_im / norm1) - (rg.xi_im / norm2) varg_3D[p] += (varg / norm1) + (varg / norm2) npair_3D[p] += ng.npairs else: gt_3D[p] += ng.xi / norm1 gx_3D[p] += ng.xi_im / norm1 varg_3D[p] += varg / norm1 npair_3D[p] += ng.npairs elif corr == 'nn': nn = treecorr.NNCorrelation(conf_pi) rr = treecorr.NNCorrelation(conf_pi) nr = treecorr.NNCorrelation(conf_pi) rn = treecorr.NNCorrelation(conf_pi) if dataf[0] == dataf[1]: nn.process(data1) rr.process(rand1) nr.process(data1, rand1) xi, varxi = nn.calculateXi(rr, nr) else: nn.process(data1, data2) rr.process(rand1, rand2) nr.process(data1, rand2) rn.process(rand1, data2) xi, varxi = nn.calculateXi(rr, nr, rn) wgg_3D[p] += xi if corr == 'ng': #gt = np.trapz(gt_3D, x=midpoints(Pi), axis=0) #gx = np.trapz(gx_3D, x=midpoints(Pi), axis=0) #gt = simps(gt_3D, x=midpoints(Pi), axis=0) gt = np.sum(gt_3D * (Pi[1] - Pi[0]), axis=0) gx = np.sum(gx_3D * (Pi[1] - Pi[0]), axis=0) varg = np.sum(varg_3D, axis=0) npair = np.sum(npair_3D, axis=0) r = ng.rnom return r, gt, gx, varg**0.5, npair elif corr == 'nn': wgg = np.sum(wgg_3D * (Pi[1] - Pi[0]), axis=0) r = nn.rnom return r, wgg
