def shear_shear_corr(pos1,pos2,shear1,shear2,k1=None,k2=None,w1=None,w2=None,same_zshell=False,same_cell=False,unique_encounter=False,num_threads=0): nbins = 6 min_sep = 0.05 # 3 arcmin max_sep = 3.0 # 180 arcmin bin_size = (max_sep-min_sep)/nbins # roughly bin_slop = 0.05/bin_size # 0.1 -> 0.05 # 2pt_pipeline for des used bin_slop: 0.01 here: https://github.com/des-science/2pt_pipeline/blob/master/pipeline/twopt_pipeline.yaml # num_threads = 5 #None #0 logger = None if same_zshell and same_cell: # auto ra, dec = pos1 # either 1 or 2 works g1, g2 = shear1 k = k1 w = w1 cat = treecorr.Catalog(g1=g1, g2=g2, k=k, ra=ra, dec=dec, w=w, ra_units='degrees', dec_units='degrees') elif same_zshell: # just wanted to distrubute the workload fairly for two encounters (didn't want to make one of the cores idle) ra1, dec1 = np.array_split(pos1[0], 2), np.array_split(pos1[1], 2) # split in half ra2, dec2 = np.array_split(pos2[0], 2), np.array_split(pos2[1], 2) g1_1st, g2_1st = np.array_split(shear1[0], 2), np.array_split(shear1[1], 2) g1_2nd, g2_2nd = np.array_split(shear2[0], 2), np.array_split(shear2[1], 2) k1 = np.array_split(k1, 2) if (k1 is not None) else [None,None] k2 = np.array_split(k2, 2) if (k2 is not None) else [None,None] w1 = np.array_split(w1, 2) if (w1 is not None) else [None,None] w2 = np.array_split(w2, 2) if (w2 is not None) else [None,None] cat1 = [treecorr.Catalog(g1=g1_1st[h], g2=g2_1st[h], k=k1[h], ra=ra1[h], dec=dec1[h], w=w1[h], ra_units='degrees', dec_units='degrees') for h in [0,1]] cat2 = [treecorr.Catalog(g1=g1_2nd[h], g2=g2_2nd[h], k=k2[h], ra=ra2[h], dec=dec2[h], w=w2[h], ra_units='degrees', dec_units='degrees') for h in [0,1]] else: ra1, dec1 = pos1 ra2, dec2 = pos2 g1_1st, g2_1st = shear1 g1_2nd, g2_2nd = shear2 cat1 = treecorr.Catalog(g1=g1_1st, g2=g2_1st, k=k1, ra=ra1, dec=dec1, w=w1, ra_units='degrees', dec_units='degrees') cat2 = treecorr.Catalog(g1=g1_2nd, g2=g2_2nd, k=k2, ra=ra2, dec=dec2, w=w2, ra_units='degrees', dec_units='degrees') gg = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_slop=bin_slop, sep_units='degrees', logger=logger) if same_zshell and same_cell: gg.process_auto(cat,num_threads=num_threads) elif same_zshell: # just wanted to distrubute the workload fairly for two encounters (didn't want to make one of the cores idle) if unique_encounter: # the following two counts shouldn't be doubled up cuz they're the same in both directions gg.process_cross(cat1[0],cat2[0],num_threads=num_threads) gg.process_cross(cat1[1],cat2[1],num_threads=num_threads) else: # in the other encounter cat1 and cat2 are switched but does not matter anyway gg.process_cross(cat2[0],cat1[1],num_threads=num_threads) gg.process_cross(cat2[1],cat1[0],num_threads=num_threads) else: gg.process_cross(cat1,cat2,num_threads=num_threads) if same_zshell and same_cell: varg1 = treecorr.calculateVarG(cat) varg2 = varg1 elif same_cell: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) else: varg1 = np.nan varg2 = np.nan return gg, varg1, varg2
def process(self,
cat1,
cat2=None,
cat3=None,
metric=None,
num_threads=None):
"""Accumulate the number of triangles of points between cat1, cat2, and cat3.
- If only 1 argument is given, then compute an auto-correlation function.
- If 2 arguments are given, then compute a cross-correlation function with the
first catalog taking two corners of the triangles. (Not implemented yet.)
- If 3 arguments are given, then compute a cross-correlation function.
All arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Note: For a correlation of multiple catalogs, it matters which corner of the
triangle comes from which catalog. The final accumulation will have
d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between
points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate
all the possible triangles between three catalogs, you should call this
multiple times with the different catalogs in different positions.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the first N field.
cat2 (Catalog): A catalog or list of catalogs for the second N field, if any.
(default: None)
cat3 (Catalog): A catalog or list of catalogs for the third N field, if any.
(default: None)
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2]
if cat3 is not None and not isinstance(cat3, list): cat3 = [cat3]
if cat2 is None and cat3 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
varg3 = varg1
self.logger.info("varg = %f: sig_g = %f", varg1, math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
elif (cat2 is None) != (cat3 is None):
raise NotImplementedError("No partial cross GGG yet.")
else:
assert cat2 is not None and cat3 is not None
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
varg3 = treecorr.calculateVarG(cat3)
self.logger.info("varg1 = %f: sig_g = %f", varg1, math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_g = %f", varg2, math.sqrt(varg2))
self.logger.info("varg3 = %f: sig_g = %f", varg3, math.sqrt(varg3))
self._process_all_cross(cat1, cat2, cat3, metric, num_threads)
self.finalize(varg1, varg2, varg3)
def process(self, cat1, cat2=None, metric=None, num_threads=None):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the first G field.
:param cat2: A catalog or list of catalogs for the second G field, if any.
(default: None)
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2]
if len(cat1) == 0:
raise AttributeError("No catalogs provided for cat1")
if metric is None:
metric = treecorr.config.get(self.config, 'metric', str,
'Euclidean')
if metric not in ['Euclidean', 'Rperp']:
raise ValueError("Invalid metric.")
if cat2 is None or len(cat2) == 0:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2,
math.sqrt(varg2))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(varg1, varg2)
def process(self,
cat1,
cat2=None,
metric=None,
num_threads=None,
comm=None,
low_mem=False):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the first G field.
cat2 (Catalog): A catalog or list of catalogs for the second G field, if any.
(default: None)
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between
processes. If used, the rank=0 process will have the final
computation. This only works if using patches. (default: None)
low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage.
This only works if using patches. (default: False)
"""
import math
self.clear()
if not isinstance(cat1, list):
self.npatch1 = cat1._npatch
cat1 = cat1.get_patches(low_mem=low_mem)
if cat2 is None: self.npatch2 = self.npatch1
if cat2 is not None and not isinstance(cat2, list):
self.npatch2 = cat2._npatch
cat2 = cat2.get_patches(low_mem=low_mem)
if cat2 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads, comm, low_mem)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2,
math.sqrt(varg2))
self._process_all_cross(cat1, cat2, metric, num_threads, comm,
low_mem)
self.finalize(varg1, varg2)
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None):
"""Accumulate the number of triangles of points between cat1, cat2, and cat3.
- If only 1 argument is given, then compute an auto-correlation function.
- If 2 arguments are given, then compute a cross-correlation function with the
first catalog taking two corners of the triangles. (Not implemented yet.)
- If 3 arguments are given, then compute a cross-correlation function.
All arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Note: For a correlation of multiple catalogs, it matters which corner of the
triangle comes from which catalog. The final accumulation will have
d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between
points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate
all the possible triangles between three catalogs, you should call this
multiple times with the different catalogs in different positions.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the first N field.
cat2 (Catalog): A catalog or list of catalogs for the second N field, if any.
(default: None)
cat3 (Catalog): A catalog or list of catalogs for the third N field, if any.
(default: None)
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2]
if cat3 is not None and not isinstance(cat3,list): cat3 = [cat3]
if cat2 is None and cat3 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
varg3 = varg1
self.logger.info("varg = %f: sig_g = %f",varg1,math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
elif (cat2 is None) != (cat3 is None):
raise NotImplementedError("No partial cross GGG yet.")
else:
assert cat2 is not None and cat3 is not None
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
varg3 = treecorr.calculateVarG(cat3)
self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2))
self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3))
self._process_all_cross(cat1,cat2,cat3, metric, num_threads)
self.finalize(varg1,varg2,varg3)
def process(self, cat1, cat2=None, metric=None, num_threads=None):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the first G field.
:param cat2: A catalog or list of catalogs for the second G field, if any.
(default: None)
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2]
if len(cat1) == 0:
raise AttributeError("No catalogs provided for cat1")
if metric is None:
metric = treecorr.config.get(self.config,'metric',str,'Euclidean')
if metric not in ['Euclidean', 'Rperp']:
raise ValueError("Invalid metric.")
if cat2 is None or len(cat2) == 0:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2))
self._process_all_cross(cat1,cat2, metric, num_threads)
self.finalize(varg1,varg2)
def process(self, cat1, cat2, metric=None, num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the K field.
cat2 (Catalog): A catalog or list of catalogs for the G field.
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if not isinstance(cat2,list): cat2 = [cat2]
vark = treecorr.calculateVarK(cat1)
varg = treecorr.calculateVarG(cat2)
self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark))
self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg))
self._process_all_cross(cat1,cat2,metric,num_threads)
self.finalize(vark,varg)
def process(self, cat1, cat2, metric=None, num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the K field.
cat2 (Catalog): A catalog or list of catalogs for the G field.
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if not isinstance(cat2, list): cat2 = [cat2]
vark = treecorr.calculateVarK(cat1)
varg = treecorr.calculateVarG(cat2)
self.logger.info("vark = %f: sig_k = %f", vark, math.sqrt(vark))
self.logger.info("varg = %f: sig_sn (per component) = %f", varg,
math.sqrt(varg))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(vark, varg)
def process(self, cat1, cat2, metric=None, num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the N field.
:param cat2: A catalog or list of catalogs for the G field.
:param metric: Which metric to use for distance measurements. Options are given
in the doc string of :class:`~treecorr.BinnedCorr2`.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if not isinstance(cat2, list): cat2 = [cat2]
varg = treecorr.calculateVarG(cat2)
self.logger.info("varg = %f: sig_sn (per component) = %f", varg,
math.sqrt(varg))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(varg)
def pos_shear_corr(pos_lens,pos_source,shear_source,k_source=None,w_lense=None,w_source=None,same_cell=False,num_threads=0): nbins = 6 min_sep = 0.05 # 3 arcmin max_sep = 3.0 # 180 arcmin bin_size = (max_sep-min_sep)/nbins # roughly bin_slop = 0.05/bin_size # 0.1 -> 0.05 # 2pt_pipeline for des used bin_slop: 0.01 here: https://github.com/des-science/2pt_pipeline/blob/master/pipeline/twopt_pipeline.yaml logger = None ra_lens, dec_lens = pos_lens ra_source, dec_source = pos_source g1_source, g2_source = shear_source # foreground (lens) cat_lens = treecorr.Catalog(ra=ra_lens, dec=dec_lens, w=w_lense, ra_units='degrees', dec_units='degrees') # background (source) cat_source = treecorr.Catalog(ra=ra_source, dec=dec_source, w=w_source, g1=g1_source, g2=g2_source, k=k_source, ra_units='degrees', dec_units='degrees') ng = treecorr.NGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_slop=bin_slop, sep_units='degrees', logger=logger) ng.process_cross(cat_lens,cat_source,num_threads=num_threads) # there's no process_auto for this object # one shear variance per pixel per source zbin varg = treecorr.calculateVarG(cat_source) if same_cell else np.nan return ng, varg
def process(self, cat1, cat2=None, metric='Euclidean', num_threads=None):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the first G field.
:param cat2: A catalog or list of catalogs for the second G field, if any.
(default: None)
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth r1,r2, if d is the normal Euclidean distance
and Rparallel = |r1 - r2|, then Rperp^2 = d^2 - Rparallel^2.
(default: 'Euclidean')
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2]
if len(cat1) == 0:
raise AttributeError("No catalogs provided for cat1")
if metric not in ['Euclidean', 'Rperp']:
raise ValueError("Invalid metric.")
if cat2 is None or len(cat2) == 0:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2))
self._process_all_cross(cat1,cat2, metric, num_threads)
self.finalize(varg1,varg2)
def process(self, cat1, cat2=None, metric=None, num_threads=None):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the first G field.
cat2 (Catalog): A catalog or list of catalogs for the second G field, if any.
(default: None)
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2]
if cat2 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1,
math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2,
math.sqrt(varg2))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(varg1, varg2)
def process(self, cat1, cat2=None, metric=None, num_threads=None):
"""Compute the correlation function.
If only 1 argument is given, then compute an auto-correlation function.
If 2 arguments are given, then compute a cross-correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Parameters:
cat1 (Catalog): A catalog or list of catalogs for the first G field.
cat2 (Catalog): A catalog or list of catalogs for the second G field, if any.
(default: None)
metric (str): Which metric to use. See `Metrics` for details.
(default: 'Euclidean'; this value can also be given in the
constructor in the config dict.)
num_threads (int): How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given
in the constructor in the config dict.)
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2]
if cat2 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
varg1 = treecorr.calculateVarG(cat1)
varg2 = treecorr.calculateVarG(cat2)
self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2))
self._process_all_cross(cat1,cat2, metric, num_threads)
self.finalize(varg1,varg2)
def process(self, cat1, cat2, metric=None, num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the K field.
:param cat2: A catalog or list of catalogs for the G field.
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
- 'Rlens' = the projected distance perpendicular to the first point
in the pair (taken to be a lens) to the line of sight to the second
point (e.g. a lensed source galaxy).
- 'Arc' = the true great circle distance for spherical coordinates.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if not isinstance(cat2, list): cat2 = [cat2]
if len(cat1) == 0:
raise ValueError("No catalogs provided for cat1")
if len(cat2) == 0:
raise ValueError("No catalogs provided for cat2")
vark = treecorr.calculateVarK(cat1)
varg = treecorr.calculateVarG(cat2)
self.logger.info("vark = %f: sig_k = %f", vark, math.sqrt(vark))
self.logger.info("varg = %f: sig_sn (per component) = %f", varg,
math.sqrt(varg))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(vark, varg)
def process(self, cat1, cat2, metric=None, num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the K field.
:param cat2: A catalog or list of catalogs for the G field.
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
- 'Rlens' = the projected distance perpendicular to the first point
in the pair (taken to be a lens) to the line of sight to the second
point (e.g. a lensed source galaxy).
- 'Arc' = the true great circle distance for spherical coordinates.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if not isinstance(cat2,list): cat2 = [cat2]
if len(cat1) == 0:
raise ValueError("No catalogs provided for cat1")
if len(cat2) == 0:
raise ValueError("No catalogs provided for cat2")
vark = treecorr.calculateVarK(cat1)
varg = treecorr.calculateVarG(cat2)
self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark))
self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg))
self._process_all_cross(cat1,cat2,metric,num_threads)
self.finalize(vark,varg)
def process(self, cat1, cat2, metric="Euclidean", num_threads=None):
"""Compute the correlation function.
Both arguments may be lists, in which case all items in the list are used
for that element of the correlation.
:param cat1: A catalog or list of catalogs for the N field.
:param cat2: A catalog or list of catalogs for the G field.
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth r1,r2, if d is the normal Euclidean distance
and Rparallel = |r1 - r2|, then Rperp^2 = d^2 - Rparallel^2.
(default: 'Euclidean')
:param num_threads: How many OpenMP threads to use during the calculation.
(default: None, which means to first check for a num_threads parameter
in self.config, then default to querying the number of cpu cores and
try to use that many threads.) Note that this won't work if the system's
C compiler is clang, such as on MacOS systems.
"""
import math
self.clear()
if not isinstance(cat1, list):
cat1 = [cat1]
if not isinstance(cat2, list):
cat2 = [cat2]
if len(cat1) == 0:
raise ValueError("No catalogs provided for cat1")
if len(cat2) == 0:
raise ValueError("No catalogs provided for cat2")
varg = treecorr.calculateVarG(cat2)
self.logger.info("varg = %f: sig_sn (per component) = %f", varg, math.sqrt(varg))
self._process_all_cross(cat1, cat2, metric, num_threads)
self.finalize(varg)
def test_var():
nobj = 5000
rng = np.random.RandomState(8675309)
# First without weights
cats = []
allg1 = []
allg2 = []
allk = []
for i in range(10):
x = rng.random_sample(nobj)
y = rng.random_sample(nobj)
g1 = rng.random_sample(nobj) - 0.5
g2 = rng.random_sample(nobj) - 0.5
k = rng.random_sample(nobj) - 0.5
cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, k=k)
varg = (np.sum(g1**2) + np.sum(g2**2)) / (2.*len(g1))
vark = np.sum(k**2) / len(k)
assert np.isclose(cat.vark, vark)
assert np.isclose(cat.varg, varg)
assert np.isclose(treecorr.calculateVarK(cat), vark)
assert np.isclose(treecorr.calculateVarK([cat]), vark)
assert np.isclose(treecorr.calculateVarG(cat), varg)
assert np.isclose(treecorr.calculateVarG([cat]), varg)
cats.append(cat)
allg1.extend(g1)
allg2.extend(g2)
allk.extend(k)
allg1 = np.array(allg1)
allg2 = np.array(allg2)
allk = np.array(allk)
varg = (np.sum(allg1**2) + np.sum(allg2**2)) / (2. * len(allg1))
vark = np.sum(allk**2) / len(allk)
assert np.isclose(treecorr.calculateVarG(cats), varg)
assert np.isclose(treecorr.calculateVarK(cats), vark)
# Now with weights
cats = []
allg1 = []
allg2 = []
allk = []
allw = []
for i in range(10):
x = rng.random_sample(nobj)
y = rng.random_sample(nobj)
w = rng.random_sample(nobj)
g1 = rng.random_sample(nobj)
g2 = rng.random_sample(nobj)
k = rng.random_sample(nobj)
cat = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, k=k)
varg = np.sum(w**2 * (g1**2 + g2**2)) / np.sum(w) / 2.
vark = np.sum(w**2 * k**2) / np.sum(w)
assert np.isclose(cat.varg, varg)
assert np.isclose(cat.vark, vark)
assert np.isclose(treecorr.calculateVarG(cat), varg)
assert np.isclose(treecorr.calculateVarG([cat]), varg)
assert np.isclose(treecorr.calculateVarK(cat), vark)
assert np.isclose(treecorr.calculateVarK([cat]), vark)
cats.append(cat)
allg1.extend(g1)
allg2.extend(g2)
allk.extend(k)
allw.extend(w)
allg1 = np.array(allg1)
allg2 = np.array(allg2)
allk = np.array(allk)
allw = np.array(allw)
varg = np.sum(allw**2 * (allg1**2 + allg2**2)) / np.sum(allw) / 2.
vark = np.sum(allw**2 * allk**2) / np.sum(allw)
assert np.isclose(treecorr.calculateVarG(cats), varg)
assert np.isclose(treecorr.calculateVarK(cats), vark)
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None):
"""Accumulate the number of triangles of points between cat1, cat2, and cat3.
- If only 1 argument is given, then compute an auto-correlation function.
- If 2 arguments are given, then compute a cross-correlation function with the
first catalog taking two corners of the triangles. (Not implemented yet.)
- If 3 arguments are given, then compute a cross-correlation function.
All arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Note: For a correlation of multiple catalogs, it matters which corner of the
triangle comes from which catalog. The final accumulation will have
d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between
points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate
all the possible triangles between three catalogs, you should call this
multiple times with the different catalogs in different positions.
:param cat1: A catalog or list of catalogs for the first N field.
:param cat2: A catalog or list of catalogs for the second N field, if any.
(default: None)
:param cat3: A catalog or list of catalogs for the third N field, if any.
(default: None)
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1,list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2]
if cat3 is not None and not isinstance(cat3,list): cat3 = [cat3]
if len(cat1) == 0:
raise ValueError("No catalogs provided for cat1")
if cat2 is not None and len(cat2) == 0:
cat2 = None
if cat3 is not None and len(cat3) == 0:
cat3 = None
if cat2 is None and cat3 is not None:
raise NotImplemented("No partial cross GGG yet.")
if cat3 is None and cat2 is not None:
raise NotImplemented("No partial cross GGG yet.")
if cat2 is None and cat3 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
varg3 = varg1
self.logger.info("varg = %f: sig_g = %f",varg1,math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
assert cat2 is not None and cat3 is not None
varg1 = treecorr.calculateVarK(cat1)
varg2 = treecorr.calculateVarK(cat2)
varg3 = treecorr.calculateVarK(cat3)
self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2))
self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3))
self._process_all_cross(cat1,cat2,cat3, metric, num_threads)
self.finalize(varg1,varg2,varg3)
def compute_w(dataf, randf, config, estimator='PW1', compensated=1, nbins_rpar=30, random_oversampling=10., verbosity=1, largePi=0, **kwargs): """ dataf: paths to galaxy samples randf: paths to random points corresponding to galaxy samples config: path to config file, or dict specifying file types; column names/numbers, etc. a la TreeCorr configuration estimator: 'PW1', 'PW2', 'AS' or 'wgg' to specify correlation & estimator nbins_rpar: number of line-of-sight bins for 3D correlation function -- specify limits in config arg """ assert estimator in ['PW1', 'PW2', 'AS', 'wgg'], "for IA: estimator must be 'PW1/2' (pair_weighted 1=RDs, 2=RRs norm) or 'AS' (average shear), for clustering: 'wgg'" assert hasattr(dataf, '__iter__'), "dataf must be list/tuple of 2x paths; density, shapes for IA, or density1, density2 for clustering (these can be the same!)" assert hasattr(randf, '__iter__'), "randf must be list/tuple of 2x paths for randoms corresponding to each of dataf (these can be the same!)" nbins_rpar = int(nbins_rpar) random_oversampling = float(random_oversampling) verbosity = int(verbosity) largePi = int(largePi) if type(config) == str: config = treecorr.read_config(config) if config['file_type'] == 'ASCII': config['ra_col'] = int(config['ra_col']) config['dec_col'] = int(config['dec_col']) config['r_col'] = int(config['r_col']) config['g1_col'] = int(config['g1_col']) config['g2_col'] = int(config['g2_col']) config['verbose'] = verbosity if not largePi: Pi = np.linspace(config['min_rpar'], config['max_rpar'], nbins_rpar + 1) else: dPi = (config['max_rpar'] - config['min_rpar']) / nbins_rpar Pi = np.arange(config['min_rpar']*1.5, config['max_rpar']*1.5 + dPi, step=dPi, dtype=float) if estimator in ['PW1', 'PW2', 'AS']: corr = 'ng' gt_3D = np.zeros([len(Pi)-1, config['nbins']]) gx_3D = np.zeros([len(Pi)-1, config['nbins']]) varg_3D = np.zeros([len(Pi)-1, config['nbins']]) npair_3D = np.zeros([len(Pi)-1, config['nbins']]) elif estimator == 'wgg': corr = 'nn' wgg_3D = np.zeros([len(Pi)-1, config['nbins']]) else: raise ValueError, "unsupported estimator choice" config_r = config.copy() config_r['flip_g1'] = False config_r['flip_g2'] = False data1 = treecorr.Catalog(dataf[0], config) # 1 = density/lenses data2 = treecorr.Catalog(dataf[1], config) # 2 = shapes rand1 = treecorr.Catalog(randf[0], config_r, is_rand=1) rand2 = treecorr.Catalog(randf[1], config_r, is_rand=1) f1 = data1.ntot * random_oversampling / float(rand1.ntot) f2 = data2.ntot * random_oversampling / float(rand2.ntot) rand1.w = np.array(np.random.rand(rand1.ntot) < f1, dtype=float) rand2.w = np.array(np.random.rand(rand2.ntot) < f2, dtype=float) varg = treecorr.calculateVarG(data2) for p in tqdm(range(len(Pi)-1), ascii=True, desc='Correlating'): if largePi & any(abs(Pi[p:p+2]) < config['max_rpar']): continue conf_pi = config.copy() conf_pi['min_rpar'] = Pi[p] conf_pi['max_rpar'] = Pi[p+1] if corr == 'ng': ng = treecorr.NGCorrelation(conf_pi) rg = treecorr.NGCorrelation(conf_pi) ng.process_cross(data1, data2) rg.process_cross(rand1, data2) if estimator == 'PW1': # RDs norm f = data1.ntot / rand1.w.sum() norm1 = rg.weight * f norm2 = rg.weight if estimator == 'PW2': # RRs norm RRs = get_RRs(rand1, rand2, conf_pi, **kwargs) f1 = data1.ntot / rand1.w.sum() f2 = data2.ntot / rand2.w.sum() norm1 = RRs * f1 * f2 norm2 = RRs * f2 elif estimator == 'AS': # DDs norm norm1 = ng.weight norm2 = rg.weight if int(compensated): gt_3D[p] += (ng.xi / norm1) - (rg.xi / norm2) gx_3D[p] += (ng.xi_im / norm1) - (rg.xi_im / norm2) varg_3D[p] += (varg / norm1) + (varg / norm2) npair_3D[p] += ng.npairs else: gt_3D[p] += ng.xi / norm1 gx_3D[p] += ng.xi_im / norm1 varg_3D[p] += varg / norm1 npair_3D[p] += ng.npairs elif corr == 'nn': nn = treecorr.NNCorrelation(conf_pi) rr = treecorr.NNCorrelation(conf_pi) nr = treecorr.NNCorrelation(conf_pi) rn = treecorr.NNCorrelation(conf_pi) if dataf[0] == dataf[1]: nn.process(data1) rr.process(rand1) nr.process(data1, rand1) xi, varxi = nn.calculateXi(rr, nr) else: nn.process(data1, data2) rr.process(rand1, rand2) nr.process(data1, rand2) rn.process(rand1, data2) xi, varxi = nn.calculateXi(rr, nr, rn) wgg_3D[p] += xi if corr == 'ng': #gt = np.trapz(gt_3D, x=midpoints(Pi), axis=0) #gx = np.trapz(gx_3D, x=midpoints(Pi), axis=0) #gt = simps(gt_3D, x=midpoints(Pi), axis=0) gt = np.sum(gt_3D * (Pi[1] - Pi[0]), axis=0) gx = np.sum(gx_3D * (Pi[1] - Pi[0]), axis=0) varg = np.sum(varg_3D, axis=0) npair = np.sum(npair_3D, axis=0) r = ng.rnom return r, gt, gx, varg**0.5, npair elif corr == 'nn': wgg = np.sum(wgg_3D * (Pi[1] - Pi[0]), axis=0) r = nn.rnom return r, wgg
def process(self,
cat1,
cat2=None,
cat3=None,
metric=None,
num_threads=None):
"""Accumulate the number of triangles of points between cat1, cat2, and cat3.
- If only 1 argument is given, then compute an auto-correlation function.
- If 2 arguments are given, then compute a cross-correlation function with the
first catalog taking two corners of the triangles. (Not implemented yet.)
- If 3 arguments are given, then compute a cross-correlation function.
All arguments may be lists, in which case all items in the list are used
for that element of the correlation.
Note: For a correlation of multiple catalogs, it matters which corner of the
triangle comes from which catalog. The final accumulation will have
d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between
points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate
all the possible triangles between three catalogs, you should call this
multiple times with the different catalogs in different positions.
:param cat1: A catalog or list of catalogs for the first N field.
:param cat2: A catalog or list of catalogs for the second N field, if any.
(default: None)
:param cat3: A catalog or list of catalogs for the third N field, if any.
(default: None)
:param metric: Which metric to use for distance measurements. Options are:
- 'Euclidean' = straight line Euclidean distance between two points.
For spherical coordinates (ra,dec without r), this is the chord
distance between points on the unit sphere.
- 'Rperp' = the perpendicular component of the distance. For two points
with distance from Earth `r1, r2`, if `d` is the normal Euclidean
distance and :math:`Rparallel = |r1-r2|`, then we define
:math:`Rperp^2 = d^2 - Rparallel^2`.
(default: 'Euclidean'; this value can also be given in the constructor
in the config dict.)
:param num_threads: How many OpenMP threads to use during the calculation.
(default: use the number of cpu cores; this value can also be given in
the constructor in the config dict.) Note that this won't work if the
system's C compiler is clang prior to version 3.7.
"""
import math
self.clear()
if not isinstance(cat1, list): cat1 = [cat1]
if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2]
if cat3 is not None and not isinstance(cat3, list): cat3 = [cat3]
if len(cat1) == 0:
raise ValueError("No catalogs provided for cat1")
if cat2 is not None and len(cat2) == 0:
cat2 = None
if cat3 is not None and len(cat3) == 0:
cat3 = None
if cat2 is None and cat3 is not None:
raise NotImplemented("No partial cross GGG yet.")
if cat3 is None and cat2 is not None:
raise NotImplemented("No partial cross GGG yet.")
if cat2 is None and cat3 is None:
varg1 = treecorr.calculateVarG(cat1)
varg2 = varg1
varg3 = varg1
self.logger.info("varg = %f: sig_g = %f", varg1, math.sqrt(varg1))
self._process_all_auto(cat1, metric, num_threads)
else:
assert cat2 is not None and cat3 is not None
varg1 = treecorr.calculateVarK(cat1)
varg2 = treecorr.calculateVarK(cat2)
varg3 = treecorr.calculateVarK(cat3)
self.logger.info("varg1 = %f: sig_g = %f", varg1, math.sqrt(varg1))
self.logger.info("varg2 = %f: sig_g = %f", varg2, math.sqrt(varg2))
self.logger.info("varg3 = %f: sig_g = %f", varg3, math.sqrt(varg3))
self._process_all_cross(cat1, cat2, cat3, metric, num_threads)
self.finalize(varg1, varg2, varg3)
def measure_cross_rho(tile_data, max_sep, tags=None, prefix='piff'):
"""Compute the rho statistics
"""
import treecorr
ntilings = len(tile_data)
print('len(tile_data) = ', ntilings)
de1 = [d['obs_e1'] - d[prefix + '_e1'] for d in tile_data]
de2 = [d['obs_e2'] - d[prefix + '_e2'] for d in tile_data]
dt = [(d['obs_T'] - d[prefix + '_T']) / d['obs_T'] for d in tile_data]
for k in range(len(tile_data)):
print('k = ', k)
print('mean de = ', np.mean(de1[k]), np.mean(de2[k]))
print('mean dt = ', np.mean(dt[k]))
ecats = [
treecorr.Catalog(ra=d['ra'],
dec=d['dec'],
ra_units='deg',
dec_units='deg',
g1=d['obs_e1'],
g2=d['obs_e2']) for d in tile_data
]
for cat in ecats:
cat.name = 'ecat'
decats = [
treecorr.Catalog(ra=d['ra'],
dec=d['dec'],
ra_units='deg',
dec_units='deg',
g1=de1[k],
g2=de2[k]) for k, d in enumerate(tile_data)
]
for cat in decats:
cat.name = 'decat'
dtcats = [
treecorr.Catalog(ra=d['ra'],
dec=d['dec'],
ra_units='deg',
dec_units='deg',
k=dt[k],
g1=d['obs_e1'] * dt[k],
g2=d['obs_e2'] * dt[k])
for k, d in enumerate(tile_data)
]
for cat in dtcats:
cat.name = 'dtcat'
if tags is not None:
for catlist in [ecats, decats, dtcats]:
for cat, tag in zip(catlist, tags):
cat.name = tag + ":" + cat.name
bin_config = dict(
sep_units='arcmin',
bin_slop=0.1,
min_sep=0.5,
max_sep=max_sep,
bin_size=0.2,
#min_sep = 2.5,
#max_sep = 250,
#nbins = 20,
)
results = []
for (catlist1, catlist2) in [(decats, decats), (ecats, decats),
(dtcats, dtcats), (decats, dtcats),
(ecats, dtcats)]:
catnames1 = [cat.name for cat in catlist1]
catnames2 = [cat.name for cat in catlist2]
print('Doing correlation of %s vs %s' % (catnames1, catnames2))
rho = treecorr.GGCorrelation(bin_config, verbose=2)
# Avoid all auto correlations:
for i in range(ntilings):
for j in range(ntilings):
if i == j: continue
if catlist1 is catlist2 and i > j: continue
print('names: ', catlist1[i].name, catlist2[j].name)
rho.process_cross(catlist1[i], catlist2[j])
varg1 = treecorr.calculateVarG(catlist1)
varg2 = treecorr.calculateVarG(catlist2)
rho.finalize(varg1, varg2)
results.append(rho)
return results
def measure_cross_rho(tile_data, max_sep, tags=None, prefix='piff', lucas=False):
"""Compute the rho statistics
"""
import treecorr
ntilings = len(tile_data)
print('len(tile_data) = ',ntilings)
q1 = [ d['obs_e1']-d[prefix+'_e1'] for d in tile_data ]
q2 = [ d['obs_e2']-d[prefix+'_e2'] for d in tile_data ]
dt = [ (d['obs_T']-d[prefix+'_T'])/d['obs_T'] for d in tile_data ]
for k in range(len(tile_data)):
print('k = ',k)
print('mean q = ',np.mean(q1[k]),np.mean(q2[k]))
print('mean dt = ',np.mean(dt[k]))
ecats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
g1=d['obs_e1'], g2=d['obs_e2'])
for d in tile_data ]
for cat in ecats: cat.name = 'ecat'
qcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
g1=q1[k], g2=q2[k]) for k,d in enumerate(tile_data) ]
for cat in qcats: cat.name = 'qcat'
dtcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
k=dt[k], g1=d['obs_e1']*dt[k], g2=d['obs_e2']*dt[k])
for k,d in enumerate(tile_data) ]
for cat in dtcats: cat.name = 'dtcat'
if tags is not None:
for catlist in [ ecats, qcats, dtcats]:
for cat, tag in zip(catlist, tags):
cat.name = tag + ":" + cat.name
bin_config = dict(
sep_units = 'arcmin',
bin_slop = 0.1,
min_sep = 0.5,
max_sep = max_sep,
bin_size = 0.2,
)
if lucas:
bin_config['min_sep'] = 2.5
bin_config['max_sep'] = 250.
bin_config['nbins'] = 20
del bin_config['bin_size']
results = []
for (catlist1, catlist2) in [ (qcats, qcats),
(ecats, qcats),
(dtcats, dtcats),
(qcats, dtcats),
(ecats, dtcats) ]:
catnames1 = [ cat.name for cat in catlist1 ]
catnames2 = [ cat.name for cat in catlist2 ]
print('Doing correlation of %s vs %s'%(catnames1, catnames2))
rho = treecorr.GGCorrelation(bin_config, verbose=2)
# Avoid all auto correlations:
for i in range(ntilings):
for j in range(ntilings):
if i == j: continue
if catlist1 is catlist2 and i > j: continue
print('names: ',catlist1[i].name,catlist2[j].name)
rho.process_cross(catlist1[i], catlist2[j])
varg1 = treecorr.calculateVarG(catlist1)
varg2 = treecorr.calculateVarG(catlist2)
rho.finalize(varg1, varg2)
results.append(rho)
return results
def main():
args = parse_args()
# Make the work directory if it does not exist yet.
cat_dir = os.path.expanduser(args.psf_cats)
print 'psfcats dir = ',cat_dir
print os.path.join(cat_dir, '*_psf.fits')
cat_files = sorted(glob.glob(os.path.join(cat_dir, '*_psf.fits')))
min_sep = 0.5
max_sep = 20
bin_size = 0.3
bin_slop = 0.1
rho1 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=0)
rho2 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=0)
rho3 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=0)
rho4 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=0)
rho5 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=0)
all_ra = []
all_dec = []
all_e1 = []
all_e2 = []
all_s = []
for file in cat_files:
print 'Processing ',file
try:
with pyfits.open(file) as pyf:
data = pyf[1].data.copy()
except:
print 'Failed to read data from %s. Skiping.'%file
continue
#print 'len(data) = ',len(data)
#print 'shape = ',data.shape
mask = data['flag'] == 0
#print 'len(mask) = ',len(mask)
#print 'nonzero(mask) = ',numpy.sum(mask)
if mask.sum() == 0:
print ' All objects in this file are flagged.'
continue
ra = data['ra'][mask]
dec = data['dec'][mask]
e1 = data['e1'][mask]
e2 = data['e2'][mask]
s = data['size'][mask]
m_e1 = data['psf_e1'][mask]
m_e2 = data['psf_e2'][mask]
m_s = data['psf_size'][mask]
all_ra.append(ra)
all_dec.append(dec)
all_e1.append(e1)
all_e2.append(e2)
all_s.append(s)
de1 = e1-m_e1
de2 = e2-m_e2
dt = (s**2-m_s**2)/s**2
#print 'e1 = ',e1
#print 'e2 = ',e2
#print 'de1 = ',de1
#print 'de2 = ',de2
#print 'dt = ',dt
ecat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=e1, g2=e2)
decat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=de1, g2=de2)
dtcat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg',
k=dt, g1=dt*e1, g2=dt*e2)
ecat.name = 'ecat'
decat.name = 'decat'
dtcat.name = 'dtcat'
for (cat1, cat2, rho) in [ (decat, decat, rho1),
(ecat, decat, rho2),
(dtcat, dtcat, rho3),
(decat, dtcat, rho4),
(ecat, dtcat, rho5) ]:
#print 'Doing correlation of %s vs %s'%(cat1.name, cat2.name)
if cat1 is cat2:
rho.process_auto(cat1)
else:
rho.process_cross(cat1, cat2)
ra = numpy.concatenate(all_ra)
dec = numpy.concatenate(all_dec)
e1 = numpy.concatenate(all_e1)
e2 = numpy.concatenate(all_e2)
s = numpy.concatenate(all_s)
allcat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=e1, g2=e2)
varg = treecorr.calculateVarG(allcat)
for rho in [ rho1, rho2, rho3, rho4, rho5 ]:
rho.finalize(varg, varg)
print '\nFinished processing all files'
rho1.write(os.path.join(cat_dir,'rho1.fits'))
rho2.write(os.path.join(cat_dir,'rho2.fits'))
rho3.write(os.path.join(cat_dir,'rho3.fits'))
rho4.write(os.path.join(cat_dir,'rho4.fits'))
rho5.write(os.path.join(cat_dir,'rho5.fits'))
def measure_cross_rho(tile_data, max_sep, tags=None, prefix=''):
"""Compute the rho statistics
"""
import treecorr
ntilings = len(tile_data)
print 'len(tile_data) = ',ntilings
de1 = [ d[prefix+'e1']-d['psf_e1'] for d in tile_data ]
de2 = [ d[prefix+'e2']-d['psf_e2'] for d in tile_data ]
dt = [ (d[prefix+'size']**2-d['psf_size']**2)/d['size']**2 for d in tile_data ]
for k in range(len(tile_data)):
print 'k = ',k
print 'mean de = ',numpy.mean(de1[k]),numpy.mean(de2[k])
print 'mean dt = ',numpy.mean(dt[k])
ecats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
g1=d['e1'], g2=d['e2'])
for d in tile_data ]
for cat in ecats: cat.name = 'ecat'
decats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
g1=de1[k], g2=de2[k]) for k,d in enumerate(tile_data) ]
for cat in decats: cat.name = 'decat'
dtcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg',
k=dt[k], g1=d['e1']*dt[k], g2=d['e2']*dt[k])
for k,d in enumerate(tile_data) ]
for cat in dtcats: cat.name = 'dtcat'
if tags is not None:
for catlist in [ ecats, decats, dtcats]:
for cat, tag in zip(catlist, tags):
cat.name = tag + ":" + cat.name
min_sep = 0.3
bin_size = 0.2
bin_slop = 0.3
results = []
for (catlist1, catlist2) in [ (decats, decats),
(ecats, decats),
(dtcats, dtcats),
(decats, dtcats),
(ecats, dtcats) ]:
catnames1 = [ cat.name for cat in catlist1 ]
catnames2 = [ cat.name for cat in catlist2 ]
print 'Doing correlation of %s vs %s'%(catnames1, catnames2)
rho = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin',
bin_size=bin_size, bin_slop=bin_slop, verbose=2)
# Avoid all auto correlations:
for i in range(ntilings):
for j in range(ntilings):
if i == j: continue
if catlist1 is catlist2 and i > j: continue
print 'names: ',catlist1[i].name,catlist2[j].name
rho.process_cross(catlist1[i], catlist2[j])
varg1 = treecorr.calculateVarG(catlist1)
varg2 = treecorr.calculateVarG(catlist2)
rho.finalize(varg1, varg2)
results.append(rho)
return results
def test_pieces():
# Test that we can do the calculation in pieces and recombine the results
ncats = 3
data_cats = []
nlens = 1000
nsource = 30000
gamma0 = 0.05
r0 = 10.
L = 50. * r0
numpy.random.seed(8675309)
xl = (numpy.random.random_sample(nlens)-0.5) * L
yl = (numpy.random.random_sample(nlens)-0.5) * L
xs = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L
ys = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L
g1 = numpy.zeros( (nsource,ncats) )
g2 = numpy.zeros( (nsource,ncats) )
w = numpy.random.random_sample( (nsource,ncats) ) + 0.5
for x,y in zip(xl,yl):
dx = xs-x
dy = ys-y
r2 = dx**2 + dy**2
gammat = gamma0 * numpy.exp(-0.5*r2/r0**2)
g1 += -gammat * (dx**2-dy**2)/r2
g2 += -gammat * (2.*dx*dy)/r2
lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin')
source_cats = [ treecorr.Catalog(x=xs[:,k], y=ys[:,k], g1=g1[:,k], g2=g2[:,k], w=w[:,k],
x_units='arcmin', y_units='arcmin') for k in range(ncats) ]
full_source_cat = treecorr.Catalog(x=xs.flatten(), y=ys.flatten(), w=w.flatten(),
g1=g1.flatten(), g2=g2.flatten(),
x_units='arcmin', y_units='arcmin')
for k in range(ncats):
# These could each be done on different machines in a real world application.
ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=2)
# These should use process_cross, not process, since we don't want to call finalize.
ng.process_cross(lens_cat, source_cats[k])
ng.write(os.path.join('output','ng_piece%d.fits'%k))
full_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=2)
full_ng.process(lens_cat, full_source_cat)
pieces_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin')
for k in range(ncats):
ng = pieces_ng.copy()
ng.read(os.path.join('output','ng_piece%d.fits'%k))
pieces_ng += ng
varg = treecorr.calculateVarG(source_cats)
pieces_ng.finalize(varg)
print 'max error in meanr = ',numpy.max(pieces_ng.meanr - full_ng.meanr),
print ' max meanr = ',numpy.max(full_ng.meanr)
print 'max error in meanlogr = ',numpy.max(pieces_ng.meanlogr - full_ng.meanlogr),
print ' max meanlogr = ',numpy.max(full_ng.meanlogr)
print 'max error in npairs = ',numpy.max(pieces_ng.npairs - full_ng.npairs),
print ' max npairs = ',numpy.max(full_ng.npairs)
print 'max error in weight = ',numpy.max(pieces_ng.weight - full_ng.weight),
print ' max weight = ',numpy.max(full_ng.weight)
print 'max error in xi = ',numpy.max(pieces_ng.xi - full_ng.xi),
print ' max xi = ',numpy.max(full_ng.xi)
print 'max error in xi_im = ',numpy.max(pieces_ng.xi_im - full_ng.xi_im),
print ' max xi_im = ',numpy.max(full_ng.xi_im)
print 'max error in varxi = ',numpy.max(pieces_ng.varxi - full_ng.varxi),
print ' max varxi = ',numpy.max(full_ng.varxi)
numpy.testing.assert_almost_equal(pieces_ng.meanr, full_ng.meanr, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.meanlogr, full_ng.meanlogr, decimal=4)
numpy.testing.assert_almost_equal(pieces_ng.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.weight*1.e-5, full_ng.weight*1.e-5, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.xi*1.e2, full_ng.xi*1.e2, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.xi_im*1.e2, full_ng.xi_im*1.e2, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.varxi*1.e6, full_ng.varxi*1.e6, decimal=3)
def test_pieces():
# Test that we can do the calculation in pieces and recombine the results
import time
ncats = 3
data_cats = []
nlens = 1000
nsource = 30000
gamma0 = 0.05
r0 = 10.
L = 50. * r0
numpy.random.seed(8675309)
xl = (numpy.random.random_sample(nlens)-0.5) * L
yl = (numpy.random.random_sample(nlens)-0.5) * L
xs = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L
ys = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L
g1 = numpy.zeros( (nsource,ncats) )
g2 = numpy.zeros( (nsource,ncats) )
w = numpy.random.random_sample( (nsource,ncats) ) + 0.5
for x,y in zip(xl,yl):
dx = xs-x
dy = ys-y
r2 = dx**2 + dy**2
gammat = gamma0 * numpy.exp(-0.5*r2/r0**2)
g1 += -gammat * (dx**2-dy**2)/r2
g2 += -gammat * (2.*dx*dy)/r2
lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin')
source_cats = [ treecorr.Catalog(x=xs[:,k], y=ys[:,k], g1=g1[:,k], g2=g2[:,k], w=w[:,k],
x_units='arcmin', y_units='arcmin') for k in range(ncats) ]
full_source_cat = treecorr.Catalog(x=xs.flatten(), y=ys.flatten(), w=w.flatten(),
g1=g1.flatten(), g2=g2.flatten(),
x_units='arcmin', y_units='arcmin')
t0 = time.time()
for k in range(ncats):
# These could each be done on different machines in a real world application.
ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=1)
# These should use process_cross, not process, since we don't want to call finalize.
ng.process_cross(lens_cat, source_cats[k])
ng.write(os.path.join('output','ng_piece_%d.fits'%k))
pieces_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin')
for k in range(ncats):
ng = pieces_ng.copy()
ng.read(os.path.join('output','ng_piece_%d.fits'%k))
pieces_ng += ng
varg = treecorr.calculateVarG(source_cats)
pieces_ng.finalize(varg)
t1 = time.time()
print ('time for piece-wise processing (including I/O) = ',t1-t0)
full_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin',
verbose=1)
full_ng.process(lens_cat, full_source_cat)
t2 = time.time()
print ('time for full processing = ',t2-t1)
print('max error in meanr = ',numpy.max(pieces_ng.meanr - full_ng.meanr),)
print(' max meanr = ',numpy.max(full_ng.meanr))
print('max error in meanlogr = ',numpy.max(pieces_ng.meanlogr - full_ng.meanlogr),)
print(' max meanlogr = ',numpy.max(full_ng.meanlogr))
print('max error in npairs = ',numpy.max(pieces_ng.npairs - full_ng.npairs),)
print(' max npairs = ',numpy.max(full_ng.npairs))
print('max error in weight = ',numpy.max(pieces_ng.weight - full_ng.weight),)
print(' max weight = ',numpy.max(full_ng.weight))
print('max error in xi = ',numpy.max(pieces_ng.xi - full_ng.xi),)
print(' max xi = ',numpy.max(full_ng.xi))
print('max error in xi_im = ',numpy.max(pieces_ng.xi_im - full_ng.xi_im),)
print(' max xi_im = ',numpy.max(full_ng.xi_im))
print('max error in varxi = ',numpy.max(pieces_ng.varxi - full_ng.varxi),)
print(' max varxi = ',numpy.max(full_ng.varxi))
numpy.testing.assert_almost_equal(pieces_ng.meanr, full_ng.meanr, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.meanlogr, full_ng.meanlogr, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.weight*1.e-5, full_ng.weight*1.e-5, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.xi*1.e1, full_ng.xi*1.e1, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.xi_im*1.e1, full_ng.xi_im*1.e1, decimal=2)
numpy.testing.assert_almost_equal(pieces_ng.varxi*1.e5, full_ng.varxi*1.e5, decimal=2)
# A different way to do this can produce results that are essentially identical to the
# full calculation. We can use wpos = w, but set w = 0 for the items in the pieces catalogs
# that we don't want to include. This will force the tree to be built identically in each
# case, but only use the subset of items in the calculation. The sum of all these should
# be identical to the full calculation aside from order of calculation differences.
# However, we lose some to speed, since there are a lot more wasted calculations along the
# way that have to be duplicated in each piece.
w2 = [ numpy.empty(w.shape) for k in range(ncats) ]
for k in range(ncats):
w2[k][:,:] = 0.
w2[k][:,k] = w[:,k]
source_cats2 = [ treecorr.Catalog(x=xs.flatten(), y=ys.flatten(),
g1=g1.flatten(), g2=g2.flatten(),
wpos=w.flatten(), w=w2[k].flatten(),
x_units='arcmin', y_units='arcmin') for k in range(ncats) ]
t3 = time.time()
ng2 = [ full_ng.copy() for k in range(ncats) ]
for k in range(ncats):
ng2[k].clear()
ng2[k].process_cross(lens_cat, source_cats2[k])
pieces_ng2 = full_ng.copy()
pieces_ng2.clear()
for k in range(ncats):
pieces_ng2 += ng2[k]
pieces_ng2.finalize(varg)
t4 = time.time()
print ('time for zero-weight piece-wise processing = ',t4-t3)
print('max error in meanr = ',numpy.max(pieces_ng2.meanr - full_ng.meanr),)
print(' max meanr = ',numpy.max(full_ng.meanr))
print('max error in meanlogr = ',numpy.max(pieces_ng2.meanlogr - full_ng.meanlogr),)
print(' max meanlogr = ',numpy.max(full_ng.meanlogr))
print('max error in npairs = ',numpy.max(pieces_ng2.npairs - full_ng.npairs),)
print(' max npairs = ',numpy.max(full_ng.npairs))
print('max error in weight = ',numpy.max(pieces_ng2.weight - full_ng.weight),)
print(' max weight = ',numpy.max(full_ng.weight))
print('max error in xi = ',numpy.max(pieces_ng2.xi - full_ng.xi),)
print(' max xi = ',numpy.max(full_ng.xi))
print('max error in xi_im = ',numpy.max(pieces_ng2.xi_im - full_ng.xi_im),)
print(' max xi_im = ',numpy.max(full_ng.xi_im))
print('max error in varxi = ',numpy.max(pieces_ng2.varxi - full_ng.varxi),)
print(' max varxi = ',numpy.max(full_ng.varxi))
numpy.testing.assert_almost_equal(pieces_ng2.meanr, full_ng.meanr, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.meanlogr, full_ng.meanlogr, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.weight*1.e-5, full_ng.weight*1.e-5, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.xi*1.e1, full_ng.xi*1.e1, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.xi_im*1.e1, full_ng.xi_im*1.e1, decimal=8)
numpy.testing.assert_almost_equal(pieces_ng2.varxi*1.e5, full_ng.varxi*1.e5, decimal=8)
def test_var():
nobj = 5000
np.random.seed(8675309)
# First without weights
cats = []
allg1 = []
allg2 = []
allk = []
for i in range(10):
x = np.random.random_sample(nobj)
y = np.random.random_sample(nobj)
g1 = np.random.random_sample(nobj) - 0.5
g2 = np.random.random_sample(nobj) - 0.5
k = np.random.random_sample(nobj) - 0.5
cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, k=k)
varg = (np.sum(g1**2) + np.sum(g2**2)) / (2.*len(g1))
vark = np.sum(k**2) / len(k)
assert np.isclose(cat.vark, vark)
assert np.isclose(cat.varg, varg)
assert np.isclose(treecorr.calculateVarK(cat), vark)
assert np.isclose(treecorr.calculateVarK([cat]), vark)
assert np.isclose(treecorr.calculateVarG(cat), varg)
assert np.isclose(treecorr.calculateVarG([cat]), varg)
cats.append(cat)
allg1.extend(g1)
allg2.extend(g2)
allk.extend(k)
allg1 = np.array(allg1)
allg2 = np.array(allg2)
allk = np.array(allk)
varg = (np.sum(allg1**2) + np.sum(allg2**2)) / (2. * len(allg1))
vark = np.sum(allk**2) / len(allk)
assert np.isclose(treecorr.calculateVarG(cats), varg)
assert np.isclose(treecorr.calculateVarK(cats), vark)
# Now with weights
cats = []
allg1 = []
allg2 = []
allk = []
allw = []
for i in range(10):
x = np.random.random_sample(nobj)
y = np.random.random_sample(nobj)
w = np.random.random_sample(nobj)
g1 = np.random.random_sample(nobj)
g2 = np.random.random_sample(nobj)
k = np.random.random_sample(nobj)
cat = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, k=k)
varg = np.sum(w**2 * (g1**2 + g2**2)) / np.sum(w) / 2.
vark = np.sum(w**2 * k**2) / np.sum(w)
assert np.isclose(cat.varg, varg)
assert np.isclose(cat.vark, vark)
assert np.isclose(treecorr.calculateVarG(cat), varg)
assert np.isclose(treecorr.calculateVarG([cat]), varg)
assert np.isclose(treecorr.calculateVarK(cat), vark)
assert np.isclose(treecorr.calculateVarK([cat]), vark)
cats.append(cat)
allg1.extend(g1)
allg2.extend(g2)
allk.extend(k)
allw.extend(w)
allg1 = np.array(allg1)
allg2 = np.array(allg2)
allk = np.array(allk)
allw = np.array(allw)
varg = np.sum(allw**2 * (allg1**2 + allg2**2)) / np.sum(allw) / 2.
vark = np.sum(allw**2 * allk**2) / np.sum(allw)
assert np.isclose(treecorr.calculateVarG(cats), varg)
assert np.isclose(treecorr.calculateVarK(cats), vark)
