def shear_shear_corr(pos1,pos2,shear1,shear2,k1=None,k2=None,w1=None,w2=None,same_zshell=False,same_cell=False,unique_encounter=False,num_threads=0): nbins = 6 min_sep = 0.05 # 3 arcmin max_sep = 3.0 # 180 arcmin bin_size = (max_sep-min_sep)/nbins # roughly bin_slop = 0.05/bin_size # 0.1 -> 0.05 # 2pt_pipeline for des used bin_slop: 0.01 here: https://github.com/des-science/2pt_pipeline/blob/master/pipeline/twopt_pipeline.yaml # num_threads = 5 #None #0 logger = None if same_zshell and same_cell: # auto ra, dec = pos1 # either 1 or 2 works g1, g2 = shear1 k = k1 w = w1 cat = treecorr.Catalog(g1=g1, g2=g2, k=k, ra=ra, dec=dec, w=w, ra_units='degrees', dec_units='degrees') elif same_zshell: # just wanted to distrubute the workload fairly for two encounters (didn't want to make one of the cores idle) ra1, dec1 = np.array_split(pos1[0], 2), np.array_split(pos1[1], 2) # split in half ra2, dec2 = np.array_split(pos2[0], 2), np.array_split(pos2[1], 2) g1_1st, g2_1st = np.array_split(shear1[0], 2), np.array_split(shear1[1], 2) g1_2nd, g2_2nd = np.array_split(shear2[0], 2), np.array_split(shear2[1], 2) k1 = np.array_split(k1, 2) if (k1 is not None) else [None,None] k2 = np.array_split(k2, 2) if (k2 is not None) else [None,None] w1 = np.array_split(w1, 2) if (w1 is not None) else [None,None] w2 = np.array_split(w2, 2) if (w2 is not None) else [None,None] cat1 = [treecorr.Catalog(g1=g1_1st[h], g2=g2_1st[h], k=k1[h], ra=ra1[h], dec=dec1[h], w=w1[h], ra_units='degrees', dec_units='degrees') for h in [0,1]] cat2 = [treecorr.Catalog(g1=g1_2nd[h], g2=g2_2nd[h], k=k2[h], ra=ra2[h], dec=dec2[h], w=w2[h], ra_units='degrees', dec_units='degrees') for h in [0,1]] else: ra1, dec1 = pos1 ra2, dec2 = pos2 g1_1st, g2_1st = shear1 g1_2nd, g2_2nd = shear2 cat1 = treecorr.Catalog(g1=g1_1st, g2=g2_1st, k=k1, ra=ra1, dec=dec1, w=w1, ra_units='degrees', dec_units='degrees') cat2 = treecorr.Catalog(g1=g1_2nd, g2=g2_2nd, k=k2, ra=ra2, dec=dec2, w=w2, ra_units='degrees', dec_units='degrees') gg = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_slop=bin_slop, sep_units='degrees', logger=logger) if same_zshell and same_cell: gg.process_auto(cat,num_threads=num_threads) elif same_zshell: # just wanted to distrubute the workload fairly for two encounters (didn't want to make one of the cores idle) if unique_encounter: # the following two counts shouldn't be doubled up cuz they're the same in both directions gg.process_cross(cat1[0],cat2[0],num_threads=num_threads) gg.process_cross(cat1[1],cat2[1],num_threads=num_threads) else: # in the other encounter cat1 and cat2 are switched but does not matter anyway gg.process_cross(cat2[0],cat1[1],num_threads=num_threads) gg.process_cross(cat2[1],cat1[0],num_threads=num_threads) else: gg.process_cross(cat1,cat2,num_threads=num_threads) if same_zshell and same_cell: varg1 = treecorr.calculateVarG(cat) varg2 = varg1 elif same_cell: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) else: varg1 = np.nan varg2 = np.nan return gg, varg1, varg2
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None): """Accumulate the number of triangles of points between cat1, cat2, and cat3. - If only 1 argument is given, then compute an auto-correlation function. - If 2 arguments are given, then compute a cross-correlation function with the first catalog taking two corners of the triangles. (Not implemented yet.) - If 3 arguments are given, then compute a cross-correlation function. All arguments may be lists, in which case all items in the list are used for that element of the correlation. Note: For a correlation of multiple catalogs, it matters which corner of the triangle comes from which catalog. The final accumulation will have d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate all the possible triangles between three catalogs, you should call this multiple times with the different catalogs in different positions. Parameters: cat1 (Catalog): A catalog or list of catalogs for the first N field. cat2 (Catalog): A catalog or list of catalogs for the second N field, if any. (default: None) cat3 (Catalog): A catalog or list of catalogs for the third N field, if any. (default: None) metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2] if cat3 is not None and not isinstance(cat3, list): cat3 = [cat3] if cat2 is None and cat3 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 varg3 = varg1 self.logger.info("varg = %f: sig_g = %f", varg1, math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) elif (cat2 is None) != (cat3 is None): raise NotImplementedError("No partial cross GGG yet.") else: assert cat2 is not None and cat3 is not None varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) varg3 = treecorr.calculateVarG(cat3) self.logger.info("varg1 = %f: sig_g = %f", varg1, math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_g = %f", varg2, math.sqrt(varg2)) self.logger.info("varg3 = %f: sig_g = %f", varg3, math.sqrt(varg3)) self._process_all_cross(cat1, cat2, cat3, metric, num_threads) self.finalize(varg1, varg2, varg3)
def process(self, cat1, cat2=None, metric=None, num_threads=None): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the first G field. :param cat2: A catalog or list of catalogs for the second G field, if any. (default: None) :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2] if len(cat1) == 0: raise AttributeError("No catalogs provided for cat1") if metric is None: metric = treecorr.config.get(self.config, 'metric', str, 'Euclidean') if metric not in ['Euclidean', 'Rperp']: raise ValueError("Invalid metric.") if cat2 is None or len(cat2) == 0: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2, math.sqrt(varg2)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(varg1, varg2)
def process(self, cat1, cat2=None, metric=None, num_threads=None, comm=None, low_mem=False): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. Parameters: cat1 (Catalog): A catalog or list of catalogs for the first G field. cat2 (Catalog): A catalog or list of catalogs for the second G field, if any. (default: None) metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) comm (mpi4py.Comm): If running MPI, an mpi4py Comm object to communicate between processes. If used, the rank=0 process will have the final computation. This only works if using patches. (default: None) low_mem (bool): Whether to sacrifice a little speed to try to reduce memory usage. This only works if using patches. (default: False) """ import math self.clear() if not isinstance(cat1, list): self.npatch1 = cat1._npatch cat1 = cat1.get_patches(low_mem=low_mem) if cat2 is None: self.npatch2 = self.npatch1 if cat2 is not None and not isinstance(cat2, list): self.npatch2 = cat2._npatch cat2 = cat2.get_patches(low_mem=low_mem) if cat2 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads, comm, low_mem) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2, math.sqrt(varg2)) self._process_all_cross(cat1, cat2, metric, num_threads, comm, low_mem) self.finalize(varg1, varg2)
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None): """Accumulate the number of triangles of points between cat1, cat2, and cat3. - If only 1 argument is given, then compute an auto-correlation function. - If 2 arguments are given, then compute a cross-correlation function with the first catalog taking two corners of the triangles. (Not implemented yet.) - If 3 arguments are given, then compute a cross-correlation function. All arguments may be lists, in which case all items in the list are used for that element of the correlation. Note: For a correlation of multiple catalogs, it matters which corner of the triangle comes from which catalog. The final accumulation will have d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate all the possible triangles between three catalogs, you should call this multiple times with the different catalogs in different positions. Parameters: cat1 (Catalog): A catalog or list of catalogs for the first N field. cat2 (Catalog): A catalog or list of catalogs for the second N field, if any. (default: None) cat3 (Catalog): A catalog or list of catalogs for the third N field, if any. (default: None) metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2] if cat3 is not None and not isinstance(cat3,list): cat3 = [cat3] if cat2 is None and cat3 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 varg3 = varg1 self.logger.info("varg = %f: sig_g = %f",varg1,math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) elif (cat2 is None) != (cat3 is None): raise NotImplementedError("No partial cross GGG yet.") else: assert cat2 is not None and cat3 is not None varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) varg3 = treecorr.calculateVarG(cat3) self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2)) self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3)) self._process_all_cross(cat1,cat2,cat3, metric, num_threads) self.finalize(varg1,varg2,varg3)
def process(self, cat1, cat2=None, metric=None, num_threads=None): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the first G field. :param cat2: A catalog or list of catalogs for the second G field, if any. (default: None) :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2] if len(cat1) == 0: raise AttributeError("No catalogs provided for cat1") if metric is None: metric = treecorr.config.get(self.config,'metric',str,'Euclidean') if metric not in ['Euclidean', 'Rperp']: raise ValueError("Invalid metric.") if cat2 is None or len(cat2) == 0: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2)) self._process_all_cross(cat1,cat2, metric, num_threads) self.finalize(varg1,varg2)
def process(self, cat1, cat2, metric=None, num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. Parameters: cat1 (Catalog): A catalog or list of catalogs for the K field. cat2 (Catalog): A catalog or list of catalogs for the G field. metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if not isinstance(cat2,list): cat2 = [cat2] vark = treecorr.calculateVarK(cat1) varg = treecorr.calculateVarG(cat2) self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark)) self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg)) self._process_all_cross(cat1,cat2,metric,num_threads) self.finalize(vark,varg)
def process(self, cat1, cat2, metric=None, num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. Parameters: cat1 (Catalog): A catalog or list of catalogs for the K field. cat2 (Catalog): A catalog or list of catalogs for the G field. metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if not isinstance(cat2, list): cat2 = [cat2] vark = treecorr.calculateVarK(cat1) varg = treecorr.calculateVarG(cat2) self.logger.info("vark = %f: sig_k = %f", vark, math.sqrt(vark)) self.logger.info("varg = %f: sig_sn (per component) = %f", varg, math.sqrt(varg)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(vark, varg)
def process(self, cat1, cat2, metric=None, num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the N field. :param cat2: A catalog or list of catalogs for the G field. :param metric: Which metric to use for distance measurements. Options are given in the doc string of :class:`~treecorr.BinnedCorr2`. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if not isinstance(cat2, list): cat2 = [cat2] varg = treecorr.calculateVarG(cat2) self.logger.info("varg = %f: sig_sn (per component) = %f", varg, math.sqrt(varg)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(varg)
def pos_shear_corr(pos_lens,pos_source,shear_source,k_source=None,w_lense=None,w_source=None,same_cell=False,num_threads=0): nbins = 6 min_sep = 0.05 # 3 arcmin max_sep = 3.0 # 180 arcmin bin_size = (max_sep-min_sep)/nbins # roughly bin_slop = 0.05/bin_size # 0.1 -> 0.05 # 2pt_pipeline for des used bin_slop: 0.01 here: https://github.com/des-science/2pt_pipeline/blob/master/pipeline/twopt_pipeline.yaml logger = None ra_lens, dec_lens = pos_lens ra_source, dec_source = pos_source g1_source, g2_source = shear_source # foreground (lens) cat_lens = treecorr.Catalog(ra=ra_lens, dec=dec_lens, w=w_lense, ra_units='degrees', dec_units='degrees') # background (source) cat_source = treecorr.Catalog(ra=ra_source, dec=dec_source, w=w_source, g1=g1_source, g2=g2_source, k=k_source, ra_units='degrees', dec_units='degrees') ng = treecorr.NGCorrelation(min_sep=min_sep, max_sep=max_sep, nbins=nbins, bin_slop=bin_slop, sep_units='degrees', logger=logger) ng.process_cross(cat_lens,cat_source,num_threads=num_threads) # there's no process_auto for this object # one shear variance per pixel per source zbin varg = treecorr.calculateVarG(cat_source) if same_cell else np.nan return ng, varg
def process(self, cat1, cat2=None, metric='Euclidean', num_threads=None): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the first G field. :param cat2: A catalog or list of catalogs for the second G field, if any. (default: None) :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth r1,r2, if d is the normal Euclidean distance and Rparallel = |r1 - r2|, then Rperp^2 = d^2 - Rparallel^2. (default: 'Euclidean') """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2] if len(cat1) == 0: raise AttributeError("No catalogs provided for cat1") if metric not in ['Euclidean', 'Rperp']: raise ValueError("Invalid metric.") if cat2 is None or len(cat2) == 0: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2)) self._process_all_cross(cat1,cat2, metric, num_threads) self.finalize(varg1,varg2)
def process(self, cat1, cat2=None, metric=None, num_threads=None): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. Parameters: cat1 (Catalog): A catalog or list of catalogs for the first G field. cat2 (Catalog): A catalog or list of catalogs for the second G field, if any. (default: None) metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2] if cat2 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f", varg1, math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f", varg2, math.sqrt(varg2)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(varg1, varg2)
def process(self, cat1, cat2=None, metric=None, num_threads=None): """Compute the correlation function. If only 1 argument is given, then compute an auto-correlation function. If 2 arguments are given, then compute a cross-correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. Parameters: cat1 (Catalog): A catalog or list of catalogs for the first G field. cat2 (Catalog): A catalog or list of catalogs for the second G field, if any. (default: None) metric (str): Which metric to use. See `Metrics` for details. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) num_threads (int): How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2] if cat2 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 self.logger.info("varg = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: varg1 = treecorr.calculateVarG(cat1) varg2 = treecorr.calculateVarG(cat2) self.logger.info("varg1 = %f: sig_sn (per component) = %f",varg1,math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_sn (per component) = %f",varg2,math.sqrt(varg2)) self._process_all_cross(cat1,cat2, metric, num_threads) self.finalize(varg1,varg2)
def process(self, cat1, cat2, metric=None, num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the K field. :param cat2: A catalog or list of catalogs for the G field. :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. - 'Rlens' = the projected distance perpendicular to the first point in the pair (taken to be a lens) to the line of sight to the second point (e.g. a lensed source galaxy). - 'Arc' = the true great circle distance for spherical coordinates. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if not isinstance(cat2, list): cat2 = [cat2] if len(cat1) == 0: raise ValueError("No catalogs provided for cat1") if len(cat2) == 0: raise ValueError("No catalogs provided for cat2") vark = treecorr.calculateVarK(cat1) varg = treecorr.calculateVarG(cat2) self.logger.info("vark = %f: sig_k = %f", vark, math.sqrt(vark)) self.logger.info("varg = %f: sig_sn (per component) = %f", varg, math.sqrt(varg)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(vark, varg)
def process(self, cat1, cat2, metric=None, num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the K field. :param cat2: A catalog or list of catalogs for the G field. :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. - 'Rlens' = the projected distance perpendicular to the first point in the pair (taken to be a lens) to the line of sight to the second point (e.g. a lensed source galaxy). - 'Arc' = the true great circle distance for spherical coordinates. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if not isinstance(cat2,list): cat2 = [cat2] if len(cat1) == 0: raise ValueError("No catalogs provided for cat1") if len(cat2) == 0: raise ValueError("No catalogs provided for cat2") vark = treecorr.calculateVarK(cat1) varg = treecorr.calculateVarG(cat2) self.logger.info("vark = %f: sig_k = %f",vark,math.sqrt(vark)) self.logger.info("varg = %f: sig_sn (per component) = %f",varg,math.sqrt(varg)) self._process_all_cross(cat1,cat2,metric,num_threads) self.finalize(vark,varg)
def process(self, cat1, cat2, metric="Euclidean", num_threads=None): """Compute the correlation function. Both arguments may be lists, in which case all items in the list are used for that element of the correlation. :param cat1: A catalog or list of catalogs for the N field. :param cat2: A catalog or list of catalogs for the G field. :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth r1,r2, if d is the normal Euclidean distance and Rparallel = |r1 - r2|, then Rperp^2 = d^2 - Rparallel^2. (default: 'Euclidean') :param num_threads: How many OpenMP threads to use during the calculation. (default: None, which means to first check for a num_threads parameter in self.config, then default to querying the number of cpu cores and try to use that many threads.) Note that this won't work if the system's C compiler is clang, such as on MacOS systems. """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if not isinstance(cat2, list): cat2 = [cat2] if len(cat1) == 0: raise ValueError("No catalogs provided for cat1") if len(cat2) == 0: raise ValueError("No catalogs provided for cat2") varg = treecorr.calculateVarG(cat2) self.logger.info("varg = %f: sig_sn (per component) = %f", varg, math.sqrt(varg)) self._process_all_cross(cat1, cat2, metric, num_threads) self.finalize(varg)
def test_var(): nobj = 5000 rng = np.random.RandomState(8675309) # First without weights cats = [] allg1 = [] allg2 = [] allk = [] for i in range(10): x = rng.random_sample(nobj) y = rng.random_sample(nobj) g1 = rng.random_sample(nobj) - 0.5 g2 = rng.random_sample(nobj) - 0.5 k = rng.random_sample(nobj) - 0.5 cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, k=k) varg = (np.sum(g1**2) + np.sum(g2**2)) / (2.*len(g1)) vark = np.sum(k**2) / len(k) assert np.isclose(cat.vark, vark) assert np.isclose(cat.varg, varg) assert np.isclose(treecorr.calculateVarK(cat), vark) assert np.isclose(treecorr.calculateVarK([cat]), vark) assert np.isclose(treecorr.calculateVarG(cat), varg) assert np.isclose(treecorr.calculateVarG([cat]), varg) cats.append(cat) allg1.extend(g1) allg2.extend(g2) allk.extend(k) allg1 = np.array(allg1) allg2 = np.array(allg2) allk = np.array(allk) varg = (np.sum(allg1**2) + np.sum(allg2**2)) / (2. * len(allg1)) vark = np.sum(allk**2) / len(allk) assert np.isclose(treecorr.calculateVarG(cats), varg) assert np.isclose(treecorr.calculateVarK(cats), vark) # Now with weights cats = [] allg1 = [] allg2 = [] allk = [] allw = [] for i in range(10): x = rng.random_sample(nobj) y = rng.random_sample(nobj) w = rng.random_sample(nobj) g1 = rng.random_sample(nobj) g2 = rng.random_sample(nobj) k = rng.random_sample(nobj) cat = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, k=k) varg = np.sum(w**2 * (g1**2 + g2**2)) / np.sum(w) / 2. vark = np.sum(w**2 * k**2) / np.sum(w) assert np.isclose(cat.varg, varg) assert np.isclose(cat.vark, vark) assert np.isclose(treecorr.calculateVarG(cat), varg) assert np.isclose(treecorr.calculateVarG([cat]), varg) assert np.isclose(treecorr.calculateVarK(cat), vark) assert np.isclose(treecorr.calculateVarK([cat]), vark) cats.append(cat) allg1.extend(g1) allg2.extend(g2) allk.extend(k) allw.extend(w) allg1 = np.array(allg1) allg2 = np.array(allg2) allk = np.array(allk) allw = np.array(allw) varg = np.sum(allw**2 * (allg1**2 + allg2**2)) / np.sum(allw) / 2. vark = np.sum(allw**2 * allk**2) / np.sum(allw) assert np.isclose(treecorr.calculateVarG(cats), varg) assert np.isclose(treecorr.calculateVarK(cats), vark)
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None): """Accumulate the number of triangles of points between cat1, cat2, and cat3. - If only 1 argument is given, then compute an auto-correlation function. - If 2 arguments are given, then compute a cross-correlation function with the first catalog taking two corners of the triangles. (Not implemented yet.) - If 3 arguments are given, then compute a cross-correlation function. All arguments may be lists, in which case all items in the list are used for that element of the correlation. Note: For a correlation of multiple catalogs, it matters which corner of the triangle comes from which catalog. The final accumulation will have d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate all the possible triangles between three catalogs, you should call this multiple times with the different catalogs in different positions. :param cat1: A catalog or list of catalogs for the first N field. :param cat2: A catalog or list of catalogs for the second N field, if any. (default: None) :param cat3: A catalog or list of catalogs for the third N field, if any. (default: None) :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1,list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2,list): cat2 = [cat2] if cat3 is not None and not isinstance(cat3,list): cat3 = [cat3] if len(cat1) == 0: raise ValueError("No catalogs provided for cat1") if cat2 is not None and len(cat2) == 0: cat2 = None if cat3 is not None and len(cat3) == 0: cat3 = None if cat2 is None and cat3 is not None: raise NotImplemented("No partial cross GGG yet.") if cat3 is None and cat2 is not None: raise NotImplemented("No partial cross GGG yet.") if cat2 is None and cat3 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 varg3 = varg1 self.logger.info("varg = %f: sig_g = %f",varg1,math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: assert cat2 is not None and cat3 is not None varg1 = treecorr.calculateVarK(cat1) varg2 = treecorr.calculateVarK(cat2) varg3 = treecorr.calculateVarK(cat3) self.logger.info("varg1 = %f: sig_g = %f",varg1,math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_g = %f",varg2,math.sqrt(varg2)) self.logger.info("varg3 = %f: sig_g = %f",varg3,math.sqrt(varg3)) self._process_all_cross(cat1,cat2,cat3, metric, num_threads) self.finalize(varg1,varg2,varg3)
def compute_w(dataf, randf, config, estimator='PW1', compensated=1, nbins_rpar=30, random_oversampling=10., verbosity=1, largePi=0, **kwargs): """ dataf: paths to galaxy samples randf: paths to random points corresponding to galaxy samples config: path to config file, or dict specifying file types; column names/numbers, etc. a la TreeCorr configuration estimator: 'PW1', 'PW2', 'AS' or 'wgg' to specify correlation & estimator nbins_rpar: number of line-of-sight bins for 3D correlation function -- specify limits in config arg """ assert estimator in ['PW1', 'PW2', 'AS', 'wgg'], "for IA: estimator must be 'PW1/2' (pair_weighted 1=RDs, 2=RRs norm) or 'AS' (average shear), for clustering: 'wgg'" assert hasattr(dataf, '__iter__'), "dataf must be list/tuple of 2x paths; density, shapes for IA, or density1, density2 for clustering (these can be the same!)" assert hasattr(randf, '__iter__'), "randf must be list/tuple of 2x paths for randoms corresponding to each of dataf (these can be the same!)" nbins_rpar = int(nbins_rpar) random_oversampling = float(random_oversampling) verbosity = int(verbosity) largePi = int(largePi) if type(config) == str: config = treecorr.read_config(config) if config['file_type'] == 'ASCII': config['ra_col'] = int(config['ra_col']) config['dec_col'] = int(config['dec_col']) config['r_col'] = int(config['r_col']) config['g1_col'] = int(config['g1_col']) config['g2_col'] = int(config['g2_col']) config['verbose'] = verbosity if not largePi: Pi = np.linspace(config['min_rpar'], config['max_rpar'], nbins_rpar + 1) else: dPi = (config['max_rpar'] - config['min_rpar']) / nbins_rpar Pi = np.arange(config['min_rpar']*1.5, config['max_rpar']*1.5 + dPi, step=dPi, dtype=float) if estimator in ['PW1', 'PW2', 'AS']: corr = 'ng' gt_3D = np.zeros([len(Pi)-1, config['nbins']]) gx_3D = np.zeros([len(Pi)-1, config['nbins']]) varg_3D = np.zeros([len(Pi)-1, config['nbins']]) npair_3D = np.zeros([len(Pi)-1, config['nbins']]) elif estimator == 'wgg': corr = 'nn' wgg_3D = np.zeros([len(Pi)-1, config['nbins']]) else: raise ValueError, "unsupported estimator choice" config_r = config.copy() config_r['flip_g1'] = False config_r['flip_g2'] = False data1 = treecorr.Catalog(dataf[0], config) # 1 = density/lenses data2 = treecorr.Catalog(dataf[1], config) # 2 = shapes rand1 = treecorr.Catalog(randf[0], config_r, is_rand=1) rand2 = treecorr.Catalog(randf[1], config_r, is_rand=1) f1 = data1.ntot * random_oversampling / float(rand1.ntot) f2 = data2.ntot * random_oversampling / float(rand2.ntot) rand1.w = np.array(np.random.rand(rand1.ntot) < f1, dtype=float) rand2.w = np.array(np.random.rand(rand2.ntot) < f2, dtype=float) varg = treecorr.calculateVarG(data2) for p in tqdm(range(len(Pi)-1), ascii=True, desc='Correlating'): if largePi & any(abs(Pi[p:p+2]) < config['max_rpar']): continue conf_pi = config.copy() conf_pi['min_rpar'] = Pi[p] conf_pi['max_rpar'] = Pi[p+1] if corr == 'ng': ng = treecorr.NGCorrelation(conf_pi) rg = treecorr.NGCorrelation(conf_pi) ng.process_cross(data1, data2) rg.process_cross(rand1, data2) if estimator == 'PW1': # RDs norm f = data1.ntot / rand1.w.sum() norm1 = rg.weight * f norm2 = rg.weight if estimator == 'PW2': # RRs norm RRs = get_RRs(rand1, rand2, conf_pi, **kwargs) f1 = data1.ntot / rand1.w.sum() f2 = data2.ntot / rand2.w.sum() norm1 = RRs * f1 * f2 norm2 = RRs * f2 elif estimator == 'AS': # DDs norm norm1 = ng.weight norm2 = rg.weight if int(compensated): gt_3D[p] += (ng.xi / norm1) - (rg.xi / norm2) gx_3D[p] += (ng.xi_im / norm1) - (rg.xi_im / norm2) varg_3D[p] += (varg / norm1) + (varg / norm2) npair_3D[p] += ng.npairs else: gt_3D[p] += ng.xi / norm1 gx_3D[p] += ng.xi_im / norm1 varg_3D[p] += varg / norm1 npair_3D[p] += ng.npairs elif corr == 'nn': nn = treecorr.NNCorrelation(conf_pi) rr = treecorr.NNCorrelation(conf_pi) nr = treecorr.NNCorrelation(conf_pi) rn = treecorr.NNCorrelation(conf_pi) if dataf[0] == dataf[1]: nn.process(data1) rr.process(rand1) nr.process(data1, rand1) xi, varxi = nn.calculateXi(rr, nr) else: nn.process(data1, data2) rr.process(rand1, rand2) nr.process(data1, rand2) rn.process(rand1, data2) xi, varxi = nn.calculateXi(rr, nr, rn) wgg_3D[p] += xi if corr == 'ng': #gt = np.trapz(gt_3D, x=midpoints(Pi), axis=0) #gx = np.trapz(gx_3D, x=midpoints(Pi), axis=0) #gt = simps(gt_3D, x=midpoints(Pi), axis=0) gt = np.sum(gt_3D * (Pi[1] - Pi[0]), axis=0) gx = np.sum(gx_3D * (Pi[1] - Pi[0]), axis=0) varg = np.sum(varg_3D, axis=0) npair = np.sum(npair_3D, axis=0) r = ng.rnom return r, gt, gx, varg**0.5, npair elif corr == 'nn': wgg = np.sum(wgg_3D * (Pi[1] - Pi[0]), axis=0) r = nn.rnom return r, wgg
def process(self, cat1, cat2=None, cat3=None, metric=None, num_threads=None): """Accumulate the number of triangles of points between cat1, cat2, and cat3. - If only 1 argument is given, then compute an auto-correlation function. - If 2 arguments are given, then compute a cross-correlation function with the first catalog taking two corners of the triangles. (Not implemented yet.) - If 3 arguments are given, then compute a cross-correlation function. All arguments may be lists, in which case all items in the list are used for that element of the correlation. Note: For a correlation of multiple catalogs, it matters which corner of the triangle comes from which catalog. The final accumulation will have d1 > d2 > d3 where d1 is between two points in cat2,cat3; d2 is between points in cat1,cat3; and d3 is between points in cat1,cat2. To accumulate all the possible triangles between three catalogs, you should call this multiple times with the different catalogs in different positions. :param cat1: A catalog or list of catalogs for the first N field. :param cat2: A catalog or list of catalogs for the second N field, if any. (default: None) :param cat3: A catalog or list of catalogs for the third N field, if any. (default: None) :param metric: Which metric to use for distance measurements. Options are: - 'Euclidean' = straight line Euclidean distance between two points. For spherical coordinates (ra,dec without r), this is the chord distance between points on the unit sphere. - 'Rperp' = the perpendicular component of the distance. For two points with distance from Earth `r1, r2`, if `d` is the normal Euclidean distance and :math:`Rparallel = |r1-r2|`, then we define :math:`Rperp^2 = d^2 - Rparallel^2`. (default: 'Euclidean'; this value can also be given in the constructor in the config dict.) :param num_threads: How many OpenMP threads to use during the calculation. (default: use the number of cpu cores; this value can also be given in the constructor in the config dict.) Note that this won't work if the system's C compiler is clang prior to version 3.7. """ import math self.clear() if not isinstance(cat1, list): cat1 = [cat1] if cat2 is not None and not isinstance(cat2, list): cat2 = [cat2] if cat3 is not None and not isinstance(cat3, list): cat3 = [cat3] if len(cat1) == 0: raise ValueError("No catalogs provided for cat1") if cat2 is not None and len(cat2) == 0: cat2 = None if cat3 is not None and len(cat3) == 0: cat3 = None if cat2 is None and cat3 is not None: raise NotImplemented("No partial cross GGG yet.") if cat3 is None and cat2 is not None: raise NotImplemented("No partial cross GGG yet.") if cat2 is None and cat3 is None: varg1 = treecorr.calculateVarG(cat1) varg2 = varg1 varg3 = varg1 self.logger.info("varg = %f: sig_g = %f", varg1, math.sqrt(varg1)) self._process_all_auto(cat1, metric, num_threads) else: assert cat2 is not None and cat3 is not None varg1 = treecorr.calculateVarK(cat1) varg2 = treecorr.calculateVarK(cat2) varg3 = treecorr.calculateVarK(cat3) self.logger.info("varg1 = %f: sig_g = %f", varg1, math.sqrt(varg1)) self.logger.info("varg2 = %f: sig_g = %f", varg2, math.sqrt(varg2)) self.logger.info("varg3 = %f: sig_g = %f", varg3, math.sqrt(varg3)) self._process_all_cross(cat1, cat2, cat3, metric, num_threads) self.finalize(varg1, varg2, varg3)
def measure_cross_rho(tile_data, max_sep, tags=None, prefix='piff'): """Compute the rho statistics """ import treecorr ntilings = len(tile_data) print('len(tile_data) = ', ntilings) de1 = [d['obs_e1'] - d[prefix + '_e1'] for d in tile_data] de2 = [d['obs_e2'] - d[prefix + '_e2'] for d in tile_data] dt = [(d['obs_T'] - d[prefix + '_T']) / d['obs_T'] for d in tile_data] for k in range(len(tile_data)): print('k = ', k) print('mean de = ', np.mean(de1[k]), np.mean(de2[k])) print('mean dt = ', np.mean(dt[k])) ecats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=d['obs_e1'], g2=d['obs_e2']) for d in tile_data ] for cat in ecats: cat.name = 'ecat' decats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=de1[k], g2=de2[k]) for k, d in enumerate(tile_data) ] for cat in decats: cat.name = 'decat' dtcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', k=dt[k], g1=d['obs_e1'] * dt[k], g2=d['obs_e2'] * dt[k]) for k, d in enumerate(tile_data) ] for cat in dtcats: cat.name = 'dtcat' if tags is not None: for catlist in [ecats, decats, dtcats]: for cat, tag in zip(catlist, tags): cat.name = tag + ":" + cat.name bin_config = dict( sep_units='arcmin', bin_slop=0.1, min_sep=0.5, max_sep=max_sep, bin_size=0.2, #min_sep = 2.5, #max_sep = 250, #nbins = 20, ) results = [] for (catlist1, catlist2) in [(decats, decats), (ecats, decats), (dtcats, dtcats), (decats, dtcats), (ecats, dtcats)]: catnames1 = [cat.name for cat in catlist1] catnames2 = [cat.name for cat in catlist2] print('Doing correlation of %s vs %s' % (catnames1, catnames2)) rho = treecorr.GGCorrelation(bin_config, verbose=2) # Avoid all auto correlations: for i in range(ntilings): for j in range(ntilings): if i == j: continue if catlist1 is catlist2 and i > j: continue print('names: ', catlist1[i].name, catlist2[j].name) rho.process_cross(catlist1[i], catlist2[j]) varg1 = treecorr.calculateVarG(catlist1) varg2 = treecorr.calculateVarG(catlist2) rho.finalize(varg1, varg2) results.append(rho) return results
def measure_cross_rho(tile_data, max_sep, tags=None, prefix='piff', lucas=False): """Compute the rho statistics """ import treecorr ntilings = len(tile_data) print('len(tile_data) = ',ntilings) q1 = [ d['obs_e1']-d[prefix+'_e1'] for d in tile_data ] q2 = [ d['obs_e2']-d[prefix+'_e2'] for d in tile_data ] dt = [ (d['obs_T']-d[prefix+'_T'])/d['obs_T'] for d in tile_data ] for k in range(len(tile_data)): print('k = ',k) print('mean q = ',np.mean(q1[k]),np.mean(q2[k])) print('mean dt = ',np.mean(dt[k])) ecats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=d['obs_e1'], g2=d['obs_e2']) for d in tile_data ] for cat in ecats: cat.name = 'ecat' qcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=q1[k], g2=q2[k]) for k,d in enumerate(tile_data) ] for cat in qcats: cat.name = 'qcat' dtcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', k=dt[k], g1=d['obs_e1']*dt[k], g2=d['obs_e2']*dt[k]) for k,d in enumerate(tile_data) ] for cat in dtcats: cat.name = 'dtcat' if tags is not None: for catlist in [ ecats, qcats, dtcats]: for cat, tag in zip(catlist, tags): cat.name = tag + ":" + cat.name bin_config = dict( sep_units = 'arcmin', bin_slop = 0.1, min_sep = 0.5, max_sep = max_sep, bin_size = 0.2, ) if lucas: bin_config['min_sep'] = 2.5 bin_config['max_sep'] = 250. bin_config['nbins'] = 20 del bin_config['bin_size'] results = [] for (catlist1, catlist2) in [ (qcats, qcats), (ecats, qcats), (dtcats, dtcats), (qcats, dtcats), (ecats, dtcats) ]: catnames1 = [ cat.name for cat in catlist1 ] catnames2 = [ cat.name for cat in catlist2 ] print('Doing correlation of %s vs %s'%(catnames1, catnames2)) rho = treecorr.GGCorrelation(bin_config, verbose=2) # Avoid all auto correlations: for i in range(ntilings): for j in range(ntilings): if i == j: continue if catlist1 is catlist2 and i > j: continue print('names: ',catlist1[i].name,catlist2[j].name) rho.process_cross(catlist1[i], catlist2[j]) varg1 = treecorr.calculateVarG(catlist1) varg2 = treecorr.calculateVarG(catlist2) rho.finalize(varg1, varg2) results.append(rho) return results
def main(): args = parse_args() # Make the work directory if it does not exist yet. cat_dir = os.path.expanduser(args.psf_cats) print 'psfcats dir = ',cat_dir print os.path.join(cat_dir, '*_psf.fits') cat_files = sorted(glob.glob(os.path.join(cat_dir, '*_psf.fits'))) min_sep = 0.5 max_sep = 20 bin_size = 0.3 bin_slop = 0.1 rho1 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=0) rho2 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=0) rho3 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=0) rho4 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=0) rho5 = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=0) all_ra = [] all_dec = [] all_e1 = [] all_e2 = [] all_s = [] for file in cat_files: print 'Processing ',file try: with pyfits.open(file) as pyf: data = pyf[1].data.copy() except: print 'Failed to read data from %s. Skiping.'%file continue #print 'len(data) = ',len(data) #print 'shape = ',data.shape mask = data['flag'] == 0 #print 'len(mask) = ',len(mask) #print 'nonzero(mask) = ',numpy.sum(mask) if mask.sum() == 0: print ' All objects in this file are flagged.' continue ra = data['ra'][mask] dec = data['dec'][mask] e1 = data['e1'][mask] e2 = data['e2'][mask] s = data['size'][mask] m_e1 = data['psf_e1'][mask] m_e2 = data['psf_e2'][mask] m_s = data['psf_size'][mask] all_ra.append(ra) all_dec.append(dec) all_e1.append(e1) all_e2.append(e2) all_s.append(s) de1 = e1-m_e1 de2 = e2-m_e2 dt = (s**2-m_s**2)/s**2 #print 'e1 = ',e1 #print 'e2 = ',e2 #print 'de1 = ',de1 #print 'de2 = ',de2 #print 'dt = ',dt ecat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=e1, g2=e2) decat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=de1, g2=de2) dtcat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', k=dt, g1=dt*e1, g2=dt*e2) ecat.name = 'ecat' decat.name = 'decat' dtcat.name = 'dtcat' for (cat1, cat2, rho) in [ (decat, decat, rho1), (ecat, decat, rho2), (dtcat, dtcat, rho3), (decat, dtcat, rho4), (ecat, dtcat, rho5) ]: #print 'Doing correlation of %s vs %s'%(cat1.name, cat2.name) if cat1 is cat2: rho.process_auto(cat1) else: rho.process_cross(cat1, cat2) ra = numpy.concatenate(all_ra) dec = numpy.concatenate(all_dec) e1 = numpy.concatenate(all_e1) e2 = numpy.concatenate(all_e2) s = numpy.concatenate(all_s) allcat = treecorr.Catalog(ra=ra, dec=dec, ra_units='deg', dec_units='deg', g1=e1, g2=e2) varg = treecorr.calculateVarG(allcat) for rho in [ rho1, rho2, rho3, rho4, rho5 ]: rho.finalize(varg, varg) print '\nFinished processing all files' rho1.write(os.path.join(cat_dir,'rho1.fits')) rho2.write(os.path.join(cat_dir,'rho2.fits')) rho3.write(os.path.join(cat_dir,'rho3.fits')) rho4.write(os.path.join(cat_dir,'rho4.fits')) rho5.write(os.path.join(cat_dir,'rho5.fits'))
def measure_cross_rho(tile_data, max_sep, tags=None, prefix=''): """Compute the rho statistics """ import treecorr ntilings = len(tile_data) print 'len(tile_data) = ',ntilings de1 = [ d[prefix+'e1']-d['psf_e1'] for d in tile_data ] de2 = [ d[prefix+'e2']-d['psf_e2'] for d in tile_data ] dt = [ (d[prefix+'size']**2-d['psf_size']**2)/d['size']**2 for d in tile_data ] for k in range(len(tile_data)): print 'k = ',k print 'mean de = ',numpy.mean(de1[k]),numpy.mean(de2[k]) print 'mean dt = ',numpy.mean(dt[k]) ecats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=d['e1'], g2=d['e2']) for d in tile_data ] for cat in ecats: cat.name = 'ecat' decats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', g1=de1[k], g2=de2[k]) for k,d in enumerate(tile_data) ] for cat in decats: cat.name = 'decat' dtcats = [ treecorr.Catalog(ra=d['ra'], dec=d['dec'], ra_units='deg', dec_units='deg', k=dt[k], g1=d['e1']*dt[k], g2=d['e2']*dt[k]) for k,d in enumerate(tile_data) ] for cat in dtcats: cat.name = 'dtcat' if tags is not None: for catlist in [ ecats, decats, dtcats]: for cat, tag in zip(catlist, tags): cat.name = tag + ":" + cat.name min_sep = 0.3 bin_size = 0.2 bin_slop = 0.3 results = [] for (catlist1, catlist2) in [ (decats, decats), (ecats, decats), (dtcats, dtcats), (decats, dtcats), (ecats, dtcats) ]: catnames1 = [ cat.name for cat in catlist1 ] catnames2 = [ cat.name for cat in catlist2 ] print 'Doing correlation of %s vs %s'%(catnames1, catnames2) rho = treecorr.GGCorrelation(min_sep=min_sep, max_sep=max_sep, sep_units='arcmin', bin_size=bin_size, bin_slop=bin_slop, verbose=2) # Avoid all auto correlations: for i in range(ntilings): for j in range(ntilings): if i == j: continue if catlist1 is catlist2 and i > j: continue print 'names: ',catlist1[i].name,catlist2[j].name rho.process_cross(catlist1[i], catlist2[j]) varg1 = treecorr.calculateVarG(catlist1) varg2 = treecorr.calculateVarG(catlist2) rho.finalize(varg1, varg2) results.append(rho) return results
def test_pieces(): # Test that we can do the calculation in pieces and recombine the results ncats = 3 data_cats = [] nlens = 1000 nsource = 30000 gamma0 = 0.05 r0 = 10. L = 50. * r0 numpy.random.seed(8675309) xl = (numpy.random.random_sample(nlens)-0.5) * L yl = (numpy.random.random_sample(nlens)-0.5) * L xs = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L ys = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L g1 = numpy.zeros( (nsource,ncats) ) g2 = numpy.zeros( (nsource,ncats) ) w = numpy.random.random_sample( (nsource,ncats) ) + 0.5 for x,y in zip(xl,yl): dx = xs-x dy = ys-y r2 = dx**2 + dy**2 gammat = gamma0 * numpy.exp(-0.5*r2/r0**2) g1 += -gammat * (dx**2-dy**2)/r2 g2 += -gammat * (2.*dx*dy)/r2 lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin') source_cats = [ treecorr.Catalog(x=xs[:,k], y=ys[:,k], g1=g1[:,k], g2=g2[:,k], w=w[:,k], x_units='arcmin', y_units='arcmin') for k in range(ncats) ] full_source_cat = treecorr.Catalog(x=xs.flatten(), y=ys.flatten(), w=w.flatten(), g1=g1.flatten(), g2=g2.flatten(), x_units='arcmin', y_units='arcmin') for k in range(ncats): # These could each be done on different machines in a real world application. ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin', verbose=2) # These should use process_cross, not process, since we don't want to call finalize. ng.process_cross(lens_cat, source_cats[k]) ng.write(os.path.join('output','ng_piece%d.fits'%k)) full_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin', verbose=2) full_ng.process(lens_cat, full_source_cat) pieces_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin') for k in range(ncats): ng = pieces_ng.copy() ng.read(os.path.join('output','ng_piece%d.fits'%k)) pieces_ng += ng varg = treecorr.calculateVarG(source_cats) pieces_ng.finalize(varg) print 'max error in meanr = ',numpy.max(pieces_ng.meanr - full_ng.meanr), print ' max meanr = ',numpy.max(full_ng.meanr) print 'max error in meanlogr = ',numpy.max(pieces_ng.meanlogr - full_ng.meanlogr), print ' max meanlogr = ',numpy.max(full_ng.meanlogr) print 'max error in npairs = ',numpy.max(pieces_ng.npairs - full_ng.npairs), print ' max npairs = ',numpy.max(full_ng.npairs) print 'max error in weight = ',numpy.max(pieces_ng.weight - full_ng.weight), print ' max weight = ',numpy.max(full_ng.weight) print 'max error in xi = ',numpy.max(pieces_ng.xi - full_ng.xi), print ' max xi = ',numpy.max(full_ng.xi) print 'max error in xi_im = ',numpy.max(pieces_ng.xi_im - full_ng.xi_im), print ' max xi_im = ',numpy.max(full_ng.xi_im) print 'max error in varxi = ',numpy.max(pieces_ng.varxi - full_ng.varxi), print ' max varxi = ',numpy.max(full_ng.varxi) numpy.testing.assert_almost_equal(pieces_ng.meanr, full_ng.meanr, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.meanlogr, full_ng.meanlogr, decimal=4) numpy.testing.assert_almost_equal(pieces_ng.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.weight*1.e-5, full_ng.weight*1.e-5, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.xi*1.e2, full_ng.xi*1.e2, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.xi_im*1.e2, full_ng.xi_im*1.e2, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.varxi*1.e6, full_ng.varxi*1.e6, decimal=3)
def test_pieces(): # Test that we can do the calculation in pieces and recombine the results import time ncats = 3 data_cats = [] nlens = 1000 nsource = 30000 gamma0 = 0.05 r0 = 10. L = 50. * r0 numpy.random.seed(8675309) xl = (numpy.random.random_sample(nlens)-0.5) * L yl = (numpy.random.random_sample(nlens)-0.5) * L xs = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L ys = (numpy.random.random_sample( (nsource,ncats) )-0.5) * L g1 = numpy.zeros( (nsource,ncats) ) g2 = numpy.zeros( (nsource,ncats) ) w = numpy.random.random_sample( (nsource,ncats) ) + 0.5 for x,y in zip(xl,yl): dx = xs-x dy = ys-y r2 = dx**2 + dy**2 gammat = gamma0 * numpy.exp(-0.5*r2/r0**2) g1 += -gammat * (dx**2-dy**2)/r2 g2 += -gammat * (2.*dx*dy)/r2 lens_cat = treecorr.Catalog(x=xl, y=yl, x_units='arcmin', y_units='arcmin') source_cats = [ treecorr.Catalog(x=xs[:,k], y=ys[:,k], g1=g1[:,k], g2=g2[:,k], w=w[:,k], x_units='arcmin', y_units='arcmin') for k in range(ncats) ] full_source_cat = treecorr.Catalog(x=xs.flatten(), y=ys.flatten(), w=w.flatten(), g1=g1.flatten(), g2=g2.flatten(), x_units='arcmin', y_units='arcmin') t0 = time.time() for k in range(ncats): # These could each be done on different machines in a real world application. ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin', verbose=1) # These should use process_cross, not process, since we don't want to call finalize. ng.process_cross(lens_cat, source_cats[k]) ng.write(os.path.join('output','ng_piece_%d.fits'%k)) pieces_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin') for k in range(ncats): ng = pieces_ng.copy() ng.read(os.path.join('output','ng_piece_%d.fits'%k)) pieces_ng += ng varg = treecorr.calculateVarG(source_cats) pieces_ng.finalize(varg) t1 = time.time() print ('time for piece-wise processing (including I/O) = ',t1-t0) full_ng = treecorr.NGCorrelation(bin_size=0.1, min_sep=1., max_sep=25., sep_units='arcmin', verbose=1) full_ng.process(lens_cat, full_source_cat) t2 = time.time() print ('time for full processing = ',t2-t1) print('max error in meanr = ',numpy.max(pieces_ng.meanr - full_ng.meanr),) print(' max meanr = ',numpy.max(full_ng.meanr)) print('max error in meanlogr = ',numpy.max(pieces_ng.meanlogr - full_ng.meanlogr),) print(' max meanlogr = ',numpy.max(full_ng.meanlogr)) print('max error in npairs = ',numpy.max(pieces_ng.npairs - full_ng.npairs),) print(' max npairs = ',numpy.max(full_ng.npairs)) print('max error in weight = ',numpy.max(pieces_ng.weight - full_ng.weight),) print(' max weight = ',numpy.max(full_ng.weight)) print('max error in xi = ',numpy.max(pieces_ng.xi - full_ng.xi),) print(' max xi = ',numpy.max(full_ng.xi)) print('max error in xi_im = ',numpy.max(pieces_ng.xi_im - full_ng.xi_im),) print(' max xi_im = ',numpy.max(full_ng.xi_im)) print('max error in varxi = ',numpy.max(pieces_ng.varxi - full_ng.varxi),) print(' max varxi = ',numpy.max(full_ng.varxi)) numpy.testing.assert_almost_equal(pieces_ng.meanr, full_ng.meanr, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.meanlogr, full_ng.meanlogr, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.weight*1.e-5, full_ng.weight*1.e-5, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.xi*1.e1, full_ng.xi*1.e1, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.xi_im*1.e1, full_ng.xi_im*1.e1, decimal=2) numpy.testing.assert_almost_equal(pieces_ng.varxi*1.e5, full_ng.varxi*1.e5, decimal=2) # A different way to do this can produce results that are essentially identical to the # full calculation. We can use wpos = w, but set w = 0 for the items in the pieces catalogs # that we don't want to include. This will force the tree to be built identically in each # case, but only use the subset of items in the calculation. The sum of all these should # be identical to the full calculation aside from order of calculation differences. # However, we lose some to speed, since there are a lot more wasted calculations along the # way that have to be duplicated in each piece. w2 = [ numpy.empty(w.shape) for k in range(ncats) ] for k in range(ncats): w2[k][:,:] = 0. w2[k][:,k] = w[:,k] source_cats2 = [ treecorr.Catalog(x=xs.flatten(), y=ys.flatten(), g1=g1.flatten(), g2=g2.flatten(), wpos=w.flatten(), w=w2[k].flatten(), x_units='arcmin', y_units='arcmin') for k in range(ncats) ] t3 = time.time() ng2 = [ full_ng.copy() for k in range(ncats) ] for k in range(ncats): ng2[k].clear() ng2[k].process_cross(lens_cat, source_cats2[k]) pieces_ng2 = full_ng.copy() pieces_ng2.clear() for k in range(ncats): pieces_ng2 += ng2[k] pieces_ng2.finalize(varg) t4 = time.time() print ('time for zero-weight piece-wise processing = ',t4-t3) print('max error in meanr = ',numpy.max(pieces_ng2.meanr - full_ng.meanr),) print(' max meanr = ',numpy.max(full_ng.meanr)) print('max error in meanlogr = ',numpy.max(pieces_ng2.meanlogr - full_ng.meanlogr),) print(' max meanlogr = ',numpy.max(full_ng.meanlogr)) print('max error in npairs = ',numpy.max(pieces_ng2.npairs - full_ng.npairs),) print(' max npairs = ',numpy.max(full_ng.npairs)) print('max error in weight = ',numpy.max(pieces_ng2.weight - full_ng.weight),) print(' max weight = ',numpy.max(full_ng.weight)) print('max error in xi = ',numpy.max(pieces_ng2.xi - full_ng.xi),) print(' max xi = ',numpy.max(full_ng.xi)) print('max error in xi_im = ',numpy.max(pieces_ng2.xi_im - full_ng.xi_im),) print(' max xi_im = ',numpy.max(full_ng.xi_im)) print('max error in varxi = ',numpy.max(pieces_ng2.varxi - full_ng.varxi),) print(' max varxi = ',numpy.max(full_ng.varxi)) numpy.testing.assert_almost_equal(pieces_ng2.meanr, full_ng.meanr, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.meanlogr, full_ng.meanlogr, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.npairs*1.e-5, full_ng.npairs*1.e-5, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.weight*1.e-5, full_ng.weight*1.e-5, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.xi*1.e1, full_ng.xi*1.e1, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.xi_im*1.e1, full_ng.xi_im*1.e1, decimal=8) numpy.testing.assert_almost_equal(pieces_ng2.varxi*1.e5, full_ng.varxi*1.e5, decimal=8)
def test_var(): nobj = 5000 np.random.seed(8675309) # First without weights cats = [] allg1 = [] allg2 = [] allk = [] for i in range(10): x = np.random.random_sample(nobj) y = np.random.random_sample(nobj) g1 = np.random.random_sample(nobj) - 0.5 g2 = np.random.random_sample(nobj) - 0.5 k = np.random.random_sample(nobj) - 0.5 cat = treecorr.Catalog(x=x, y=y, g1=g1, g2=g2, k=k) varg = (np.sum(g1**2) + np.sum(g2**2)) / (2.*len(g1)) vark = np.sum(k**2) / len(k) assert np.isclose(cat.vark, vark) assert np.isclose(cat.varg, varg) assert np.isclose(treecorr.calculateVarK(cat), vark) assert np.isclose(treecorr.calculateVarK([cat]), vark) assert np.isclose(treecorr.calculateVarG(cat), varg) assert np.isclose(treecorr.calculateVarG([cat]), varg) cats.append(cat) allg1.extend(g1) allg2.extend(g2) allk.extend(k) allg1 = np.array(allg1) allg2 = np.array(allg2) allk = np.array(allk) varg = (np.sum(allg1**2) + np.sum(allg2**2)) / (2. * len(allg1)) vark = np.sum(allk**2) / len(allk) assert np.isclose(treecorr.calculateVarG(cats), varg) assert np.isclose(treecorr.calculateVarK(cats), vark) # Now with weights cats = [] allg1 = [] allg2 = [] allk = [] allw = [] for i in range(10): x = np.random.random_sample(nobj) y = np.random.random_sample(nobj) w = np.random.random_sample(nobj) g1 = np.random.random_sample(nobj) g2 = np.random.random_sample(nobj) k = np.random.random_sample(nobj) cat = treecorr.Catalog(x=x, y=y, w=w, g1=g1, g2=g2, k=k) varg = np.sum(w**2 * (g1**2 + g2**2)) / np.sum(w) / 2. vark = np.sum(w**2 * k**2) / np.sum(w) assert np.isclose(cat.varg, varg) assert np.isclose(cat.vark, vark) assert np.isclose(treecorr.calculateVarG(cat), varg) assert np.isclose(treecorr.calculateVarG([cat]), varg) assert np.isclose(treecorr.calculateVarK(cat), vark) assert np.isclose(treecorr.calculateVarK([cat]), vark) cats.append(cat) allg1.extend(g1) allg2.extend(g2) allk.extend(k) allw.extend(w) allg1 = np.array(allg1) allg2 = np.array(allg2) allk = np.array(allk) allw = np.array(allw) varg = np.sum(allw**2 * (allg1**2 + allg2**2)) / np.sum(allw) / 2. vark = np.sum(allw**2 * allk**2) / np.sum(allw) assert np.isclose(treecorr.calculateVarG(cats), varg) assert np.isclose(treecorr.calculateVarK(cats), vark)