def __init__( self, Nmesh, DataSource, Painter=None, paintbrush="cic", paintNmesh=None, dataset="PaintGrid", Nfile=0, writeFourier=False, ): # set the input self.Nmesh = Nmesh self.datasource = DataSource self.dataset = dataset self.writeFourier = writeFourier # set the painter if Painter is None: Painter = Painter.create("DefaultPainter", paintbrush=paintbrush) self.painter = Painter self.painter.paintbrush = paintbrush # Nmesh for the painter if paintNmesh is None: paintNmesh = self.Nmesh self.paintNmesh = paintNmesh self.Nfile = Nfile if self.Nfile == 0: chunksize = 1024 * 1024 * 512 self.Nfile = (self.Nmesh * self.Nmesh * self.Nmesh + chunksize - 1) // chunksize
def FieldType(ns): """ Construct and return a `Field`: a tuple of (`DataSource`, `Painter`, `Transfer`) Notes ----- * the default Painter is set to `DefaultPainter` * the default Transfer chain is set to [`NormalizeDC`, `RemoveDC`, `AnisotropicCIC`] Parameters ---------- fields_dict : OrderedDict an ordered dictionary where the keys are Plugin names and the values are instantiated Plugins Returns ------- field : list list of (DataSource, Painter, Transfer) """ # define the default Painter and Transfer default_painter = Painter.create("DefaultPainter") default_transfer = [Transfer.create(x) for x in ['NormalizeDC', 'RemoveDC', 'AnisotropicCIC']] # start with a default option for (DataSource, Painter, Transfer) field = [None, default_painter, default_transfer] # set the DataSource if 'DataSource' not in ns: raise ValueError("exactly one `DataSource` per field must be specified") field[0] = getattr(ns, 'DataSource') # set the Painter if 'Painter' in ns: field[1] = getattr(ns, 'Painter') # set the Transfer if 'Transfer' in ns: field[2] = getattr(ns, 'Transfer') return field
def run(self): """ Run the TidalTensor Algorithm """ from itertools import product if self.smoothing is None: self.smoothing = self.field.BoxSize[0] / self.Nmesh elif (self.field.BoxSize / self.Nmesh > self.smoothing).any(): raise ValueError("smoothing is too small") painter = Painter.create("DefaultPainter", weight="Mass", paintbrush="cic") real, stats = painter.paint(self.pm, self.field) complex = real.r2c() for t in [self.Smoothing, self.NormalizeDC]: t(complex.pm, complex) with self.points.open() as stream: [[Position]] = stream.read(['Position'], full=True) layout = self.pm.decompose(Position) pos1 = layout.exchange(Position) value = numpy.empty((3, 3, len(Position))) for u, v in product(range(3), range(3)): if self.comm.rank == 0: self.logger.info("Working on tensor element (%d, %d)" % (u, v)) c2 = complex.copy() self.TidalTensor(u, v)(c2.pm, c2) c2.c2r(real) v1 = real.readout(pos1) v1 = layout.gather(v1) value[u, v] = v1 return value.transpose((2, 0, 1))
def run(self): """ Run the TidalTensor Algorithm """ from itertools import product if self.smoothing is None: self.smoothing = self.field.BoxSize[0] / self.Nmesh elif (self.field.BoxSize / self.Nmesh > self.smoothing).any(): raise ValueError("smoothing is too small") painter = Painter.create("DefaultPainter", weight="Mass", paintbrush="cic") real, stats = painter.paint(self.pm, self.field) complex = real.r2c() for t in [self.Smoothing, self.NormalizeDC]: t(complex.pm, complex) with self.points.open() as stream: [[Position ]] = stream.read(['Position'], full=True) layout = self.pm.decompose(Position) pos1 = layout.exchange(Position) value = numpy.empty((3, 3, len(Position))) for u, v in product(range(3), range(3)): if self.comm.rank == 0: self.logger.info("Working on tensor element (%d, %d)" % (u, v)) c2 = complex.copy() self.TidalTensor(u, v)(c2.pm, c2) c2.c2r(real) v1 = real.readout(pos1) v1 = layout.gather(v1) value[u, v] = v1 return value.transpose((2, 0, 1))
def __init__(self, Nmesh, DataSource, Painter=None, paintbrush='cic', paintNmesh=None, dataset='PaintGrid', Nfile=0, writeFourier=False): # set the input self.Nmesh = Nmesh self.datasource = DataSource self.dataset = dataset self.writeFourier = writeFourier # set the painter if Painter is None: Painter = Painter.create("DefaultPainter", paintbrush=paintbrush) self.painter = Painter self.painter.paintbrush = paintbrush # Nmesh for the painter if paintNmesh is None: paintNmesh = self.Nmesh self.paintNmesh = paintNmesh self.Nfile = Nfile if self.Nfile == 0: chunksize = 1024 * 1024 * 512 self.Nfile = (self.Nmesh * self.Nmesh * self.Nmesh + chunksize - 1)// chunksize
def run(self): """ Run the Subsample algorithm """ import mpsort from astropy.utils.misc import NumpyRNGContext if self.smoothing is None: self.smoothing = self.datasource.BoxSize[0] / self.Nmesh[0] elif (self.datasource.BoxSize / self.Nmesh > self.smoothing).any(): raise ValueError("smoothing is too small") def Smoothing(pm, complex): k = pm.k k2 = 0 for ki in k: ki2 = ki**2 complex[:] *= numpy.exp(-0.5 * ki2 * self.smoothing**2) def NormalizeDC(pm, complex): """ removes the DC amplitude. This effectively divides by the mean """ w = pm.w comm = pm.comm ind = [] value = 0.0 found = True for wi in w: if (wi != 0).all(): found = False break ind.append((wi == 0).nonzero()[0][0]) if found: ind = tuple(ind) value = numpy.abs(complex[ind]) value = comm.allreduce(value) complex[:] /= value # open the datasource and keep the cache with self.datasource.keep_cache(): painter = Painter.create("DefaultPainter", paintbrush='cic') real, stats = painter.paint(self.pm, self.datasource) complex = real.r2c() for t in [Smoothing, NormalizeDC]: t(self.pm, complex) complex.c2r(real) columns = ['Position', 'ID', 'Velocity'] local_seed = utils.local_random_seed(self.seed, self.comm) dtype = numpy.dtype([ ('Position', ('f4', 3)), ('Velocity', ('f4', 3)), ('ID', 'u8'), ('Density', 'f4'), ]) subsample = [numpy.empty(0, dtype=dtype)] with self.datasource.open() as stream: for Position, ID, Velocity in stream.read(columns): with NumpyRNGContext(local_seed): u = numpy.random.uniform(size=len(ID)) keep = u < self.ratio Nkeep = keep.sum() if Nkeep == 0: continue data = numpy.empty(Nkeep, dtype=dtype) data['Position'][:] = Position[keep] data['Velocity'][:] = Velocity[keep] data['ID'][:] = ID[keep] layout = self.pm.decompose(data['Position']) pos1 = layout.exchange(data['Position']) density1 = real.readout(pos1) density = layout.gather(density1) # normalize the position after reading out density! data['Position'][:] /= self.datasource.BoxSize data['Velocity'][:] /= self.datasource.BoxSize data['Density'][:] = density subsample.append(data) subsample = numpy.concatenate(subsample) mpsort.sort(subsample, orderby='ID', comm=self.comm) return subsample
def __init__(self, data, randoms, BoxSize=None, BoxPad=0.02, compute_fkp_weights=False, P0_fkp=None, nbar=None, fsky=None): """ Parameters ---------- data : DataSource the DataSource that corresponds to the 'data' catalog randoms : DataSource the DataSource that corresponds to the 'randoms' catalog BoxSize : {float, array_like (3,)}, optional the size of the Cartesian to box when gridding the data and randoms; if not provided, the box will automatically be computed from the maximum extent of the randoms catalog BoxPad : float, optional if BoxSize is not provided, apply this padding to the box that is automatically created from the randoms catalog; default is ``0.02`` compute_fkp_weights : bool, optional if ``True``, compute and apply FKP weights using `P0_fkp` and the number density n(z) column from the input data sources; default is ``False`` P0_fkp : float, optional if ``compute_fkp_weights=True``, use this value in in the FKP weights, which are defined as, :math:`w_\mathrm{FKP} = 1 / (1 + n_g(x) * P_0)` nbar : {str, float}, optional either the name of a file, giving ``(z, n(z))`` in columns, or a scalar float, which gives the constant n(z) value to use fsky : float, optional if ``nbar = None``, then the n(z) is computed from the randoms catalog, and the sky fraction covered by the survey is required to properly normalize the number density (for the volume calculation) """ # set the cosmology self.cosmo = data.cosmo if self.cosmo is None: raise ValueError("FKPCatalog requires a cosmology") if data.cosmo is not randoms.cosmo: raise ValueError("mismatch between cosmology instances of `data` and `randoms` in `FKPCatalog`") # set the comm self.comm = data.comm if data.comm is not randoms.comm: raise ValueError("mismatch between communicators of `data` and `randoms` in `FKPCatalog`") # data and randoms datasources self.data = data self.randoms = randoms # set the BoxSize and BoxPad self.BoxSize = BoxSize if self.BoxSize is not None: self.BoxSize = DataSource.BoxSizeParser(self.BoxSize) self.BoxPad = BoxPad # weights configuration self.compute_fkp_weights = compute_fkp_weights self.P0_fkp = P0_fkp # n(z) configuration self.nbar = nbar self.fsky = fsky # use the default painter when painting self.painter = Painter.create('DefaultPainter')
def run(self): """ Run the Subsample algorithm """ import mpsort from astropy.utils.misc import NumpyRNGContext if self.smoothing is None: self.smoothing = self.datasource.BoxSize[0] / self.Nmesh[0] elif (self.datasource.BoxSize / self.Nmesh > self.smoothing).any(): raise ValueError("smoothing is too small") def Smoothing(pm, complex): k = pm.k k2 = 0 for ki in k: ki2 = ki ** 2 complex[:] *= numpy.exp(-0.5 * ki2 * self.smoothing ** 2) def NormalizeDC(pm, complex): """ removes the DC amplitude. This effectively divides by the mean """ w = pm.w comm = pm.comm ind = [] value = 0.0 found = True for wi in w: if (wi != 0).all(): found = False break ind.append((wi == 0).nonzero()[0][0]) if found: ind = tuple(ind) value = numpy.abs(complex[ind]) value = comm.allreduce(value) complex[:] /= value # open the datasource and keep the cache with self.datasource.keep_cache(): painter = Painter.create("DefaultPainter", paintbrush="cic") real, stats = painter.paint(self.pm, self.datasource) complex = real.r2c() for t in [Smoothing, NormalizeDC]: t(self.pm, complex) complex.c2r(real) columns = ["Position", "ID", "Velocity"] local_seed = utils.local_random_seed(self.seed, self.comm) dtype = numpy.dtype([("Position", ("f4", 3)), ("Velocity", ("f4", 3)), ("ID", "u8"), ("Density", "f4")]) subsample = [numpy.empty(0, dtype=dtype)] with self.datasource.open() as stream: for Position, ID, Velocity in stream.read(columns): with NumpyRNGContext(local_seed): u = numpy.random.uniform(size=len(ID)) keep = u < self.ratio Nkeep = keep.sum() if Nkeep == 0: continue data = numpy.empty(Nkeep, dtype=dtype) data["Position"][:] = Position[keep] data["Velocity"][:] = Velocity[keep] data["ID"][:] = ID[keep] layout = self.pm.decompose(data["Position"]) pos1 = layout.exchange(data["Position"]) density1 = real.readout(pos1) density = layout.gather(density1) # normalize the position after reading out density! data["Position"][:] /= self.datasource.BoxSize data["Velocity"][:] /= self.datasource.BoxSize data["Density"][:] = density subsample.append(data) subsample = numpy.concatenate(subsample) mpsort.sort(subsample, orderby="ID", comm=self.comm) return subsample
def __init__(self, data, randoms, BoxSize=None, BoxPad=0.02, compute_fkp_weights=False, P0_fkp=None, nbar=None, fsky=None): """ Parameters ---------- data : DataSource the DataSource that corresponds to the 'data' catalog randoms : DataSource the DataSource that corresponds to the 'randoms' catalog BoxSize : {float, array_like (3,)}, optional the size of the Cartesian to box when gridding the data and randoms; if not provided, the box will automatically be computed from the maximum extent of the randoms catalog BoxPad : float, optional if BoxSize is not provided, apply this padding to the box that is automatically created from the randoms catalog; default is ``0.02`` compute_fkp_weights : bool, optional if ``True``, compute and apply FKP weights using `P0_fkp` and the number density n(z) column from the input data sources; default is ``False`` P0_fkp : float, optional if ``compute_fkp_weights=True``, use this value in in the FKP weights, which are defined as, :math:`w_\mathrm{FKP} = 1 / (1 + n_g(x) * P_0)` nbar : {str, float}, optional either the name of a file, giving ``(z, n(z))`` in columns, or a scalar float, which gives the constant n(z) value to use fsky : float, optional if ``nbar = None``, then the n(z) is computed from the randoms catalog, and the sky fraction covered by the survey is required to properly normalize the number density (for the volume calculation) """ # set the cosmology self.cosmo = data.cosmo if self.cosmo is None: raise ValueError("FKPCatalog requires a cosmology") if data.cosmo is not randoms.cosmo: raise ValueError( "mismatch between cosmology instances of `data` and `randoms` in `FKPCatalog`" ) # set the comm self.comm = data.comm if data.comm is not randoms.comm: raise ValueError( "mismatch between communicators of `data` and `randoms` in `FKPCatalog`" ) # data and randoms datasources self.data = data self.randoms = randoms # set the BoxSize and BoxPad self.BoxSize = BoxSize if self.BoxSize is not None: self.BoxSize = DataSource.BoxSizeParser(self.BoxSize) self.BoxPad = BoxPad # weights configuration self.compute_fkp_weights = compute_fkp_weights self.P0_fkp = P0_fkp # n(z) configuration self.nbar = nbar self.fsky = fsky # use the default painter when painting self.painter = Painter.create('DefaultPainter')