def __init__(
self,
Nmesh,
DataSource,
Painter=None,
paintbrush="cic",
paintNmesh=None,
dataset="PaintGrid",
Nfile=0,
writeFourier=False,
):
# set the input
self.Nmesh = Nmesh
self.datasource = DataSource
self.dataset = dataset
self.writeFourier = writeFourier
# set the painter
if Painter is None:
Painter = Painter.create("DefaultPainter", paintbrush=paintbrush)
self.painter = Painter
self.painter.paintbrush = paintbrush
# Nmesh for the painter
if paintNmesh is None:
paintNmesh = self.Nmesh
self.paintNmesh = paintNmesh
self.Nfile = Nfile
if self.Nfile == 0:
chunksize = 1024 * 1024 * 512
self.Nfile = (self.Nmesh * self.Nmesh * self.Nmesh + chunksize - 1) // chunksize
def FieldType(ns):
"""
Construct and return a `Field`:
a tuple of (`DataSource`, `Painter`, `Transfer`)
Notes
-----
* the default Painter is set to `DefaultPainter`
* the default Transfer chain is set to
[`NormalizeDC`, `RemoveDC`, `AnisotropicCIC`]
Parameters
----------
fields_dict : OrderedDict
an ordered dictionary where the keys are Plugin names
and the values are instantiated Plugins
Returns
-------
field : list
list of (DataSource, Painter, Transfer)
"""
# define the default Painter and Transfer
default_painter = Painter.create("DefaultPainter")
default_transfer = [Transfer.create(x) for x in ['NormalizeDC', 'RemoveDC', 'AnisotropicCIC']]
# start with a default option for (DataSource, Painter, Transfer)
field = [None, default_painter, default_transfer]
# set the DataSource
if 'DataSource' not in ns:
raise ValueError("exactly one `DataSource` per field must be specified")
field[0] = getattr(ns, 'DataSource')
# set the Painter
if 'Painter' in ns:
field[1] = getattr(ns, 'Painter')
# set the Transfer
if 'Transfer' in ns:
field[2] = getattr(ns, 'Transfer')
return field
def run(self):
"""
Run the TidalTensor Algorithm
"""
from itertools import product
if self.smoothing is None:
self.smoothing = self.field.BoxSize[0] / self.Nmesh
elif (self.field.BoxSize / self.Nmesh > self.smoothing).any():
raise ValueError("smoothing is too small")
painter = Painter.create("DefaultPainter",
weight="Mass",
paintbrush="cic")
real, stats = painter.paint(self.pm, self.field)
complex = real.r2c()
for t in [self.Smoothing, self.NormalizeDC]:
t(complex.pm, complex)
with self.points.open() as stream:
[[Position]] = stream.read(['Position'], full=True)
layout = self.pm.decompose(Position)
pos1 = layout.exchange(Position)
value = numpy.empty((3, 3, len(Position)))
for u, v in product(range(3), range(3)):
if self.comm.rank == 0:
self.logger.info("Working on tensor element (%d, %d)" % (u, v))
c2 = complex.copy()
self.TidalTensor(u, v)(c2.pm, c2)
c2.c2r(real)
v1 = real.readout(pos1)
v1 = layout.gather(v1)
value[u, v] = v1
return value.transpose((2, 0, 1))
def run(self):
"""
Run the TidalTensor Algorithm
"""
from itertools import product
if self.smoothing is None:
self.smoothing = self.field.BoxSize[0] / self.Nmesh
elif (self.field.BoxSize / self.Nmesh > self.smoothing).any():
raise ValueError("smoothing is too small")
painter = Painter.create("DefaultPainter", weight="Mass", paintbrush="cic")
real, stats = painter.paint(self.pm, self.field)
complex = real.r2c()
for t in [self.Smoothing, self.NormalizeDC]:
t(complex.pm, complex)
with self.points.open() as stream:
[[Position ]] = stream.read(['Position'], full=True)
layout = self.pm.decompose(Position)
pos1 = layout.exchange(Position)
value = numpy.empty((3, 3, len(Position)))
for u, v in product(range(3), range(3)):
if self.comm.rank == 0:
self.logger.info("Working on tensor element (%d, %d)" % (u, v))
c2 = complex.copy()
self.TidalTensor(u, v)(c2.pm, c2)
c2.c2r(real)
v1 = real.readout(pos1)
v1 = layout.gather(v1)
value[u, v] = v1
return value.transpose((2, 0, 1))
def __init__(self, Nmesh, DataSource, Painter=None, paintbrush='cic', paintNmesh=None,
dataset='PaintGrid', Nfile=0, writeFourier=False):
# set the input
self.Nmesh = Nmesh
self.datasource = DataSource
self.dataset = dataset
self.writeFourier = writeFourier
# set the painter
if Painter is None:
Painter = Painter.create("DefaultPainter", paintbrush=paintbrush)
self.painter = Painter
self.painter.paintbrush = paintbrush
# Nmesh for the painter
if paintNmesh is None:
paintNmesh = self.Nmesh
self.paintNmesh = paintNmesh
self.Nfile = Nfile
if self.Nfile == 0:
chunksize = 1024 * 1024 * 512
self.Nfile = (self.Nmesh * self.Nmesh * self.Nmesh + chunksize - 1)// chunksize
def run(self):
"""
Run the Subsample algorithm
"""
import mpsort
from astropy.utils.misc import NumpyRNGContext
if self.smoothing is None:
self.smoothing = self.datasource.BoxSize[0] / self.Nmesh[0]
elif (self.datasource.BoxSize / self.Nmesh > self.smoothing).any():
raise ValueError("smoothing is too small")
def Smoothing(pm, complex):
k = pm.k
k2 = 0
for ki in k:
ki2 = ki**2
complex[:] *= numpy.exp(-0.5 * ki2 * self.smoothing**2)
def NormalizeDC(pm, complex):
""" removes the DC amplitude. This effectively
divides by the mean
"""
w = pm.w
comm = pm.comm
ind = []
value = 0.0
found = True
for wi in w:
if (wi != 0).all():
found = False
break
ind.append((wi == 0).nonzero()[0][0])
if found:
ind = tuple(ind)
value = numpy.abs(complex[ind])
value = comm.allreduce(value)
complex[:] /= value
# open the datasource and keep the cache
with self.datasource.keep_cache():
painter = Painter.create("DefaultPainter", paintbrush='cic')
real, stats = painter.paint(self.pm, self.datasource)
complex = real.r2c()
for t in [Smoothing, NormalizeDC]:
t(self.pm, complex)
complex.c2r(real)
columns = ['Position', 'ID', 'Velocity']
local_seed = utils.local_random_seed(self.seed, self.comm)
dtype = numpy.dtype([
('Position', ('f4', 3)),
('Velocity', ('f4', 3)),
('ID', 'u8'),
('Density', 'f4'),
])
subsample = [numpy.empty(0, dtype=dtype)]
with self.datasource.open() as stream:
for Position, ID, Velocity in stream.read(columns):
with NumpyRNGContext(local_seed):
u = numpy.random.uniform(size=len(ID))
keep = u < self.ratio
Nkeep = keep.sum()
if Nkeep == 0: continue
data = numpy.empty(Nkeep, dtype=dtype)
data['Position'][:] = Position[keep]
data['Velocity'][:] = Velocity[keep]
data['ID'][:] = ID[keep]
layout = self.pm.decompose(data['Position'])
pos1 = layout.exchange(data['Position'])
density1 = real.readout(pos1)
density = layout.gather(density1)
# normalize the position after reading out density!
data['Position'][:] /= self.datasource.BoxSize
data['Velocity'][:] /= self.datasource.BoxSize
data['Density'][:] = density
subsample.append(data)
subsample = numpy.concatenate(subsample)
mpsort.sort(subsample, orderby='ID', comm=self.comm)
return subsample
def __init__(self, data,
randoms,
BoxSize=None,
BoxPad=0.02,
compute_fkp_weights=False,
P0_fkp=None,
nbar=None,
fsky=None):
"""
Parameters
----------
data : DataSource
the DataSource that corresponds to the 'data' catalog
randoms : DataSource
the DataSource that corresponds to the 'randoms' catalog
BoxSize : {float, array_like (3,)}, optional
the size of the Cartesian to box when gridding the data and randoms;
if not provided, the box will automatically be computed from the
maximum extent of the randoms catalog
BoxPad : float, optional
if BoxSize is not provided, apply this padding to the box that
is automatically created from the randoms catalog; default is ``0.02``
compute_fkp_weights : bool, optional
if ``True``, compute and apply FKP weights using `P0_fkp` and
the number density n(z) column from the input data sources;
default is ``False``
P0_fkp : float, optional
if ``compute_fkp_weights=True``, use this value in in the FKP
weights, which are defined as, :math:`w_\mathrm{FKP} = 1 / (1 + n_g(x) * P_0)`
nbar : {str, float}, optional
either the name of a file, giving ``(z, n(z))`` in columns, or a scalar float,
which gives the constant n(z) value to use
fsky : float, optional
if ``nbar = None``, then the n(z) is computed from the randoms catalog, and
the sky fraction covered by the survey is required to properly normalize
the number density (for the volume calculation)
"""
# set the cosmology
self.cosmo = data.cosmo
if self.cosmo is None:
raise ValueError("FKPCatalog requires a cosmology")
if data.cosmo is not randoms.cosmo:
raise ValueError("mismatch between cosmology instances of `data` and `randoms` in `FKPCatalog`")
# set the comm
self.comm = data.comm
if data.comm is not randoms.comm:
raise ValueError("mismatch between communicators of `data` and `randoms` in `FKPCatalog`")
# data and randoms datasources
self.data = data
self.randoms = randoms
# set the BoxSize and BoxPad
self.BoxSize = BoxSize
if self.BoxSize is not None:
self.BoxSize = DataSource.BoxSizeParser(self.BoxSize)
self.BoxPad = BoxPad
# weights configuration
self.compute_fkp_weights = compute_fkp_weights
self.P0_fkp = P0_fkp
# n(z) configuration
self.nbar = nbar
self.fsky = fsky
# use the default painter when painting
self.painter = Painter.create('DefaultPainter')
def run(self):
"""
Run the Subsample algorithm
"""
import mpsort
from astropy.utils.misc import NumpyRNGContext
if self.smoothing is None:
self.smoothing = self.datasource.BoxSize[0] / self.Nmesh[0]
elif (self.datasource.BoxSize / self.Nmesh > self.smoothing).any():
raise ValueError("smoothing is too small")
def Smoothing(pm, complex):
k = pm.k
k2 = 0
for ki in k:
ki2 = ki ** 2
complex[:] *= numpy.exp(-0.5 * ki2 * self.smoothing ** 2)
def NormalizeDC(pm, complex):
""" removes the DC amplitude. This effectively
divides by the mean
"""
w = pm.w
comm = pm.comm
ind = []
value = 0.0
found = True
for wi in w:
if (wi != 0).all():
found = False
break
ind.append((wi == 0).nonzero()[0][0])
if found:
ind = tuple(ind)
value = numpy.abs(complex[ind])
value = comm.allreduce(value)
complex[:] /= value
# open the datasource and keep the cache
with self.datasource.keep_cache():
painter = Painter.create("DefaultPainter", paintbrush="cic")
real, stats = painter.paint(self.pm, self.datasource)
complex = real.r2c()
for t in [Smoothing, NormalizeDC]:
t(self.pm, complex)
complex.c2r(real)
columns = ["Position", "ID", "Velocity"]
local_seed = utils.local_random_seed(self.seed, self.comm)
dtype = numpy.dtype([("Position", ("f4", 3)), ("Velocity", ("f4", 3)), ("ID", "u8"), ("Density", "f4")])
subsample = [numpy.empty(0, dtype=dtype)]
with self.datasource.open() as stream:
for Position, ID, Velocity in stream.read(columns):
with NumpyRNGContext(local_seed):
u = numpy.random.uniform(size=len(ID))
keep = u < self.ratio
Nkeep = keep.sum()
if Nkeep == 0:
continue
data = numpy.empty(Nkeep, dtype=dtype)
data["Position"][:] = Position[keep]
data["Velocity"][:] = Velocity[keep]
data["ID"][:] = ID[keep]
layout = self.pm.decompose(data["Position"])
pos1 = layout.exchange(data["Position"])
density1 = real.readout(pos1)
density = layout.gather(density1)
# normalize the position after reading out density!
data["Position"][:] /= self.datasource.BoxSize
data["Velocity"][:] /= self.datasource.BoxSize
data["Density"][:] = density
subsample.append(data)
subsample = numpy.concatenate(subsample)
mpsort.sort(subsample, orderby="ID", comm=self.comm)
return subsample
def __init__(self,
data,
randoms,
BoxSize=None,
BoxPad=0.02,
compute_fkp_weights=False,
P0_fkp=None,
nbar=None,
fsky=None):
"""
Parameters
----------
data : DataSource
the DataSource that corresponds to the 'data' catalog
randoms : DataSource
the DataSource that corresponds to the 'randoms' catalog
BoxSize : {float, array_like (3,)}, optional
the size of the Cartesian to box when gridding the data and randoms;
if not provided, the box will automatically be computed from the
maximum extent of the randoms catalog
BoxPad : float, optional
if BoxSize is not provided, apply this padding to the box that
is automatically created from the randoms catalog; default is ``0.02``
compute_fkp_weights : bool, optional
if ``True``, compute and apply FKP weights using `P0_fkp` and
the number density n(z) column from the input data sources;
default is ``False``
P0_fkp : float, optional
if ``compute_fkp_weights=True``, use this value in in the FKP
weights, which are defined as, :math:`w_\mathrm{FKP} = 1 / (1 + n_g(x) * P_0)`
nbar : {str, float}, optional
either the name of a file, giving ``(z, n(z))`` in columns, or a scalar float,
which gives the constant n(z) value to use
fsky : float, optional
if ``nbar = None``, then the n(z) is computed from the randoms catalog, and
the sky fraction covered by the survey is required to properly normalize
the number density (for the volume calculation)
"""
# set the cosmology
self.cosmo = data.cosmo
if self.cosmo is None:
raise ValueError("FKPCatalog requires a cosmology")
if data.cosmo is not randoms.cosmo:
raise ValueError(
"mismatch between cosmology instances of `data` and `randoms` in `FKPCatalog`"
)
# set the comm
self.comm = data.comm
if data.comm is not randoms.comm:
raise ValueError(
"mismatch between communicators of `data` and `randoms` in `FKPCatalog`"
)
# data and randoms datasources
self.data = data
self.randoms = randoms
# set the BoxSize and BoxPad
self.BoxSize = BoxSize
if self.BoxSize is not None:
self.BoxSize = DataSource.BoxSizeParser(self.BoxSize)
self.BoxPad = BoxPad
# weights configuration
self.compute_fkp_weights = compute_fkp_weights
self.P0_fkp = P0_fkp
# n(z) configuration
self.nbar = nbar
self.fsky = fsky
# use the default painter when painting
self.painter = Painter.create('DefaultPainter')
