def _check_consistency(self):
for gi in range(self.connectivity_indices.shape[0]):
ind = self.connectivity_indices[gi, :] - self._index_offset
coords = self.connectivity_coords[ind, :]
for i in range(3):
assert(np.unique(coords[:,i]).size == 2)
mylog.debug("Connectivity is consistent.")
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
pos = np.empty((pcount, 3), dtype="float64")
pos = data_file.ds.arr(pos, "code_length")
dx = np.finfo(f['particle_position_x'].dtype).eps
dx = 2.0*self.ds.quan(dx, "code_length")
pos[:,0] = f["particle_position_x"].value
pos[:,1] = f["particle_position_y"].value
pos[:,2] = f["particle_position_z"].value
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind+pos.shape[0]] = compute_morton(
pos[:,0], pos[:,1], pos[:,2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
return morton
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
assert(len(chunks) == 1)
tags = {}
rv = {}
pyne_mesh = self.ds.pyne_mesh
mesh = pyne_mesh.mesh
for field in fields:
rv[field] = np.empty(size, dtype="float64")
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks",
size, [fname for ftype, fname in fields], ngrids)
for field in fields:
ftype, fname = field
if pyne_mesh.structured:
tag = pyne_mesh.mesh.getTagHandle('idx')
indices = [tag[ent] for ent in pyne_mesh.structured_iterate_hex()]
else:
indices = slice(None)
ds = np.asarray(getattr(pyne_mesh, fname)[indices], 'float64')
ind = 0
for chunk in chunks:
for g in chunk.objs:
ind += g.select(selector, ds, rv[field], ind) # caches
return rv
def filter_sphere(self, center, radius, myiter):
"""
Filter data by masking out data outside of a sphere defined
by a center and radius. Account for periodicity of data, allowing
left/right to be outside of the domain.
"""
# Get left/right for periodicity considerations
left = center - radius
right = center + radius
for data in myiter:
pos = np.array([data['x'].copy(), data['y'].copy(), data['z'].copy()]).T
DW = self.true_domain_width
_shift_periodic(pos, left, right, DW)
# Now get all particles that are within the sphere
mask = ((pos-center)**2).sum(axis=1)**0.5 < radius
mylog.debug("Filtering particles, returning %i out of %i" % (mask.sum(), mask.shape[0]))
if not np.any(mask):
continue
filtered = {ax: pos[:, i][mask] for i, ax in enumerate('xyz')}
for f in data.keys():
if f in 'xyz':
continue
filtered[f] = data[f][mask]
yield filtered
def get_key_data(self, key, fields):
if key > self._max_key:
raise RuntimeError("Left key is too large. Key: %i Max Key: %i" % (key, self._max_key))
base = self.indexdata['base'][key]
length = self.indexdata['len'][key] - base
if length > 0:
mylog.debug('Getting contiguous chunk of size %i starting at %i' % (length, base))
return self.get_data(slice(base, base + length), fields)
def iter_sphere_data(self, center, radius, fields):
"""
Iterate over all data within some sphere defined by a center and
a radius.
"""
_ensure_xyz_fields(fields)
mylog.debug('MIDX Loading spherical region %s to %s' %(center, radius))
inds = self.get_bbox(center-radius, center+radius)
for dd in self.filter_sphere(
center, radius,
self.iter_data(inds, fields)):
yield dd
def __init__(self, ds, domain_id):
self.ds = ds
self.domain_id = domain_id
num = os.path.basename(ds.parameter_filename).split(".")[0].split("_")[1]
rootdir = ds.root_folder
basedir = os.path.abspath(os.path.dirname(ds.parameter_filename))
basename = "%s/%%s_%s.out%05i" % (basedir, num, domain_id)
part_file_descriptor = f"{basedir}/part_file_descriptor.txt"
if ds.num_groups > 0:
igroup = ((domain_id - 1) // ds.group_size) + 1
basename = "%s/group_%05i/%%s_%s.out%05i" % (
rootdir,
igroup,
num,
domain_id,
)
else:
basename = "%s/%%s_%s.out%05i" % (basedir, num, domain_id)
for t in ["grav", "amr"]:
setattr(self, f"{t}_fn", basename % t)
self._part_file_descriptor = part_file_descriptor
self._read_amr_header()
# Autodetect field files
field_handlers = [FH(self) for FH in get_field_handlers() if FH.any_exist(ds)]
self.field_handlers = field_handlers
for fh in field_handlers:
mylog.debug("Detected fluid type %s in domain_id=%s", fh.ftype, domain_id)
fh.detect_fields(ds)
# self._add_ftype(fh.ftype)
# Autodetect particle files
particle_handlers = [
PH(self) for PH in get_particle_handlers() if PH.any_exist(ds)
]
self.particle_handlers = particle_handlers
for ph in particle_handlers:
mylog.debug(
"Detected particle type %s in domain_id=%s", ph.ptype, domain_id
)
ph.read_header()
# self._add_ptype(ph.ptype)
# Load the AMR structure
self._read_amr()
def _identify_base_chunk(self, dobj):
if getattr(dobj, "_chunk_info", None) is None:
domains = [dom for dom in self.domains if dom.included(dobj.selector)]
base_region = getattr(dobj, "base_region", dobj)
if len(domains) > 1:
mylog.debug("Identified %s intersecting domains", len(domains))
subsets = [
RAMSESDomainSubset(
base_region,
domain,
self.dataset,
num_ghost_zones=dobj._num_ghost_zones,
)
for domain in domains
]
dobj._chunk_info = subsets
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def iter_bbox_data(self, left, right, fields):
"""
Iterate over all data within a bounding box defined by a left
and a right.
"""
_ensure_xyz_fields(fields)
mylog.debug('MIDX Loading region from %s to %s' % (left, right))
inds = self.get_bbox(left, right)
# Need to put left/right in float32 to avoid fp roundoff errors
# in the bbox later.
#left = left.astype('float32')
#right = right.astype('float32')
#my_filter = bbox_filter(left, right, self.true_domain_width)
data = []
for dd in self.filter_bbox(left, right, self.iter_data(inds, fields)):
yield dd
def _identify_base_chunk(self, dobj):
"""
Take the passed in data source dobj, and use its embedded selector
to calculate the domain mask, build the reduced domain
subsets and oct counts. Attach this information to dobj.
"""
if getattr(dobj, "_chunk_info", None) is None:
# Get all octs within this oct handler
domains = [dom for dom in self.domains if
dom.included(dobj.selector)]
base_region = getattr(dobj, "base_region", dobj)
if len(domains) > 1:
mylog.debug("Identified %s intersecting domains", len(domains))
subsets = [ARTDomainSubset(base_region, domain, self.dataset)
for domain in domains]
dobj._chunk_info = subsets
dobj._current_chunk = list(self._chunk_all(dobj))[0]
def _read_amr_root(self, oct_handler):
self.level_offsets
# add the root *cell* not *oct* mesh
root_octs_side = self.ds.domain_dimensions[0] / 2
NX = np.ones(3) * root_octs_side
LE = np.array([0.0, 0.0, 0.0], dtype='float64')
RE = np.array([1.0, 1.0, 1.0], dtype='float64')
root_dx = (RE - LE) / NX
LL = LE + root_dx / 2.0
RL = RE - root_dx / 2.0
# compute floating point centers of root octs
root_fc = np.mgrid[LL[0]:RL[0]:NX[0] * 1j, LL[1]:RL[1]:NX[1] * 1j,
LL[2]:RL[2]:NX[2] * 1j]
root_fc = np.vstack([p.ravel() for p in root_fc]).T
oct_handler.add(self.domain_id, 0, root_fc)
assert (oct_handler.nocts == root_fc.shape[0])
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
root_octs_side**3, 0, oct_handler.nocts)
def _initialize_index(self, data_file, regions):
if self.index_ptype == "all":
ptypes = self.ds.particle_types_raw
pcount = sum(data_file.total_particles.values())
else:
ptypes = [self.index_ptype]
pcount = data_file.total_particles[self.index_ptype]
morton = np.empty(pcount, dtype="uint64")
if pcount == 0:
return morton
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, mode="r") as f:
if not f.keys():
return None
dx = np.finfo(f["Group"]["GroupPos"].dtype).eps
dx = 2.0 * self.ds.quan(dx, "code_length")
for ptype in ptypes:
if data_file.total_particles[ptype] == 0:
continue
pos = data_file._get_particle_positions(ptype, f=f)
pos = self.ds.arr(pos, "code_length")
if np.any(
pos.min(axis=0) < self.ds.domain_left_edge) or np.any(
pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(
pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge,
)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0],
pos[:, 1],
pos[:, 2],
self.ds.domain_left_edge,
self.ds.domain_right_edge,
)
ind += pos.shape[0]
return morton
def calculate_parentage_fractions(self, other_catalog, radius = 0.10):
parentage_fractions = {}
if self.halo_positions is None or other_catalog.halo_positions is None:
return parentage_fractions
mylog.debug("Ball-tree query with radius %0.3e", radius)
all_nearest = self.halo_kdtree.query_ball_tree(
other_catalog.halo_kdtree, radius)
pbar = get_pbar("Halo Mergers", self.halo_positions.shape[0])
for hid1, nearest in enumerate(all_nearest):
pbar.update(hid1)
parentage_fractions[hid1] = {}
HPL1 = self.read_particle_ids(hid1)
for hid2 in sorted(nearest):
HPL2 = other_catalog.read_particle_ids(hid2)
p1, p2 = HPL1.find_relative_parentage(HPL2)
parentage_fractions[hid1][hid2] = (p1, p2, HPL2.number_of_particles)
parentage_fractions[hid1]["NumberOfParticles"] = HPL1.number_of_particles
pbar.finish()
return parentage_fractions
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
assert(len(chunks) == 1)
fhandle = self._handle
rv = {}
for field in fields:
ftype, fname = field
rv[field] = np.empty(size, dtype=fhandle[field_dname(fname)].dtype)
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks",
size, [fname for ftype, fname in fields], ngrids)
for field in fields:
ftype, fname = field
ds = np.array(fhandle[field_dname(fname)][:], dtype="float64")
ind = 0
for chunk in chunks:
for g in chunk.objs:
ind += g.select(selector, ds, rv[field], ind) # caches
return rv
def iter_bbox_data(self, left, right, fields):
"""
Iterate over all data within a bounding box defined by a left
and a right.
"""
_ensure_xyz_fields(fields)
mylog.debug('MIDX Loading region from %s to %s' %(left, right))
inds = self.get_bbox(left, right)
# Need to put left/right in float32 to avoid fp roundoff errors
# in the bbox later.
#left = left.astype('float32')
#right = right.astype('float32')
#my_filter = bbox_filter(left, right, self.true_domain_width)
data = []
for dd in self.filter_bbox(
left, right,
self.iter_data(inds, fields)):
yield dd
def _reconstruct_parent_child(self):
mask = np.empty(len(self.grids), dtype="int32")
mylog.debug("First pass; identifying child grids")
for i, grid in enumerate(self.grids):
get_box_grids_level(
self.grid_left_edge[i, :],
self.grid_right_edge[i, :],
self.grid_levels[i] + 1,
self.grid_left_edge,
self.grid_right_edge,
self.grid_levels,
mask,
)
ids = np.where(mask.astype("bool")) # where is a tuple
grid._children_ids = ids[0] + grid._id_offset
mylog.debug("Second pass; identifying parents")
for i, grid in enumerate(self.grids): # Second pass
for child in grid.Children:
child._parent_id.append(i + grid._id_offset)
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
assert (len(chunks) == 1)
fhandle = self._handle
rv = {}
for field in fields:
ftype, fname = field
rv[field] = np.empty(size, dtype=fhandle[field_dname(fname)].dtype)
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks", size,
[fname for ftype, fname in fields], ngrids)
for field in fields:
ftype, fname = field
ds = np.array(fhandle[field_dname(fname)][:], dtype="float64")
ind = 0
for chunk in chunks:
for g in chunk.objs:
ind += g.select(selector, ds, rv[field], ind) # caches
return rv
def _initialize_index(self, data_file, regions):
halos = data_file.read_data(usecols=['ID'])
pcount = len(halos['ID'])
morton = np.empty(pcount, dtype='uint64')
mylog.debug('Initializing index % 5i (% 7i particles)',
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
pos = data_file._get_particle_positions('halos')
pos = data_file.ds.arr(pos, 'code_length')
dle = self.ds.domain_left_edge
dre = self.ds.domain_right_edge
if np.any(pos.min(axis=0) < dle) or np.any(pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0), dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
return morton
def _guess_headers_from_file(self, filename) -> None:
with FortranFile(filename) as fpu:
ok = False
for dp, longint in product((True, False), (True, False)):
fpu.seek(0)
try:
header_attributes = HEADER_ATTRIBUTES(double=dp,
longint=longint)
fpu.read_attrs(header_attributes)
ok = True
break
except (ValueError, OSError):
pass
if not ok:
raise OSError("Could not read headers from file %s" % filename)
istart = fpu.tell()
fpu.seek(0, 2)
iend = fpu.tell()
# Try different templates
ok = False
for name, cls in ADAPTAHOP_TEMPLATES.items():
fpu.seek(istart)
attributes = cls(longint, dp).HALO_ATTRIBUTES
mylog.debug("Trying %s(longint=%s, dp=%s)", name, longint, dp)
try:
# Try to read two halos to be sure
fpu.read_attrs(attributes)
if fpu.tell() < iend:
fpu.read_attrs(attributes)
ok = True
break
except (ValueError, OSError):
continue
if not ok:
raise OSError("Could not guess fields from file %s" % filename)
self._header_attributes = header_attributes
self._halo_attributes = attributes
def write_input(self, **kwargs):
outputs = self.outputs
ds = self.datasets[-1]
# Halo finder only supports flat lambda-CDM models for which
# omega_lambda + omega_matter = 1
omega_lambda = 1 - ds.omega_matter
# Compute last expansion factor
config = dict(
af=1 / (ds.current_redshift + 1),
lbox=ds.domain_width.in_units('Mpccm').value[0], # in Mpc
H_f=ds.hubble_constant * 100, # in km/s/Mpc
omega_f=ds.omega_matter,
lambda_f=omega_lambda,
npart=100,
method='MSM',
cdm=False,
b=0.2,
nvoisins=20,
nhop=20,
rhot=80.,
fudge=4., # parameter for adaptahop (usually 4.)
fudgepsilon=0.001, # parameter for adaptahop (usually 0.05)
alphap=1.0, # parameter for adaptahop (usually 1.)
verbose=False, # verbose parameter for both halo finder
megaverbose=False, # parameter for adaptahop
nsteps=len(outputs), # Number of time steps to analyse
FlagPeriod=1, # Periodic boundaries (1:True)
DPMMC=False, # Activate the densest point in the most massive cell
SC=True, # Activate the com in concentric spheres
dcell_min=3.05e-3 # smallest possible cell size in Mpc (levelmax=15)
)
config.update(kwargs)
res = dict2conf(config)
self.input_file = path.join(self.prefix, 'input_HaloMaker.dat')
mylog.info('Writing input parameters to %s' % self.input_file)
with open(self.input_file, 'w') as f:
mylog.debug(res)
f.write(res)
def _initialize_index(self, data_file, regions):
all_count = self._count_particles(data_file)
pcount = sum(all_count.values())
morton = np.empty(pcount, dtype="uint64")
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, mode="r") as f:
for ptype in all_count:
if ptype not in f or all_count[ptype] == 0:
continue
pos = np.empty((all_count[ptype], 3), dtype="float64")
units = _get_position_array_units(ptype, f, "x")
if ptype == "grid":
dx = f["grid"]["dx"][()].min()
dx = self.ds.quan(dx,
parse_h5_attr(f["grid"]["dx"],
"units")).to("code_length")
else:
dx = 2.0 * np.finfo(
f[ptype]["particle_position_x"].dtype).eps
dx = self.ds.quan(dx, units).to("code_length")
pos[:, 0] = _get_position_array(ptype, f, "x")
pos[:, 1] = _get_position_array(ptype, f, "y")
pos[:, 2] = _get_position_array(ptype, f, "z")
pos = self.ds.arr(pos, units).to("code_length")
dle = self.ds.domain_left_edge.to("code_length")
dre = self.ds.domain_right_edge.to("code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, dle + dx, dre - dx, pos)
if np.any(pos.min(axis=0) < dle) or np.any(
pos.max(axis=0) > dre):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
dle, dre)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], dle, dre)
ind += pos.shape[0]
return morton
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
assert (len(chunks) == 1)
tags = {}
rv = {}
pyne_mesh = self.ds.pyne_mesh
mesh = pyne_mesh.mesh
for field in fields:
rv[field] = np.empty(size, dtype="float64")
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks", size,
[fname for ftype, fname in fields], ngrids)
for field in fields:
ftype, fname = field
ds = np.asarray(getattr(pyne_mesh, fname)[:], 'float64')
ind = 0
for chunk in chunks:
for g in chunk.objs:
ind += g.select(selector, ds, rv[field], ind) # caches
return rv
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
if any((ftype != "boxlib" for ftype, fname in fields)):
raise NotImplementedError
rv = {}
for field in fields:
rv[field] = np.empty(size, dtype="float64")
ng = sum(len(c.objs) for c in chunks)
mylog.debug("Reading %s cells of %s fields in %s grids",
size, [f2 for f1, f2 in fields], ng)
ind = 0
for chunk in chunks:
data = self._read_chunk_data(chunk, fields)
for g in chunk.objs:
for field in fields:
ds = data[g.id].pop(field)
nd = g.select(selector, ds, rv[field], ind) # caches
ind += nd
data.pop(g.id)
return rv
def _reconstruct_parent_child(self):
mask = np.empty(len(self.grids), dtype="int32")
mylog.debug("First pass; identifying child grids")
for i, grid in enumerate(self.grids):
get_box_grids_level(
self.grid_left_edge[i, :],
self.grid_right_edge[i, :],
self.grid_levels[i] + 1,
self.grid_left_edge,
self.grid_right_edge,
self.grid_levels,
mask,
)
grid.Children = [
g for g in self.grids[mask.astype("bool")] if g.Level == grid.Level + 1
]
mylog.debug("Second pass; identifying parents")
for i, grid in enumerate(self.grids): # Second pass
for child in grid.Children:
child.Parent.append(grid)
def get_contiguous_chunk(self, left_key, right_key, fields):
lbase=0
if left_key > self._max_key:
raise RuntimeError("Left key is too large. Key: %i Max Key: %i" % \
(left_key, self._max_key))
right_key = min(right_key, self._max_key)
left_key = self.get_next_nonzero_chunk(left_key, right_key-1)
right_key = self.get_previous_nonzero_chunk(right_key, left_key)
lbase = self.indexdata['base'][left_key]
rbase = self.indexdata['base'][right_key]
rlen = self.indexdata['len'][right_key]
length = rbase + rlen - lbase
if length > 0:
mylog.debug('Getting contiguous chunk of size %i starting at %i' % (length, lbase))
return self.get_data(slice(lbase, lbase + length), fields)
def deposit(self, positions, fields = None, method = None,
kernel_name = 'cubic'):
# Here we perform our particle deposition.
if fields is None: fields = []
cls = getattr(particle_deposit, "deposit_%s" % method, None)
if cls is None:
raise YTParticleDepositionNotImplemented(method)
nz = self.nz
nvals = (nz, nz, nz, self.ires.size)
# We allocate number of zones, not number of octs
op = cls(nvals, kernel_name)
op.initialize()
mylog.debug("Depositing %s (%s^3) particles into %s Root Mesh",
positions.shape[0], positions.shape[0]**0.3333333, nvals[-1])
pos = np.array(positions, dtype="float64")
f64 = [np.array(f, dtype="float64") for f in fields]
self.oct_handler.deposit(op, self.base_selector, pos, f64)
vals = op.finalize()
if vals is None: return
return np.asfortranarray(vals)
def _read_fluid_selection(self, chunks, selector, fields, size):
chunks = list(chunks)
if any((ftype != "boxlib" for ftype, fname in fields)):
raise NotImplementedError
rv = {}
for field in fields:
rv[field] = np.empty(size, dtype="float64")
ng = sum(len(c.objs) for c in chunks)
mylog.debug("Reading %s cells of %s fields in %s grids",
size, [f2 for f1, f2 in fields], ng)
ind = 0
for chunk in chunks:
data = self._read_chunk_data(chunk, fields)
for g in chunk.objs:
for field in fields:
ds = data[g.id].pop(field)
nd = g.select(selector, ds, rv[field], ind) # caches
ind += nd
data.pop(g.id)
return rv
def _read_amr_level(self, oct_handler):
"""Open the oct file, read in octs level-by-level.
For each oct, only the position, index, level and domain
are needed - its position in the octree is found automatically.
The most important is finding all the information to feed
oct_handler.add
"""
self.level_offsets
f = open(self.ds._file_amr, "rb")
for level in range(1, self.ds.max_level + 1):
unitary_center, fl, iocts, nocts, root_level = \
_read_art_level_info( f,
self._level_oct_offsets, level,
coarse_grid=self.ds.domain_dimensions[0],
root_level=self.ds.root_level)
nocts_check = oct_handler.add(self.domain_id, level,
unitary_center)
assert (nocts_check == nocts)
mylog.debug("Added %07i octs on level %02i, cumulative is %07i",
nocts, level, oct_handler.nocts)
def get_cell_data(self, level, cell_iarr, fields):
"""
Get data from requested cell
This uses the raw cell index, and doesn't account for periodicity or
an expanded domain (non-power of 2).
level: int
Requested level
cell_iarr: array-like, length 3
Requested cell from given level. fields: list
Requested fields
Returns:
cell_data: dict
Dictionary of field_name, field_data
"""
cell_iarr = np.array(cell_iarr, dtype="int64")
lk, rk = self.get_key_bounds(level, cell_iarr)
mylog.debug("Reading contiguous chunk from %i to %i" % (lk, rk))
return self.get_contiguous_chunk(lk, rk, fields)
def find_min(self, field):
"""
Returns (value, center) of location of minimum for a given field
"""
gI = np.where(self.grid_levels >= 0) # Slow but pedantic
minVal = HUGE
for grid in self.grids[gI[0]]:
mylog.debug("Checking %s (level %s)", grid.id, grid.Level)
val, coord = grid.find_min(field)
if val < minVal:
minCoord = coord
minVal = val
minGrid = grid
mc = np.array(minCoord)
pos=minGrid.get_position(mc)
mylog.info("Min Value is %0.5e at %0.16f %0.16f %0.16f in grid %s at level %s", \
minVal, pos[0], pos[1], pos[2], minGrid, minGrid.Level)
self.center = pos
self.parameters["Min%sValue" % (field)] = minVal
self.parameters["Min%sPos" % (field)] = "%s" % (pos)
return minVal, pos
def check_tree(self):
for node in self.trunk.depth_traverse():
if node.grid == -1:
continue
grid = self.ds.index.grids[node.grid - self._id_offset]
dds = grid.dds
gle = grid.LeftEdge
nle = self.ds.arr(node.get_left_edge(), input_units="code_length")
nre = self.ds.arr(node.get_right_edge(), input_units="code_length")
li = np.rint((nle-gle)/dds).astype('int32')
ri = np.rint((nre-gle)/dds).astype('int32')
dims = (ri - li).astype('int32')
assert(np.all(grid.LeftEdge <= nle))
assert(np.all(grid.RightEdge >= nre))
assert(np.all(dims > 0))
# print grid, dims, li, ri
# Calculate the Volume
vol = self.trunk.kd_sum_volume()
mylog.debug('AMRKDTree volume = %e' % vol)
self.trunk.kd_node_check()
def get_cell_data(self, level, cell_iarr, fields):
"""
Get data from requested cell
This uses the raw cell index, and doesn't account for periodicity or
an expanded domain (non-power of 2).
level: int
Requested level
cell_iarr: array-like, length 3
Requested cell from given level. fields: list
Requested fields
Returns:
cell_data: dict
Dictionary of field_name, field_data
"""
cell_iarr = np.array(cell_iarr, dtype="int64")
lk, rk =self.get_key_bounds(level, cell_iarr)
mylog.debug("Reading contiguous chunk from %i to %i" % (lk, rk))
return self.get_contiguous_chunk(lk, rk, fields)
def iter_data(self, inds, fields):
num_inds = len(inds)
num_reads = 0
mylog.debug('MIDX Reading %i chunks' % num_inds)
i = 0
while (i < num_inds):
ind = inds[i]
base = self.indexdata['base'][ind]
length = self.indexdata['len'][ind]
# Concatenate aligned reads
nexti = i+1
combined = 0
while nexti < num_inds:
nextind = inds[nexti]
# print 'b: %i l: %i end: %i next: %i' % ( base, length, base + length, self.indexdata['base'][nextind] )
if combined < 1024 and base + length == self.indexdata['base'][nextind]:
length += self.indexdata['len'][nextind]
i += 1
nexti += 1
combined += 1
else:
break
chunk = slice(base, base+length)
mylog.debug('Reading chunk %i of length %i after catting %i starting at %i' % (i, length, combined, ind))
num_reads += 1
if length > 0:
data = self.get_data(chunk, fields)
yield data
del data
i += 1
mylog.debug('Read %i chunks, batched into %i reads' % (num_inds, num_reads))
def iter_data(self, inds, fields):
num_inds = len(inds)
num_reads = 0
mylog.debug('MIDX Reading %i chunks' % num_inds)
i = 0
while (i < num_inds):
ind = inds[i]
base = self.indexdata['base'][ind]
length = self.indexdata['len'][ind]
# Concatenate aligned reads
nexti = i+1
combined = 0
while nexti < num_inds:
nextind = inds[nexti]
# print 'b: %i l: %i end: %i next: %i' % ( base, length, base + length, self.indexdata['base'][nextind] )
if combined < 1024 and base + length == self.indexdata['base'][nextind]:
length += self.indexdata['len'][nextind]
i += 1
nexti += 1
combined += 1
else:
break
chunk = slice(base, base+length)
mylog.debug('Reading chunk %i of length %i after catting %i starting at %i' % (i, length, combined, ind))
num_reads += 1
if length > 0:
data = self.get_data(chunk, fields)
yield data
del data
i += 1
mylog.debug('Read %i chunks, batched into %i reads' % (num_inds, num_reads))
def check_derived_fields(self, fields_to_check=None):
deps = {}
unavailable = []
fields_to_check = fields_to_check or self.keys()
for field in fields_to_check:
mylog.debug("Checking %s", field)
if field not in self: raise RuntimeError
fi = self[field]
try:
fd = fi.get_dependencies(ds=self.ds)
except Exception as e:
if field in self._show_field_errors:
raise
if type(e) != YTFieldNotFound:
mylog.debug("Raises %s during field %s detection.",
str(type(e)), field)
continue
# This next bit checks that we can't somehow generate everything.
# We also manually update the 'requested' attribute
missing = not all(f in self.field_list for f in fd.requested)
if missing:
self.pop(field)
unavailable.append(field)
continue
fd.requested = set(fd.requested)
deps[field] = fd
mylog.debug("Succeeded with %s (needs %s)", field, fd.requested)
dfl = set(self.ds.derived_field_list).union(deps.keys())
self.ds.derived_field_list = list(sorted(dfl))
return deps, unavailable
def check_derived_fields(self, fields_to_check = None):
deps = {}
unavailable = []
fields_to_check = fields_to_check or list(self.keys())
for field in fields_to_check:
mylog.debug("Checking %s", field)
if field not in self: raise RuntimeError
fi = self[field]
try:
fd = fi.get_dependencies(ds = self.ds)
except Exception as e:
if field in self._show_field_errors:
raise
if type(e) != YTFieldNotFound:
mylog.debug("Raises %s during field %s detection.",
str(type(e)), field)
self.pop(field)
continue
# This next bit checks that we can't somehow generate everything.
# We also manually update the 'requested' attribute
missing = not all(f in self.field_list for f in fd.requested)
if missing:
self.pop(field)
unavailable.append(field)
continue
fd.requested = set(fd.requested)
deps[field] = fd
mylog.debug("Succeeded with %s (needs %s)", field, fd.requested)
dfl = set(self.ds.derived_field_list).union(deps.keys())
self.ds.derived_field_list = list(sorted(dfl))
return deps, unavailable
def __init__(self, ds, domain_id):
self.ds = ds
self.domain_id = domain_id
num = os.path.basename(
ds.parameter_filename).split(".")[0].split("_")[1]
basedir = os.path.abspath(os.path.dirname(ds.parameter_filename))
basename = "%s/%%s_%s.out%05i" % (basedir, num, domain_id)
part_file_descriptor = "%s/part_file_descriptor.txt" % basedir
for t in ['grav', 'amr']:
setattr(self, "%s_fn" % t, basename % t)
self._part_file_descriptor = part_file_descriptor
self._read_amr_header()
# self._read_hydro_header()
# Autodetect field files
field_handlers = [
FH(self) for FH in get_field_handlers() if FH.any_exist(ds)
]
self.field_handlers = field_handlers
for fh in field_handlers:
mylog.debug('Detected fluid type %s in domain_id=%s' %
(fh.ftype, domain_id))
fh.detect_fields(ds)
# self._add_ftype(fh.ftype)
# Autodetect particle files
particle_handlers = [
PH(ds, domain_id) for PH in get_particle_handlers()
if PH.any_exist(ds)
]
self.particle_handlers = particle_handlers
for ph in particle_handlers:
mylog.debug('Detected particle type %s in domain_id=%s' %
(ph.ptype, domain_id))
ph.read_header()
# self._add_ptype(ph.ptype)
# Load the AMR structure
self._read_amr()
def _initialize_index(self, data_file, regions):
if self.index_ptype == "all":
ptypes = self.ds.particle_types_raw
pcount = sum(data_file.total_particles.values())
else:
ptypes = [self.index_ptype]
pcount = data_file.total_particles[self.index_ptype]
morton = np.empty(pcount, dtype='uint64')
if pcount == 0: return morton
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
dx = np.finfo(f["Group"]["GroupPos"].dtype).eps
dx = 2.0 * self.ds.quan(dx, "code_length")
for ptype in ptypes:
if data_file.total_particles[ptype] == 0: continue
pos = f[ptype]["%sPos" % ptype].value.astype("float64")
pos = np.resize(pos, (data_file.total_particles[ptype], 3))
pos = data_file.ds.arr(pos, "code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
ind += pos.shape[0]
return morton
def filter_bbox(self, left, right, myiter):
"""
Filter data by masking out data outside of a bbox defined
by left/right. Account for periodicity of data, allowing left/right
to be outside of the domain.
"""
for data in myiter:
# mask = np.zeros_like(data, dtype='bool')
pos = np.array([data["x"].copy(), data["y"].copy(), data["z"].copy()]).T
DW = self.true_domain_width
# This hurts, but is useful for periodicity. Probably should check first
# if it is even needed for a given left/right
_shift_periodic(pos, left, right, DW)
# Now get all particles that are within the bbox
mask = np.all(pos >= left, axis=1) * np.all(pos < right, axis=1)
# print('Mask shape, sum:', mask.shape, mask.sum())
mylog.debug(
"Filtering particles, returning %i out of %i"
% (mask.sum(), mask.shape[0])
)
if not np.any(mask):
continue
filtered = {ax: pos[:, i][mask] for i, ax in enumerate("xyz")}
for f in data.keys():
if f in "xyz":
continue
filtered[f] = data[f][mask]
# for i, ax in enumerate('xyz'):
# #print(left, right)
# assert np.all(filtered[ax] >= left[i])
# assert np.all(filtered[ax] < right[i])
yield filtered
def _initialize_oct_handler(self):
"""
Just count the number of octs per domain and
allocate the requisite memory in the oct tree
"""
nv = len(self.fluid_field_list)
self.oct_handler = ARTOctreeContainer(
self.dataset.domain_dimensions / 2, # dd is # of root cells
self.dataset.domain_left_edge,
self.dataset.domain_right_edge,
1)
# The 1 here refers to domain_id == 1 always for ARTIO.
self.domains = [ARTDomainFile(self.dataset, nv, self.oct_handler, 1)]
self.octs_per_domain = [dom.level_count.sum() for dom in self.domains]
self.total_octs = sum(self.octs_per_domain)
mylog.debug("Allocating %s octs", self.total_octs)
self.oct_handler.allocate_domains(self.octs_per_domain)
domain = self.domains[0]
domain._read_amr_root(self.oct_handler)
domain._read_amr_level(self.oct_handler)
self.oct_handler.finalize()
def __init__(self, points, ds=None, field_parameters=None, data_source=None):
validate_object(ds, Dataset)
validate_object(field_parameters, dict)
validate_object(data_source, YTSelectionContainer)
validate_object(points, YTArray)
points = fix_length(points, ds)
if len(points) < 2:
raise YTException(
f"Not enough points. Expected at least 2, got {len(points)}"
)
mylog.debug("Building minimal sphere around points.")
mb = _miniball.Miniball(points)
if not mb.is_valid():
raise YTException("Could not build valid sphere around points.")
center = ds.arr(mb.center(), points.units)
radius = ds.quan(np.sqrt(mb.squared_radius()), points.units)
super(YTMinimalSphere, self).__init__(center, ds, field_parameters, data_source)
self.set_field_parameter("radius", radius)
self.set_field_parameter("center", self.center)
self.radius = radius
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
if pcount == 0:
return morton
ind = 0
ptype = 'halos'
with open(data_file.filename, "rb") as f:
pos = data_file._get_particle_positions(ptype, f=f)
pos = data_file.ds.arr(pos, "code_length")
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0), pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind + pos.shape[0]] = compute_morton(
pos[:, 0], pos[:, 1], pos[:, 2], data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
return morton
def filter_bbox(self, left, right, myiter):
"""
Filter data by masking out data outside of a bbox defined
by left/right. Account for periodicity of data, allowing left/right
to be outside of the domain.
"""
for data in myiter:
#mask = np.zeros_like(data, dtype='bool')
pos = np.array([data['x'].copy(), data['y'].copy(), data['z'].copy()]).T
DW = self.true_domain_width
# This hurts, but is useful for periodicity. Probably should check first
# if it is even needed for a given left/right
_shift_periodic(pos, left, right, DW)
# Now get all particles that are within the bbox
mask = np.all(pos >= left, axis=1) * np.all(pos < right, axis=1)
#print 'Mask shape, sum:', mask.shape, mask.sum()
mylog.debug("Filtering particles, returning %i out of %i" % (mask.sum(), mask.shape[0]))
if not np.any(mask):
continue
filtered = {ax: pos[:, i][mask] for i, ax in enumerate('xyz')}
for f in data.keys():
if f in 'xyz':
continue
filtered[f] = data[f][mask]
#for i, ax in enumerate('xyz'):
# #print left, right
# assert np.all(filtered[ax] >= left[i])
# assert np.all(filtered[ax] < right[i])
yield filtered
def _initialize_index(self, data_file, regions):
pcount = data_file.header["num_halos"]
morton = np.empty(pcount, dtype='uint64')
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
if pcount == 0: return morton
ind = 0
with open(data_file.filename, "rb") as f:
f.seek(data_file._position_offset, os.SEEK_SET)
halos = np.fromfile(f, dtype=self._halo_dt, count = pcount)
pos = np.empty((halos.size, 3), dtype="float64")
# These positions are in Mpc, *not* "code" units
pos = data_file.ds.arr(pos, "code_length")
dx = np.finfo(halos['particle_position_x'].dtype).eps
dx = 2.0*self.ds.quan(dx, "code_length")
pos[:,0] = halos["particle_position_x"]
pos[:,1] = halos["particle_position_y"]
pos[:,2] = halos["particle_position_z"]
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
del halos
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind+pos.shape[0]] = compute_morton(
pos[:,0], pos[:,1], pos[:,2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
return morton
def get_contiguous_chunk(self, left_key, right_key, fields):
liarr = self.get_ind_from_key(left_key)
riarr = self.get_ind_from_key(right_key)
lbase=0
llen = 0
if left_key > self._max_key:
raise RuntimeError("Left key is too large. Key: %i Max Key: %i" % \
(left_key, self._max_key))
right_key = min(right_key, self._max_key)
left_key = self.get_next_nonzero_chunk(left_key, right_key-1)
right_key = self.get_previous_nonzero_chunk(right_key, left_key)
lbase = self.indexdata['base'][left_key]
llen = self.indexdata['len'][left_key]
rbase = self.indexdata['base'][right_key]
rlen = self.indexdata['len'][right_key]
length = rbase + rlen - lbase
if length > 0:
mylog.debug('Getting contiguous chunk of size %i starting at %i' % (length, lbase))
return self.get_data(slice(lbase, lbase + length), fields)
def _initialize_index(self, data_file, regions):
pcount = sum(data_file.total_particles.values())
morton = np.empty(pcount, dtype='uint64')
if pcount == 0: return morton
mylog.debug("Initializing index % 5i (% 7i particles)",
data_file.file_id, pcount)
ind = 0
with h5py.File(data_file.filename, "r") as f:
if not f.keys(): return None
dx = np.finfo(f["Group"]["GroupPos"].dtype).eps
dx = 2.0*self.ds.quan(dx, "code_length")
for ptype in data_file.ds.particle_types_raw:
if data_file.total_particles[ptype] == 0: continue
pos = f[ptype]["%sPos" % ptype].value.astype("float64")
pos = np.resize(pos, (data_file.total_particles[ptype], 3))
pos = data_file.ds.arr(pos, "code_length")
# These are 32 bit numbers, so we give a little lee-way.
# Otherwise, for big sets of particles, we often will bump into the
# domain edges. This helps alleviate that.
np.clip(pos, self.ds.domain_left_edge + dx,
self.ds.domain_right_edge - dx, pos)
if np.any(pos.min(axis=0) < self.ds.domain_left_edge) or \
np.any(pos.max(axis=0) > self.ds.domain_right_edge):
raise YTDomainOverflow(pos.min(axis=0),
pos.max(axis=0),
self.ds.domain_left_edge,
self.ds.domain_right_edge)
regions.add_data_file(pos, data_file.file_id)
morton[ind:ind+pos.shape[0]] = compute_morton(
pos[:,0], pos[:,1], pos[:,2],
data_file.ds.domain_left_edge,
data_file.ds.domain_right_edge)
ind += pos.shape[0]
return morton
def iter_filtered_bbox_fields(self, left, right, data,
pos_fields, fields):
"""
This function should be destroyed, as it will only work with units.
"""
kpcuq = left.in_units('kpccm').uq
mpcuq = left.in_units('Mpccm/h').uq
DW = (self.true_domain_width * kpcuq).in_units('Mpc/h')
if pos_fields is None:
pos_fields = 'x','y','z'
xf, yf, zf = pos_fields
mylog.debug("Using position fields: %s" % pos_fields)
# I'm sorry.
pos = mpcuq * np.array([data[xf].in_units('Mpccm/h'), data[yf].in_units('Mpccm/h'), data[zf].in_units('Mpccm/h')]).T
# This hurts, but is useful for periodicity. Probably should check first
# if it is even needed for a given left/right
_shift_periodic(pos, left, right, DW)
mylog.debug("Periodic filtering, %s %s %s %s" % (left, right, pos.min(axis=0), pos.max(axis=0)))
# Now get all particles that are within the bbox
mask = np.all(pos >= left, axis=1) * np.all(pos < right, axis=1)
mylog.debug("Filtering particles, returning %i out of %i" % (mask.sum(), mask.shape[0]))
if np.any(mask):
for i,f in enumerate(pos_fields):
yield f, pos[:, i][mask]
for f in fields:
if f in pos_fields:
continue
# print 'yielding nonpos field', f
yield f, data[f][mask]
def _read_fluid_selection(self, chunks, selector, fields, size):
from sys import version
rv = {}
chunks = list(chunks)
if selector.__class__.__name__ == "GridSelector":
if not (len(chunks) == len(chunks[0].objs) == 1):
raise RuntimeError
grid = chunks[0].objs[0]
h5f = h5py.File(grid.filename, 'r')
gds = h5f.get(_grid_dname(grid.id))
for ftype, fname in fields:
if self.ds.field_ordering == 1:
rv[(ftype, fname)] = gds.get(fname).value.swapaxes(0, 2)
else:
rv[(ftype, fname)] = gds.get(fname).value
h5f.close()
return rv
if size is None:
size = sum((grid.count(selector) for chunk in chunks
for grid in chunk.objs))
if any((ftype != "gdf" for ftype, fname in fields)):
raise NotImplementedError
for field in fields:
ftype, fname = field
fsize = size
# check the dtype instead
rv[field] = np.empty(fsize, dtype="float64")
ngrids = sum(len(chunk.objs) for chunk in chunks)
mylog.debug("Reading %s cells of %s fields in %s blocks",
size, [fname for ftype, fname in fields], ngrids)
ind = 0
for chunk in chunks:
fid = None
for grid in chunk.objs:
if grid.filename is None:
continue
if fid is None:
if version < '3':
fid = h5py.h5f.open(grid.filename,h5py.h5f.ACC_RDONLY)
else:
fid = h5py.h5f.open(bytes(grid.filename,'utf-8'),h5py.h5f.ACC_RDONLY)
if self.ds.field_ordering == 1:
# check the dtype instead
data = np.empty(grid.ActiveDimensions[::-1],
dtype="float64")
data_view = data.swapaxes(0, 2)
else:
# check the dtype instead
data_view = data = np.empty(grid.ActiveDimensions,
dtype="float64")
for field in fields:
ftype, fname = field
if version < '3':
dg = h5py.h5d.open(fid, _field_dname(grid.id, fname))
else:
dg = h5py.h5d.open(fid, bytes(_field_dname(grid.id, fname),'utf-8'))
dg.read(h5py.h5s.ALL, h5py.h5s.ALL, data)
# caches
nd = grid.select(selector, data_view, rv[field], ind)
ind += nd # I don't get that part, only last nd is added
if fid is not None:
fid.close()
return rv
def iter_ibbox_data(self, left, right, fields):
mylog.debug('MIDX Loading region from %s to %s' %(left, right))
inds = self.get_ibbox(left, right)
return self.iter_data(inds, fields)
def set_bounds(self):
if ('x_min' in self.sdfdata.parameters and 'x_max' in self.sdfdata.parameters) or \
('theta_min' in self.sdfdata.parameters and 'theta_max' in self.sdfdata.parameters):
if 'x_min' in self.sdfdata.parameters:
rmin = np.array([self.sdfdata.parameters['x_min'],
self.sdfdata.parameters['y_min'],
self.sdfdata.parameters['z_min']])
rmax = np.array([self.sdfdata.parameters['x_max'],
self.sdfdata.parameters['y_max'],
self.sdfdata.parameters['z_max']])
elif 'theta_min' in self.sdfdata.parameters:
rmin = np.array([self.sdfdata.parameters['r_min'],
self.sdfdata.parameters['theta_min'],
self.sdfdata.parameters['phi_min']])
rmax = np.array([self.sdfdata.parameters['r_max'],
self.sdfdata.parameters['theta_max'],
self.sdfdata.parameters['phi_max']])
self._fix_rexact(rmin, rmax)
self.true_domain_left = self.rmin.copy()
self.true_domain_right = self.rmax.copy()
self.true_domain_width = self.rmax - self.rmin
self.domain_width = self.rmax - self.rmin
self.domain_dims = 1 << self.level
self.domain_buffer = 0
self.domain_active_dims = self.domain_dims
else:
mylog.debug("Setting up older data")
rx = self.sdfdata.parameters.get('Rx')
ry = self.sdfdata.parameters.get('Ry')
rz = self.sdfdata.parameters.get('Rz')
a = self.sdfdata.parameters.get("a", 1.0)
rmin = -a * np.array([rx, ry, rz])
rmax = a * np.array([rx, ry, rz])
self.true_domain_left = rmin.copy()
self.true_domain_right = rmax.copy()
self.true_domain_width = rmax - rmin
expand_root = 0.0
morton_xyz = self.sdfdata.parameters.get("morton_xyz", False)
if not morton_xyz:
mylog.debug("Accounting for wandering particles")
self.wandering_particles = True
ic_Nmesh = self.sdfdata.parameters.get('ic_Nmesh',0)
# Expand root for non power-of-2
if ic_Nmesh != 0:
f2 = 1<<int(np.log2(ic_Nmesh-1)+1)
if (f2 != ic_Nmesh):
expand_root = 1.0*f2/ic_Nmesh - 1.0;
mylog.debug("Expanding: %s, %s, %s" % (f2, ic_Nmesh, expand_root))
rmin *= (1.0 + expand_root)
rmax *= (1.0 + expand_root)
self._fix_rexact(rmin, rmax)
self.domain_width = self.rmax - self.rmin
self.domain_dims = 1 << self.level
self.domain_buffer = (self.domain_dims - int(self.domain_dims/(1.0 + expand_root)))/2
self.domain_active_dims = self.domain_dims - 2*self.domain_buffer
mylog.debug("MIDX rmin: %s, rmax: %s" % (self.rmin, self.rmax))
mylog.debug("MIDX: domain_width: %s, domain_dims: %s, domain_active_dims: %s " %
(self.domain_width, self.domain_dims, self.domain_active_dims))
def get_ibbox(self, ileft, iright):
"""
Given left and right indicies, return a mask and
set of offsets+lengths into the sdf data.
"""
#print 'Getting data from ileft to iright:', ileft, iright
ix, iy, iz = (iright-ileft)*1j
mylog.debug('MIDX IBBOX: %s %s %s %s %s' % (ileft, iright, ix, iy, iz))
# plus 1 that is sliced, plus a bit since mgrid is not inclusive
Z, Y, X = np.mgrid[ileft[2]:iright[2]+1.01,
ileft[1]:iright[1]+1.01,
ileft[0]:iright[0]+1.01]
mask = slice(0, -1, None)
X = X[mask, mask, mask].astype('int64').ravel()
Y = Y[mask, mask, mask].astype('int64').ravel()
Z = Z[mask, mask, mask].astype('int64').ravel()
if self.wandering_particles:
# Need to get padded bbox around the border to catch
# wandering particles.
dmask = X < self.domain_buffer
dmask += Y < self.domain_buffer
dmask += Z < self.domain_buffer
dmask += X >= self.domain_dims
dmask += Y >= self.domain_dims
dmask += Z >= self.domain_dims
dinds = self.get_keyv([X[dmask], Y[dmask], Z[dmask]])
dinds = dinds[dinds < self._max_key]
dinds = dinds[self.indexdata['len'][dinds] > 0]
#print 'Getting boundary layers for wanderers, cells: %i' % dinds.size
# Correct For periodicity
X[X < self.domain_buffer] += self.domain_active_dims
Y[Y < self.domain_buffer] += self.domain_active_dims
Z[Z < self.domain_buffer] += self.domain_active_dims
X[X >= self.domain_buffer + self.domain_active_dims] -= self.domain_active_dims
Y[Y >= self.domain_buffer + self.domain_active_dims] -= self.domain_active_dims
Z[Z >= self.domain_buffer + self.domain_active_dims] -= self.domain_active_dims
#print 'periodic:', X.min(), X.max(), Y.min(), Y.max(), Z.min(), Z.max()
indices = self.get_keyv([X, Y, Z])
# Only mask out if we are actually getting data rather than getting indices into
# a space.
if self.valid_indexdata:
indices = indices[indices < self._max_key]
#indices = indices[self.indexdata['len'][indices] > 0]
# Faster for sparse lookups. Need better heuristic.
new_indices = []
for ind in indices:
if self.indexdata['len'][ind] > 0:
new_indices.append(ind)
indices = np.array(indices, dtype="int64")
#indices = np.array([self.get_key_ijk(x, y, z) for x, y, z in zip(X, Y, Z)])
# Here we sort the indices to batch consecutive reads together.
if self.wandering_particles:
indices = np.sort(np.append(indices, dinds))
else:
indices = np.sort(indices)
return indices
