def __init__(self,
data_source,
x_field,
x_n,
x_min,
x_max,
x_log,
y_field,
y_n,
y_min,
y_max,
y_log,
z_field,
z_n,
z_min,
z_max,
z_log,
weight_field=None):
super(Profile3D, self).__init__(data_source, weight_field)
# X
self.x_field = data_source._determine_fields(x_field)[0]
self.x_log = x_log
self.field_info[self.x_field] = \
self.data_source.ds.field_info[self.x_field]
x_min, x_max = _sanitize_min_max_units(x_min, x_max,
self.field_info[self.x_field],
self.ds.unit_registry)
self.x_bins = array_like_field(
data_source, self._get_bins(x_min, x_max, x_n, x_log),
self.x_field)
# Y
self.y_field = data_source._determine_fields(y_field)[0]
self.y_log = y_log
self.field_info[self.y_field] = \
self.data_source.ds.field_info[self.y_field]
y_min, y_max = _sanitize_min_max_units(y_min, y_max,
self.field_info[self.y_field],
self.ds.unit_registry)
self.y_bins = array_like_field(
data_source, self._get_bins(y_min, y_max, y_n, y_log),
self.y_field)
# Z
self.z_field = data_source._determine_fields(z_field)[0]
self.z_log = z_log
self.field_info[self.z_field] = \
self.data_source.ds.field_info[self.z_field]
z_min, z_max = _sanitize_min_max_units(z_min, z_max,
self.field_info[self.z_field],
self.ds.unit_registry)
self.z_bins = array_like_field(
data_source, self._get_bins(z_min, z_max, z_n, z_log),
self.z_field)
self.size = (self.x_bins.size - 1, self.y_bins.size - 1,
self.z_bins.size - 1)
self.bin_fields = (self.x_field, self.y_field, self.z_field)
self.x = 0.5 * (self.x_bins[1:] + self.x_bins[:-1])
self.y = 0.5 * (self.y_bins[1:] + self.y_bins[:-1])
self.z = 0.5 * (self.z_bins[1:] + self.z_bins[:-1])
def __init__(self,
data_source,
x_field,
x_n,
x_min,
x_max,
x_log,
weight_field=None,
override_bins_x=None):
super(Profile1D, self).__init__(data_source, weight_field)
self.x_field = data_source._determine_fields(x_field)[0]
self.field_info[self.x_field] = \
self.data_source.ds.field_info[self.x_field]
self.x_log = x_log
x_min, x_max = _sanitize_min_max_units(x_min, x_max,
self.field_info[self.x_field],
self.ds.unit_registry)
self.x_bins = array_like_field(
data_source, self._get_bins(x_min, x_max, x_n, x_log),
self.x_field)
if override_bins_x is not None:
self.x_bins = array_like_field(data_source, override_bins_x,
self.x_field)
self.size = (self.x_bins.size - 1, )
self.bin_fields = (self.x_field, )
self.x = 0.5 * (self.x_bins[1:] + self.x_bins[:-1])
def process_chunk(self, data, field, sample_fields):
field = data._determine_fields(field)[0]
ma = array_like_field(data, self._sign * HUGE, field)
vals = [array_like_field(data, -1, sf) for sf in sample_fields]
maxi = -1
if data[field].size > 0:
maxi = self._func(data[field])
ma = data[field][maxi]
vals = [data[sf][maxi] for sf in sample_fields]
return (ma, ) + tuple(vals)
def test_array_like_field_output_units():
ds = load(ISOGAL)
ad = ds.all_data()
u1 = ad["particle_mass"].units
u2 = array_like_field(ad, 1., ("all", "particle_mass")).units
assert u1 == u2
assert str(u1) == ds.fields.all.particle_mass.output_units
u1 = ad['gas', 'x'].units
u2 = array_like_field(ad, 1., ("gas", "x")).units
assert u1 == u2
assert str(u1) == ds.fields.gas.x.units
def process_chunk(self, data, field):
field = data._determine_fields(field)[0]
ma = array_like_field(data, HUGE, field)
mx = array_like_field(data, -1, "x")
my = array_like_field(data, -1, "y")
mz = array_like_field(data, -1, "z")
mini = -1
if data[field].size > 0:
mini = np.argmin(data[field])
ma = data[field][mini]
mx, my, mz = [data[ax][mini] for ax in 'xyz']
return (ma, mini, mx, my, mz)
def process_chunk(self, data, field):
field = data._determine_fields(field)[0]
ma = array_like_field(data, HUGE, field)
mx = array_like_field(data, -1, "x")
my = array_like_field(data, -1, "y")
mz = array_like_field(data, -1, "z")
mini = -1
if data[field].size > 0:
mini = np.argmin(data[field])
ma = data[field][mini]
mx, my, mz = [data[ax][mini] for ax in 'xyz']
return (ma, mini, mx, my, mz)
def process_chunk(self, data, fields, non_zero):
vals = []
for field in fields:
field = data._determine_fields(field)[0]
fd = data[field]
if non_zero: fd = fd[fd > 0.0]
if fd.size > 0:
vals += [fd.min(), fd.max()]
else:
vals += [array_like_field(data, HUGE, field),
array_like_field(data, -HUGE, field)]
return vals
def process_chunk(self, data, fields, non_zero):
vals = []
for field in fields:
field = data._determine_fields(field)[0]
fd = data[field]
if non_zero: fd = fd[fd > 0.0]
if fd.size > 0:
vals += [fd.min(), fd.max()]
else:
vals += [array_like_field(data, HUGE, field),
array_like_field(data, -HUGE, field)]
return vals
def process_chunk(self, data, field):
axis_names = data.ds.coordinates.axis_name
field = data._determine_fields(field)[0]
ma = array_like_field(data, -HUGE, field)
mx = array_like_field(data, -1, axis_names[0])
my = array_like_field(data, -1, axis_names[1])
mz = array_like_field(data, -1, axis_names[2])
maxi = -1
if data[field].size > 0:
maxi = np.argmax(data[field])
ma = data[field][maxi]
mx, my, mz = [data[ax][maxi] for ax in (axis_names[0],
axis_names[1],
axis_names[2])]
return (ma, maxi, mx, my, mz)
def process_chunk(self, data, field):
axis_names = data.ds.coordinates.axis_name
field = data._determine_fields(field)[0]
ma = array_like_field(data, -HUGE, field)
mx = array_like_field(data, -1, axis_names[0])
my = array_like_field(data, -1, axis_names[1])
mz = array_like_field(data, -1, axis_names[2])
maxi = -1
if data[field].size > 0:
maxi = np.argmax(data[field])
ma = data[field][maxi]
mx, my, mz = [
data[ax][maxi]
for ax in (axis_names[0], axis_names[1], axis_names[2])
]
return (ma, maxi, mx, my, mz)
def _get_data(self, field):
"""
Get a field to include in the trajectory collection.
The trajectory collection itself is a dict of 2D numpy arrays,
with shape (num_indices, num_steps)
"""
if field not in self.field_data:
if self.suppress_logging:
old_level = int(ytcfg.get("yt", "loglevel"))
mylog.setLevel(40)
ds_first = self.data_series[0]
dd_first = ds_first.all_data()
fd = dd_first._determine_fields(field)[0]
if field not in self.particle_fields:
if self.data_series[0].field_info[fd].particle_type:
self.particle_fields.append(field)
particles = np.empty((self.num_indices, self.num_steps))
particles[:] = np.nan
step = int(0)
pbar = get_pbar("Generating field %s in trajectories." % (field),
self.num_steps)
my_storage = {}
for i, (sto, ds) in enumerate(
self.data_series.piter(storage=my_storage)):
mask = self.masks[i]
sort = self.sorts[i]
if field in self.particle_fields:
# This is easy... just get the particle fields
dd = ds.all_data()
pfield = dd[fd].ndarray_view()[mask][sort]
else:
# This is hard... must loop over grids
pfield = np.zeros((self.num_indices))
x = self["particle_position_x"][:, step].ndarray_view()
y = self["particle_position_y"][:, step].ndarray_view()
z = self["particle_position_z"][:, step].ndarray_view()
# This will fail for non-grid index objects
particle_grids, particle_grid_inds = ds.index._find_points(
x, y, z)
for grid in particle_grids:
cube = grid.retrieve_ghost_zones(1, [fd])
CICSample_3(
x, y, z, pfield, self.num_indices, cube[fd],
np.array(grid.LeftEdge).astype(np.float64),
np.array(grid.ActiveDimensions).astype(np.int32),
grid.dds[0])
sto.result_id = ds.parameter_filename
sto.result = (self.array_indices[i], pfield)
pbar.update(step)
step += 1
pbar.finish()
for i, (fn, (indices,
pfield)) in enumerate(sorted(my_storage.items())):
particles[indices, i] = pfield
self.field_data[field] = array_like_field(dd_first, particles, fd)
if self.suppress_logging:
mylog.setLevel(old_level)
return self.field_data[field]
def _get_data(self, field):
"""
Get a field to include in the trajectory collection.
The trajectory collection itself is a dict of 2D numpy arrays,
with shape (num_indices, num_steps)
"""
if field not in self.field_data:
if self.suppress_logging:
old_level = int(ytcfg.get("yt","loglevel"))
mylog.setLevel(40)
ds_first = self.data_series[0]
dd_first = ds_first.all_data()
fd = dd_first._determine_fields(field)[0]
if field not in self.particle_fields:
if self.data_series[0].field_info[fd].particle_type:
self.particle_fields.append(field)
particles = np.empty((self.num_indices,self.num_steps))
particles[:] = np.nan
step = int(0)
pbar = get_pbar("Generating field %s in trajectories." % (field), self.num_steps)
my_storage={}
for i, (sto, ds) in enumerate(self.data_series.piter(storage=my_storage)):
mask = self.masks[i]
sort = self.sorts[i]
if field in self.particle_fields:
# This is easy... just get the particle fields
dd = ds.all_data()
pfield = dd[fd].ndarray_view()[mask][sort]
else:
# This is hard... must loop over grids
pfield = np.zeros((self.num_indices))
x = self["particle_position_x"][:,step].ndarray_view()
y = self["particle_position_y"][:,step].ndarray_view()
z = self["particle_position_z"][:,step].ndarray_view()
# This will fail for non-grid index objects
particle_grids, particle_grid_inds = ds.index._find_points(x,y,z)
for grid in particle_grids:
cube = grid.retrieve_ghost_zones(1, [fd])
CICSample_3(x,y,z,pfield,
self.num_indices,
cube[fd],
np.array(grid.LeftEdge).astype(np.float64),
np.array(grid.ActiveDimensions).astype(np.int32),
grid.dds[0])
sto.result_id = ds.parameter_filename
sto.result = (self.array_indices[i], pfield)
pbar.update(step)
step += 1
pbar.finish()
for i, (fn, (indices, pfield)) in enumerate(sorted(my_storage.items())):
particles[indices,i] = pfield
self.field_data[field] = array_like_field(dd_first, particles, fd)
if self.suppress_logging:
mylog.setLevel(old_level)
return self.field_data[field]
def _finalize_storage(self, fields, temp_storage):
# We use our main comm here
# This also will fill _field_data
for i, field in enumerate(fields):
# q values are returned as q * weight but we want just q
temp_storage.qvalues[..., i][temp_storage.used] /= \
temp_storage.weight_values[temp_storage.used]
# get the profile data from all procs
all_store = {self.comm.rank: temp_storage}
all_store = self.comm.par_combine_object(all_store,
"join",
datatype="dict")
all_val = np.zeros_like(temp_storage.values)
all_mean = np.zeros_like(temp_storage.mvalues)
all_std = np.zeros_like(temp_storage.qvalues)
all_weight = np.zeros_like(temp_storage.weight_values)
all_used = np.zeros_like(temp_storage.used, dtype="bool")
# Combine the weighted mean and standard deviation from each processor.
# For two samples with total weight, mean, and standard deviation
# given by w, m, and s, their combined mean and standard deviation are:
# m12 = (m1 * w1 + m2 * w2) / (w1 + w2)
# s12 = (m1 * (s1**2 + (m1 - m12)**2) +
# m2 * (s2**2 + (m2 - m12)**2)) / (w1 + w2)
# Here, the mvalues are m and the qvalues are s**2.
for p in sorted(all_store.keys()):
all_used += all_store[p].used
old_mean = all_mean.copy()
old_weight = all_weight.copy()
all_weight[all_store[p].used] += \
all_store[p].weight_values[all_store[p].used]
for i, field in enumerate(fields):
all_val[..., i][all_store[p].used] += \
all_store[p].values[..., i][all_store[p].used]
all_mean[..., i][all_store[p].used] = \
(all_mean[..., i] * old_weight +
all_store[p].mvalues[..., i] *
all_store[p].weight_values)[all_store[p].used] / \
all_weight[all_store[p].used]
all_std[..., i][all_store[p].used] = \
(old_weight * (all_std[..., i] +
(old_mean[..., i] - all_mean[..., i])**2) +
all_store[p].weight_values *
(all_store[p].qvalues[..., i] +
(all_store[p].mvalues[..., i] -
all_mean[..., i])**2))[all_store[p].used] / \
all_weight[all_store[p].used]
all_std = np.sqrt(all_std)
del all_store
self.used = all_used
blank = ~all_used
self.weight = all_weight
self.weight[blank] = 0.0
for i, field in enumerate(fields):
if self.weight_field is None:
self.field_data[field] = \
array_like_field(self.data_source,
all_val[...,i], field)
else:
self.field_data[field] = \
array_like_field(self.data_source,
all_mean[...,i], field)
self.standard_deviation[field] = \
array_like_field(self.data_source,
all_std[...,i], field)
self.standard_deviation[field][blank] = 0.0
self.field_data[field][blank] = 0.0
self.field_units[field] = self.field_data[field].units
if isinstance(field, tuple):
self.field_map[field[1]] = field
else:
self.field_map[field] = field
def __init__(self,
outputs,
indices,
fields=None,
suppress_logging=False,
ptype=None):
indices.sort() # Just in case the caller wasn't careful
self.field_data = YTFieldData()
self.data_series = outputs
self.masks = []
self.sorts = []
self.array_indices = []
self.indices = indices
self.num_indices = len(indices)
self.num_steps = len(outputs)
self.times = []
self.suppress_logging = suppress_logging
self.ptype = ptype
if fields is None:
fields = []
fields = list(OrderedDict.fromkeys(fields))
if self.suppress_logging:
old_level = int(ytcfg.get("yt", "loglevel"))
mylog.setLevel(40)
ds_first = self.data_series[0]
dd_first = ds_first.all_data()
fds = {}
for field in (
"particle_index",
"particle_position_x",
"particle_position_y",
"particle_position_z",
):
fds[field] = self._get_full_field_name(field)[0]
my_storage = {}
pbar = get_pbar("Constructing trajectory information",
len(self.data_series))
for i, (sto,
ds) in enumerate(self.data_series.piter(storage=my_storage)):
dd = ds.all_data()
newtags = dd[fds["particle_index"]].d.astype("int64")
mask = np.in1d(newtags, indices, assume_unique=True)
sort = np.argsort(newtags[mask])
array_indices = np.where(
np.in1d(indices, newtags, assume_unique=True))[0]
self.array_indices.append(array_indices)
self.masks.append(mask)
self.sorts.append(sort)
pfields = {}
for field in (f"particle_position_{ax}" for ax in "xyz"):
pfields[field] = dd[fds[field]].ndarray_view()[mask][sort]
sto.result_id = ds.parameter_filename
sto.result = (ds.current_time, array_indices, pfields)
pbar.update(i)
pbar.finish()
if self.suppress_logging:
mylog.setLevel(old_level)
sorted_storage = sorted(my_storage.items())
times = [time for _fn, (time, *_) in sorted_storage]
self.times = self.data_series[0].arr(times, times[0].units)
self.particle_fields = []
output_field = np.empty((self.num_indices, self.num_steps))
output_field.fill(np.nan)
for field in (f"particle_position_{ax}" for ax in "xyz"):
for i, (_fn, (_time, indices,
pfields)) in enumerate(sorted_storage):
try:
# This will fail if particles ids are
# duplicate. This is due to the fact that the rhs
# would then have a different shape as the lhs
output_field[indices, i] = pfields[field]
except ValueError as e:
raise YTIllDefinedParticleData(
"This dataset contains duplicate particle indices!"
) from e
self.field_data[field] = array_like_field(dd_first,
output_field.copy(),
fds[field])
self.particle_fields.append(field)
# Instantiate fields the caller requested
self._get_data(fields)
def _get_data(self, fields):
"""
Get a list of fields to include in the trajectory collection.
The trajectory collection itself is a dict of 2D numpy arrays,
with shape (num_indices, num_steps)
"""
missing_fields = [
field for field in fields if field not in self.field_data
]
if not missing_fields:
return
if self.suppress_logging:
old_level = int(ytcfg.get("yt", "loglevel"))
mylog.setLevel(40)
ds_first = self.data_series[0]
dd_first = ds_first.all_data()
fds = {}
new_particle_fields = []
for field in missing_fields:
fds[field] = dd_first._determine_fields(field)[0]
if field not in self.particle_fields:
if self.data_series[0]._get_field_info(
*fds[field]).particle_type:
self.particle_fields.append(field)
new_particle_fields.append(field)
grid_fields = [
field for field in missing_fields
if field not in self.particle_fields
]
step = int(0)
pbar = get_pbar(
f"Generating [{', '.join(missing_fields)}] fields in trajectories",
self.num_steps,
)
my_storage = {}
for i, (sto,
ds) in enumerate(self.data_series.piter(storage=my_storage)):
mask = self.masks[i]
sort = self.sorts[i]
pfield = {}
if new_particle_fields: # there's at least one particle field
dd = ds.all_data()
for field in new_particle_fields:
# This is easy... just get the particle fields
pfield[field] = dd[fds[field]].d[mask][sort]
if grid_fields:
# This is hard... must loop over grids
for field in grid_fields:
pfield[field] = np.zeros(self.num_indices)
x = self["particle_position_x"][:, step].d
y = self["particle_position_y"][:, step].d
z = self["particle_position_z"][:, step].d
particle_grids, particle_grid_inds = ds.index._find_points(
x, y, z)
# This will fail for non-grid index objects
for grid in particle_grids:
cube = grid.retrieve_ghost_zones(1, grid_fields)
for field in grid_fields:
CICSample_3(
x,
y,
z,
pfield[field],
self.num_indices,
cube[fds[field]],
np.array(grid.LeftEdge).astype(np.float64),
np.array(grid.ActiveDimensions).astype(np.int32),
grid.dds[0],
)
sto.result_id = ds.parameter_filename
sto.result = (self.array_indices[i], pfield)
pbar.update(step)
step += 1
pbar.finish()
output_field = np.empty((self.num_indices, self.num_steps))
output_field.fill(np.nan)
for field in missing_fields:
fd = fds[field]
for i, (_fn, (indices,
pfield)) in enumerate(sorted(my_storage.items())):
output_field[indices, i] = pfield[field]
self.field_data[field] = array_like_field(dd_first,
output_field.copy(), fd)
if self.suppress_logging:
mylog.setLevel(old_level)
def __init__(self, outputs, indices, fields=None, suppress_logging=False):
indices.sort() # Just in case the caller wasn't careful
self.field_data = YTFieldData()
self.data_series = outputs
self.masks = []
self.sorts = []
self.array_indices = []
self.indices = indices
self.num_indices = len(indices)
self.num_steps = len(outputs)
self.times = []
self.suppress_logging = suppress_logging
if fields is None: fields = []
fields = list(OrderedDict.fromkeys(fields))
if self.suppress_logging:
old_level = int(ytcfg.get("yt","loglevel"))
mylog.setLevel(40)
fds = {}
ds_first = self.data_series[0]
dd_first = ds_first.all_data()
idx_field = dd_first._determine_fields("particle_index")[0]
for field in ("particle_position_%s" % ax for ax in "xyz"):
fds[field] = dd_first._determine_fields(field)[0]
my_storage = {}
pbar = get_pbar("Constructing trajectory information", len(self.data_series))
for i, (sto, ds) in enumerate(self.data_series.piter(storage=my_storage)):
dd = ds.all_data()
newtags = dd[idx_field].d.astype("int64")
mask = np.in1d(newtags, indices, assume_unique=True)
sort = np.argsort(newtags[mask])
array_indices = np.where(np.in1d(indices, newtags, assume_unique=True))[0]
self.array_indices.append(array_indices)
self.masks.append(mask)
self.sorts.append(sort)
pfields = {}
for field in ("particle_position_%s" % ax for ax in "xyz"):
pfields[field] = dd[fds[field]].ndarray_view()[mask][sort]
sto.result_id = ds.parameter_filename
sto.result = (ds.current_time, array_indices, pfields)
pbar.update(i)
pbar.finish()
if self.suppress_logging:
mylog.setLevel(old_level)
times = []
for fn, (time, indices, pfields) in sorted(my_storage.items()):
times.append(time)
self.times = self.data_series[0].arr([time for time in times], times[0].units)
self.particle_fields = []
output_field = np.empty((self.num_indices, self.num_steps))
output_field.fill(np.nan)
for field in ("particle_position_%s" % ax for ax in "xyz"):
for i, (fn, (time, indices, pfields)) in enumerate(sorted(my_storage.items())):
output_field[indices, i] = pfields[field]
self.field_data[field] = array_like_field(
dd_first, output_field.copy(), fds[field])
self.particle_fields.append(field)
# Instantiate fields the caller requested
self._get_data(fields)
def test_array_like_field():
ds = fake_random_ds(4, particles=64)
ad = ds.all_data()
u1 = ad["particle_mass"].units
u2 = array_like_field(ad, 1., ("all", "particle_mass")).units
assert u1 == u2
