def __init__(self, ds, min_level=None, max_level=None,
data_source=None):
ParallelAnalysisInterface.__init__(self)
self.ds = ds
self.current_vcds = []
self.current_saved_grids = []
self.bricks = []
self.brick_dimensions = []
self.sdx = ds.index.get_smallest_dx()
self._initialized = False
try:
self._id_offset = ds.index.grids[0]._id_offset
except AttributeError:
self._id_offset = 0
if data_source is None:
data_source = self.ds.all_data()
self.data_source = data_source
mylog.debug('Building AMRKDTree')
self.tree = Tree(ds, self.comm.rank, self.comm.size,
min_level=min_level, max_level=max_level,
data_source=data_source)
def __init__(self,
ts,
num_readers=1,
num_writers=None,
outbase="rockstar_halos",
particle_type="all",
force_res=None,
total_particles=None,
dm_only=False,
particle_mass=None,
min_halo_size=25):
if is_root():
mylog.info(
"The citation for the Rockstar halo finder can be found at")
mylog.info("http://adsabs.harvard.edu/abs/2013ApJ...762..109B")
ParallelAnalysisInterface.__init__(self)
# Decide how we're working.
if ytcfg.getboolean("yt", "inline") is True:
self.runner = InlineRunner()
else:
self.runner = StandardRunner(num_readers, num_writers)
self.num_readers = self.runner.num_readers
self.num_writers = self.runner.num_writers
mylog.info("Rockstar is using %d readers and %d writers",
self.num_readers, self.num_writers)
# Note that Rockstar does not support subvolumes.
# We assume that all of the snapshots in the time series
# use the same domain info as the first snapshots.
if not isinstance(ts, DatasetSeries):
ts = DatasetSeries([ts])
self.ts = ts
self.particle_type = particle_type
self.outbase = six.b(outbase)
self.min_halo_size = min_halo_size
if force_res is None:
tds = ts[-1] # Cache a reference
self.force_res = tds.index.get_smallest_dx().in_units("Mpc/h")
# We have to delete now to wipe the index
del tds
else:
self.force_res = force_res
self.total_particles = total_particles
self.dm_only = dm_only
self.particle_mass = particle_mass
# Setup pool and workgroups.
self.pool, self.workgroup = self.runner.setup_pool()
p = self._setup_parameters(ts)
params = self.comm.mpi_bcast(p, root=self.pool['readers'].ranks[0])
self.__dict__.update(params)
self.handler = rockstar_interface.RockstarInterface(self.ts)
def __init__(self, data_source, weight_field=None):
self.data_source = data_source
self.ds = data_source.ds
self.field_map = {}
self.field_info = {}
self.field_data = YTFieldData()
if weight_field is not None:
self.standard_deviation = YTFieldData()
weight_field = self.data_source._determine_fields(weight_field)[0]
else:
self.standard_deviation = None
self.weight_field = weight_field
self.field_units = {}
ParallelAnalysisInterface.__init__(self, comm=data_source.comm)
def __init__(self,
halos_ds=None,
data_ds=None,
data_source=None,
finder_method=None,
finder_kwargs=None,
output_dir="halo_catalogs/catalog"):
ParallelAnalysisInterface.__init__(self)
self.halos_ds = halos_ds
self.data_ds = data_ds
self.output_dir = ensure_dir(output_dir)
if os.path.basename(self.output_dir) != ".":
self.output_prefix = os.path.basename(self.output_dir)
else:
self.output_prefix = "catalog"
if halos_ds is None:
if data_ds is None:
raise RuntimeError(
"Must specify a halos_ds, data_ds, or both.")
if finder_method is None:
raise RuntimeError(
"Must specify a halos_ds or a finder_method.")
if data_source is None:
if halos_ds is not None:
halos_ds.index
data_source = halos_ds.all_data()
else:
data_source = data_ds.all_data()
self.data_source = data_source
self.finder_method_name = finder_method
if finder_kwargs is None:
finder_kwargs = {}
if finder_method is not None:
finder_method = finding_method_registry.find(
finder_method, **finder_kwargs)
self.finder_method = finder_method
# all of the analysis actions to be performed: callbacks, filters, and quantities
self.actions = []
# fields to be written to the halo catalog
self.quantities = []
if self.halos_ds is not None:
self.add_default_quantities()
def __init__(
self,
ds,
positions,
xfield="velocity_x",
yfield="velocity_x",
zfield="velocity_x",
volume=None,
dx=None,
length=None,
direction=1,
get_magnitude=False,
):
ParallelAnalysisInterface.__init__(self)
self.ds = ds
self.start_positions = sanitize_length(positions, ds)
self.N = self.start_positions.shape[0]
# I need a data object to resolve the field names to field tuples
# via _determine_fields()
ad = self.ds.all_data()
self.xfield = ad._determine_fields(xfield)[0]
self.yfield = ad._determine_fields(yfield)[0]
self.zfield = ad._determine_fields(zfield)[0]
self.get_magnitude = get_magnitude
self.direction = np.sign(direction)
if volume is None:
volume = AMRKDTree(self.ds)
volume.set_fields([self.xfield, self.yfield, self.zfield],
[False, False, False], False)
volume.join_parallel_trees()
self.volume = volume
if dx is None:
dx = self.ds.index.get_smallest_dx()
self.dx = sanitize_length(dx, ds)
if length is None:
length = np.max(self.ds.domain_right_edge -
self.ds.domain_left_edge)
self.length = sanitize_length(length, ds)
self.steps = int(self.length / self.dx) + 1
# Fix up the dx.
self.dx = 1.0 * self.length / self.steps
self.streamlines = np.zeros((self.N, self.steps, 3), dtype="float64")
self.magnitudes = None
if self.get_magnitude:
self.magnitudes = np.zeros((self.N, self.steps), dtype="float64")
def __init__(self, ts, num_readers = 1, num_writers = None,
outbase="rockstar_halos", particle_type="all",
force_res=None, total_particles=None, dm_only=False,
particle_mass=None, min_halo_size=25):
if is_root():
mylog.info("The citation for the Rockstar halo finder can be found at")
mylog.info("http://adsabs.harvard.edu/abs/2013ApJ...762..109B")
ParallelAnalysisInterface.__init__(self)
# Decide how we're working.
if ytcfg.getboolean("yt", "inline") == True:
self.runner = InlineRunner()
else:
self.runner = StandardRunner(num_readers, num_writers)
self.num_readers = self.runner.num_readers
self.num_writers = self.runner.num_writers
mylog.info("Rockstar is using %d readers and %d writers",
self.num_readers, self.num_writers)
# Note that Rockstar does not support subvolumes.
# We assume that all of the snapshots in the time series
# use the same domain info as the first snapshots.
if not isinstance(ts, DatasetSeries):
ts = DatasetSeries([ts])
self.ts = ts
self.particle_type = particle_type
self.outbase = outbase
self.min_halo_size = min_halo_size
if force_res is None:
tds = ts[-1] # Cache a reference
self.force_res = tds.index.get_smallest_dx().in_units("Mpc/h")
# We have to delete now to wipe the index
del tds
else:
self.force_res = force_res
self.total_particles = total_particles
self.dm_only = dm_only
self.particle_mass = particle_mass
# Setup pool and workgroups.
self.pool, self.workgroup = self.runner.setup_pool()
p = self._setup_parameters(ts)
params = self.comm.mpi_bcast(p, root = self.pool['readers'].ranks[0])
self.__dict__.update(params)
self.handler = rockstar_interface.RockstarInterface(self.ts)
def __init__(self, ds, dataset_type):
ParallelAnalysisInterface.__init__(self)
self.dataset = weakref.proxy(ds)
self.ds = self.dataset
self._initialize_state_variables()
mylog.debug("Initializing data storage.")
self._initialize_data_storage()
mylog.debug("Setting up domain geometry.")
self._setup_geometry()
mylog.debug("Initializing data grid data IO")
self._setup_data_io()
# Note that this falls under the "geometry" object since it's
# potentially quite expensive, and should be done with the indexing.
mylog.debug("Detecting fields.")
self._detect_output_fields()
def __init__(self, ds, dataset_type):
ParallelAnalysisInterface.__init__(self)
self.dataset = weakref.proxy(ds)
self.ds = self.dataset
self._initialize_state_variables()
mylog.debug("Initializing data storage.")
self._initialize_data_storage()
mylog.debug("Setting up domain geometry.")
self._setup_geometry()
mylog.debug("Initializing data grid data IO")
self._setup_data_io()
# Note that this falls under the "geometry" object since it's
# potentially quite expensive, and should be done with the indexing.
mylog.debug("Detecting fields.")
self._detect_output_fields()
def __init__(self,
ds,
positions,
xfield='velocity_x',
yfield='velocity_x',
zfield='velocity_x',
volume=None,
dx=None,
length=None,
direction=1,
get_magnitude=False):
ParallelAnalysisInterface.__init__(self)
self.ds = ds
self.start_positions = np.array(positions)
self.N = self.start_positions.shape[0]
self.xfield = xfield
self.yfield = yfield
self.zfield = zfield
self.get_magnitude = get_magnitude
self.direction = np.sign(direction)
if volume is None:
volume = AMRKDTree(self.ds)
volume.set_fields([self.xfield, self.yfield, self.zfield],
[False, False, False], False)
volume.join_parallel_trees()
self.volume = volume
if dx is None:
dx = self.ds.index.get_smallest_dx()
self.dx = dx
if length is None:
length = np.max(self.ds.domain_right_edge -
self.ds.domain_left_edge)
self.length = length
self.steps = int(length / dx) + 1
# Fix up the dx.
self.dx = 1.0 * self.length / self.steps
self.streamlines = np.zeros((self.N, self.steps, 3), dtype='float64')
self.magnitudes = None
if self.get_magnitude:
self.magnitudes = np.zeros((self.N, self.steps), dtype='float64')
def __init__(self, halos_ds=None, data_ds=None,
data_source=None, finder_method=None,
finder_kwargs=None,
output_dir="halo_catalogs/catalog"):
ParallelAnalysisInterface.__init__(self)
self.halos_ds = halos_ds
self.data_ds = data_ds
self.output_dir = ensure_dir(output_dir)
if os.path.basename(self.output_dir) != ".":
self.output_prefix = os.path.basename(self.output_dir)
else:
self.output_prefix = "catalog"
if halos_ds is None:
if data_ds is None:
raise RuntimeError("Must specify a halos_ds, data_ds, or both.")
if finder_method is None:
raise RuntimeError("Must specify a halos_ds or a finder_method.")
if data_source is None:
if halos_ds is not None:
halos_ds.index
data_source = halos_ds.all_data()
else:
data_source = data_ds.all_data()
self.data_source = data_source
if finder_kwargs is None:
finder_kwargs = {}
if finder_method is not None:
finder_method = finding_method_registry.find(finder_method,
**finder_kwargs)
self.finder_method = finder_method
# all of the analysis actions to be performed: callbacks, filters, and quantities
self.actions = []
# fields to be written to the halo catalog
self.quantities = []
if not self.halos_ds is None:
self.add_default_quantities()
def __init__(self, ds, positions, xfield='velocity_x', yfield='velocity_x',
zfield='velocity_x', volume=None,
dx=None, length=None, direction=1,
get_magnitude=False):
ParallelAnalysisInterface.__init__(self)
self.ds = ds
self.start_positions = sanitize_length(positions, ds)
self.N = self.start_positions.shape[0]
# I need a data object to resolve the field names to field tuples
# via _determine_fields()
ad = self.ds.all_data()
self.xfield = ad._determine_fields(xfield)[0]
self.yfield = ad._determine_fields(yfield)[0]
self.zfield = ad._determine_fields(zfield)[0]
self.get_magnitude=get_magnitude
self.direction = np.sign(direction)
if volume is None:
volume = AMRKDTree(self.ds)
volume.set_fields([self.xfield,self.yfield,self.zfield],
[False,False,False],
False)
volume.join_parallel_trees()
self.volume = volume
if dx is None:
dx = self.ds.index.get_smallest_dx()
self.dx = sanitize_length(dx, ds)
if length is None:
length = np.max(self.ds.domain_right_edge-self.ds.domain_left_edge)
self.length = sanitize_length(length, ds)
self.steps = int(length/dx)+1
# Fix up the dx.
self.dx = 1.0*self.length/self.steps
self.streamlines = np.zeros((self.N,self.steps,3), dtype='float64')
self.magnitudes = None
if self.get_magnitude:
self.magnitudes = np.zeros((self.N,self.steps), dtype='float64')
def __init__(self,
ds,
fields,
left_edge=None,
right_edge=None,
total_values=1000000,
comm_size=10000,
length_type="lin",
length_number=10,
length_range=None,
vol_ratio=1,
salt=0,
theta=None,
phi=None):
ParallelAnalysisInterface.__init__(self)
try:
fKD
except NameError:
raise ImportError("You need to install the Forthon kD-Tree")
self._fsets = []
self.fields = fields
self.constant_theta = theta
self.constant_phi = phi
# MPI stuff.
self.size = self.comm.size
self.mine = self.comm.rank
self.vol_ratio = vol_ratio
if self.vol_ratio == -1:
self.vol_ratio = self.size
self.total_values = int(total_values / self.size)
# For communication.
self.recv_hooks = []
self.send_hooks = []
self.done_hooks = []
self.comm_size = min(int(comm_size), self.total_values)
self.ds = ds
self.nlevels = ds.index.max_level
self.period = self.ds.domain_right_edge - self.ds.domain_left_edge
self.min_edge = min(self.period)
self.index = ds.index
self.center = (ds.domain_right_edge + ds.domain_left_edge) / 2.0
# Figure out the range of ruler lengths.
if length_range == None:
length_range = [
math.sqrt(3) * self.ds.index.get_smallest_dx(),
self.min_edge / 2.
]
else:
if len(length_range) != 2:
raise ValueError("length_range must have two values.")
if length_range[1] <= length_range[0]:
raise ValueError(
"length_range[1] must be larger than length_range[0]")
if length_range[1] > self.min_edge / 2.:
length_range[1] = self.min_edge / 2.
mylog.info(
"Automatically adjusting length_range[1] to half the shortest box edge."
)
if length_range[0] == -1 or length_range[0] == -1.:
mylog.info("Automatically adjusting length_range[0] to %1.5e." % \
(math.sqrt(3) * self.ds.index.get_smallest_dx()))
length_range[0] = math.sqrt(3) * self.ds.index.get_smallest_dx()
# Make the list of ruler lengths.
if length_type == "lin":
self.lengths = np.linspace(length_range[0], length_range[1],
length_number)
elif length_type == "log":
self.lengths = np.logspace(math.log10(length_range[0]),
math.log10(length_range[1]),
length_number)
else:
# Something went wrong.
raise SyntaxError("length_type is either \"lin\" or \"log\".")
# Subdivide the volume.
if not left_edge or not right_edge:
self.left_edge = self.ds.domain_left_edge
self.right_edge = self.ds.domain_right_edge
# This ds business below has to do with changes made for halo
# finding on subvolumes and serves no purpose here except
# compatibility. This is not the best policy, if I'm honest.
ds = ds.region([0.] * 3, self.left_edge, self.right_edge)
padded, self.LE, self.RE, self.ds = \
self.partition_index_3d(ds = ds, padding=0.,
rank_ratio = self.vol_ratio)
else:
self.left_edge = left_edge
self.right_edge = right_edge
# We do this twice, first with no 'buffer' to get the unbuffered
# self.LE/RE, and then second to get a buffered self.ds.
padded, self.LE, self.RE, temp = \
self.partition_region_3d(left_edge, right_edge,
rank_ratio=self.vol_ratio)
padded, temp, temp, self.ds = \
self.partition_region_3d(left_edge - self.lengths[-1], \
right_edge + self.lengths[-1], rank_ratio=self.vol_ratio)
mylog.info("LE %s RE %s %s" %
(str(self.LE), str(self.RE), str(self.ds)))
self.width = self.ds.right_edge - self.ds.left_edge
self.mt = np.random.mtrand.RandomState(seed=1234 * self.mine + salt)
def __init__(self, ds, fields, left_edge=None, right_edge=None,
total_values=1000000, comm_size=10000, length_type="lin",
length_number=10, length_range=None, vol_ratio = 1,
salt=0, theta=None, phi=None):
ParallelAnalysisInterface.__init__(self)
try:
fKD
except NameError:
raise ImportError("You need to install the Forthon kD-Tree")
self._fsets = []
self.fields = fields
self.constant_theta = theta
self.constant_phi = phi
# MPI stuff.
self.size = self.comm.size
self.mine = self.comm.rank
self.vol_ratio = vol_ratio
if self.vol_ratio == -1:
self.vol_ratio = self.size
self.total_values = int(total_values / self.size)
# For communication.
self.recv_hooks = []
self.send_hooks = []
self.done_hooks = []
self.comm_size = min(int(comm_size), self.total_values)
self.ds = ds
self.nlevels = ds.index.max_level
self.period = self.ds.domain_right_edge - self.ds.domain_left_edge
self.min_edge = min(self.period)
self.index = ds.index
self.center = (ds.domain_right_edge + ds.domain_left_edge)/2.0
# Figure out the range of ruler lengths.
if length_range == None:
length_range = [math.sqrt(3) * self.ds.index.get_smallest_dx(),
self.min_edge/2.]
else:
if len(length_range) != 2:
raise ValueError("length_range must have two values.")
if length_range[1] <= length_range[0]:
raise ValueError("length_range[1] must be larger than length_range[0]")
if length_range[1] > self.min_edge/2.:
length_range[1] = self.min_edge/2.
mylog.info("Automatically adjusting length_range[1] to half the shortest box edge.")
if length_range[0] == -1 or length_range[0] == -1.:
mylog.info("Automatically adjusting length_range[0] to %1.5e." % \
(math.sqrt(3) * self.ds.index.get_smallest_dx()))
length_range[0] = math.sqrt(3) * self.ds.index.get_smallest_dx()
# Make the list of ruler lengths.
if length_type == "lin":
self.lengths = np.linspace(length_range[0], length_range[1],
length_number)
elif length_type == "log":
self.lengths = np.logspace(math.log10(length_range[0]),
math.log10(length_range[1]), length_number)
else:
# Something went wrong.
raise SyntaxError("length_type is either \"lin\" or \"log\".")
# Subdivide the volume.
if not left_edge or not right_edge:
self.left_edge = self.ds.domain_left_edge
self.right_edge = self.ds.domain_right_edge
# This ds business below has to do with changes made for halo
# finding on subvolumes and serves no purpose here except
# compatibility. This is not the best policy, if I'm honest.
ds = ds.region([0.]*3, self.left_edge, self.right_edge)
padded, self.LE, self.RE, self.ds = \
self.partition_index_3d(ds = ds, padding=0.,
rank_ratio = self.vol_ratio)
else:
self.left_edge = left_edge
self.right_edge = right_edge
# We do this twice, first with no 'buffer' to get the unbuffered
# self.LE/RE, and then second to get a buffered self.ds.
padded, self.LE, self.RE, temp = \
self.partition_region_3d(left_edge, right_edge,
rank_ratio=self.vol_ratio)
padded, temp, temp, self.ds = \
self.partition_region_3d(left_edge - self.lengths[-1], \
right_edge + self.lengths[-1], rank_ratio=self.vol_ratio)
mylog.info("LE %s RE %s %s" % (str(self.LE), str(self.RE), str(self.ds)))
self.width = self.ds.right_edge - self.ds.left_edge
self.mt = np.random.mtrand.RandomState(seed = 1234 * self.mine + salt)
