def main(config, config_prefix, population, gid, ref_axis, input_file, template_name, output_file, dry_run, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
h.load_file("nrngui.hoc")
h.load_file("import3d.hoc")
env = Env(config_file=config, config_prefix=config_prefix)
swc_type_defs = env.SWC_Types
if not os.path.isfile(output_file):
io_utils.make_h5types(env, output_file)
(forest_pop_ranges, _) = read_population_ranges(output_file)
(forest_population_start, forest_population_count) = forest_pop_ranges[population]
forest_population_end = forest_population_start + forest_population_count
h.load_file(input_file)
cell = getattr(h, template_name)(0, 0)
if verbose:
h.topology()
tree_dict = export_swc_dict(cell, ref_axis=ref_axis)
if (gid < forest_population_start) or (gid > forest_population_end):
gid = forest_population_start
trees_dict = { gid : tree_dict }
logger.info(pprint.pformat(trees_dict))
if not dry_run:
append_cell_trees(output_file, population, trees_dict)
def main(config, config_prefix, population, input_file, output_file, dry_run,
verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
h.load_file("nrngui.hoc")
h.load_file("import3d.hoc")
env = Env(config_file=config, config_prefix=config_prefix)
swc_type_defs = env.SWC_Types
if not os.path.isfile(output_file):
io_utils.make_h5types(env, output_file)
(forest_pop_ranges, _) = read_population_ranges(output_file)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
h.load_file(input_file)
if verbose:
h.topology()
tree_dict = export_swc_dict()
trees_dict = {0: tree_dict}
logger.info(pprint.pformat(trees_dict))
if not dry_run:
append_cell_trees(output_file, population, trees_dict)
def main(population, trees_path, index_path, chunk_size, value_chunk_size):
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
print '%i ranks have been allocated' % comm.size
sys.stdout.flush()
f = open(index_path)
lines = f.readlines()
index = {}
while lines:
for l in lines:
a = filter(None, l.split(" "))
gid = int(a[0])
newgid = int(a[1])
index[gid] = newgid
lines = f.readlines()
f.close()
selection_dict = {}
for gid, morph_dict in NeuroH5TreeGen(comm, forest_path, population, io_size=io_size):
if index.has_key(gid):
newgid = index[gid]
selection_dict[newgid] = morph_dict
append_cell_trees (comm, output_path, population, selection_dict, io_size=io_size)
def main(config, config_prefix, max_section_length, population, forest_path,
template_path, output_path, io_size, chunk_size, value_chunk_size,
dry_run, verbose):
"""
:param population: str
:param forest_path: str (path)
:param output_path: str (path)
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param verbose: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
env = Env(comm=comm,
config_file=config,
config_prefix=config_prefix,
template_paths=template_path)
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(forest_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(forest_pop_ranges, _) = read_population_ranges(forest_path)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
(pop_ranges, _) = read_population_ranges(output_path)
(population_start, population_count) = pop_ranges[population]
new_trees_dict = {}
for gid, tree_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=False):
if gid is not None:
logger.info("Rank %d received gid %d" % (rank, gid))
logger.info(pprint.pformat(tree_dict))
new_tree_dict = cells.resize_tree_sections(tree_dict,
max_section_length)
logger.info(pprint.pformat(new_tree_dict))
new_trees_dict[gid] = new_tree_dict
if not dry_run:
append_cell_trees(output_path,
population,
new_trees_dict,
io_size=io_size,
comm=comm)
comm.barrier()
if (not dry_run) and (rank == 0):
logger.info('Appended resized trees to %s' % output_path)
def write_cell_selection(env,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Writes out the data necessary to instantiate the selected cells.
:param env: an instance of the `dentate.Env` class
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
dataset_path = env.dataset_path
data_file_path = env.data_file_path
if populations is None:
pop_names = sorted(env.cell_selection.keys())
else:
pop_names = populations
for pop_name in pop_names:
gid_range = [
gid for gid in env.cell_selection[pop_name] if gid % nhosts == rank
]
trees_output_dict = {}
coords_output_dict = {}
num_cells = 0
if (pop_name in env.cell_attribute_info) and (
'Trees' in env.cell_attribute_info[pop_name]):
if rank == 0:
logger.info("*** Reading trees for population %s" % pop_name)
cell_tree_iter, _ = read_tree_selection(data_file_path, pop_name, selection=gid_range, \
topology=False, comm=env.comm)
if rank == 0:
logger.info("*** Done reading trees for population %s" %
pop_name)
for i, (gid, tree) in enumerate(cell_tree_iter):
trees_output_dict[gid] = tree
num_cells += 1
assert (len(trees_output_dict) == len(gid_range))
elif (pop_name in env.cell_attribute_info) and (
'Coordinates' in env.cell_attribute_info[pop_name]):
if rank == 0:
logger.info("*** Reading coordinates for population %s" %
pop_name)
cell_attributes_iter = read_cell_attribute_selection(data_file_path, pop_name, selection=gid_range, \
namespace='Coordinates', comm=env.comm)
if rank == 0:
logger.info("*** Done reading coordinates for population %s" %
pop_name)
for i, (gid, coords) in enumerate(cell_attributes_iter):
coords_output_dict[gid] = coords
num_cells += 1
if rank == 0:
logger.info(
"*** Writing cell selection for population %s to file %s" %
(pop_name, write_selection_file_path))
append_cell_trees(write_selection_file_path,
pop_name,
trees_output_dict,
create_index=True,
**write_kwds)
write_cell_attributes(write_selection_file_path,
pop_name,
coords_output_dict,
namespace='Coordinates',
**write_kwds)
def main(population, forest_path, output_path, index_path, types_path,
index_namespace, coords_namespace, sample_count, io_size, chunk_size,
value_chunk_size, verbose):
"""
:param population: str
:param forest_path: str (path)
:param output_path: str (path)
:param index_path: str (path)
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param verbose: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
random.seed(13)
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(types_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(forest_pop_ranges, _) = read_population_ranges(forest_path)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
(pop_ranges, _) = read_population_ranges(output_path)
(population_start, population_count) = pop_ranges[population]
if rank == 0:
logger.info('reading new cell index map...')
reindex_map1 = {}
reindex_map_gen = bcast_cell_attributes(index_path,
population,
namespace=index_namespace,
root=0,
comm=comm)
for gid, attr_dict in reindex_map_gen:
reindex_map1[gid] = attr_dict['New Cell Index'][0]
if rank == 0:
logger.info('reading cell coordinates...')
old_coords_dict = {}
coords_map_gen = bcast_cell_attributes(index_path,
population,
namespace=coords_namespace,
root=0,
comm=comm)
for gid, attr_dict in coords_map_gen:
old_coords_dict[gid] = attr_dict
gc.collect()
if rank == 0:
logger.info('sampling cell population reindex...')
N = len(reindex_map1)
if sample_count is None:
sample_count = min(population_count, N)
else:
sample_count = min(sample_count, N)
reindex_map = None
if rank == 0:
reindex_map = {}
reindex_map = dict(
random_subset(utils.viewitems(reindex_map1), sample_count))
reindex_map = comm.bcast(reindex_map, root=0)
if rank == 0:
logger.info('computing new population index...')
gid_map = {
k: i + population_start
for i, k in enumerate(sorted(reindex_map.keys()))
}
new_coords_dict = {}
new_trees_dict = {}
for gid, old_trees_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=False):
if gid is not None and gid in reindex_map:
reindex_gid = reindex_map[gid]
new_gid = gid_map[gid]
new_trees_dict[new_gid] = old_trees_dict
new_coords_dict[new_gid] = old_coords_dict[gid]
logger.info('Rank: %i mapping old gid: %i to new gid: %i' %
(rank, gid, new_gid))
append_cell_trees(output_path,
population,
new_trees_dict,
io_size=io_size,
comm=comm)
append_cell_attributes(output_path, population, new_coords_dict, \
namespace=coords_namespace, io_size=io_size, comm=comm)
comm.barrier()
if rank == 0:
logger.info('Appended reindexed trees to %s' % output_path)