def main(config, config_prefix, population, gid, ref_axis, input_file, template_name, output_file, dry_run, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) h.load_file("nrngui.hoc") h.load_file("import3d.hoc") env = Env(config_file=config, config_prefix=config_prefix) swc_type_defs = env.SWC_Types if not os.path.isfile(output_file): io_utils.make_h5types(env, output_file) (forest_pop_ranges, _) = read_population_ranges(output_file) (forest_population_start, forest_population_count) = forest_pop_ranges[population] forest_population_end = forest_population_start + forest_population_count h.load_file(input_file) cell = getattr(h, template_name)(0, 0) if verbose: h.topology() tree_dict = export_swc_dict(cell, ref_axis=ref_axis) if (gid < forest_population_start) or (gid > forest_population_end): gid = forest_population_start trees_dict = { gid : tree_dict } logger.info(pprint.pformat(trees_dict)) if not dry_run: append_cell_trees(output_file, population, trees_dict)
def main(config, config_prefix, population, input_file, output_file, dry_run, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) h.load_file("nrngui.hoc") h.load_file("import3d.hoc") env = Env(config_file=config, config_prefix=config_prefix) swc_type_defs = env.SWC_Types if not os.path.isfile(output_file): io_utils.make_h5types(env, output_file) (forest_pop_ranges, _) = read_population_ranges(output_file) (forest_population_start, forest_population_count) = forest_pop_ranges[population] h.load_file(input_file) if verbose: h.topology() tree_dict = export_swc_dict() trees_dict = {0: tree_dict} logger.info(pprint.pformat(trees_dict)) if not dry_run: append_cell_trees(output_file, population, trees_dict)
def main(population, trees_path, index_path, chunk_size, value_chunk_size): comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: print '%i ranks have been allocated' % comm.size sys.stdout.flush() f = open(index_path) lines = f.readlines() index = {} while lines: for l in lines: a = filter(None, l.split(" ")) gid = int(a[0]) newgid = int(a[1]) index[gid] = newgid lines = f.readlines() f.close() selection_dict = {} for gid, morph_dict in NeuroH5TreeGen(comm, forest_path, population, io_size=io_size): if index.has_key(gid): newgid = index[gid] selection_dict[newgid] = morph_dict append_cell_trees (comm, output_path, population, selection_dict, io_size=io_size)
def main(config, config_prefix, max_section_length, population, forest_path, template_path, output_path, io_size, chunk_size, value_chunk_size, dry_run, verbose): """ :param population: str :param forest_path: str (path) :param output_path: str (path) :param io_size: int :param chunk_size: int :param value_chunk_size: int :param verbose: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) env = Env(comm=comm, config_file=config, config_prefix=config_prefix, template_paths=template_path) if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (forest_pop_ranges, _) = read_population_ranges(forest_path) (forest_population_start, forest_population_count) = forest_pop_ranges[population] (pop_ranges, _) = read_population_ranges(output_path) (population_start, population_count) = pop_ranges[population] new_trees_dict = {} for gid, tree_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=False): if gid is not None: logger.info("Rank %d received gid %d" % (rank, gid)) logger.info(pprint.pformat(tree_dict)) new_tree_dict = cells.resize_tree_sections(tree_dict, max_section_length) logger.info(pprint.pformat(new_tree_dict)) new_trees_dict[gid] = new_tree_dict if not dry_run: append_cell_trees(output_path, population, new_trees_dict, io_size=io_size, comm=comm) comm.barrier() if (not dry_run) and (rank == 0): logger.info('Appended resized trees to %s' % output_path)
def write_cell_selection(env, write_selection_file_path, populations=None, write_kwds={}): """ Writes out the data necessary to instantiate the selected cells. :param env: an instance of the `dentate.Env` class """ if 'comm' not in write_kwds: write_kwds['comm'] = env.comm if 'io_size' not in write_kwds: write_kwds['io_size'] = env.io_size rank = int(env.comm.Get_rank()) nhosts = int(env.comm.Get_size()) dataset_path = env.dataset_path data_file_path = env.data_file_path if populations is None: pop_names = sorted(env.cell_selection.keys()) else: pop_names = populations for pop_name in pop_names: gid_range = [ gid for gid in env.cell_selection[pop_name] if gid % nhosts == rank ] trees_output_dict = {} coords_output_dict = {} num_cells = 0 if (pop_name in env.cell_attribute_info) and ( 'Trees' in env.cell_attribute_info[pop_name]): if rank == 0: logger.info("*** Reading trees for population %s" % pop_name) cell_tree_iter, _ = read_tree_selection(data_file_path, pop_name, selection=gid_range, \ topology=False, comm=env.comm) if rank == 0: logger.info("*** Done reading trees for population %s" % pop_name) for i, (gid, tree) in enumerate(cell_tree_iter): trees_output_dict[gid] = tree num_cells += 1 assert (len(trees_output_dict) == len(gid_range)) elif (pop_name in env.cell_attribute_info) and ( 'Coordinates' in env.cell_attribute_info[pop_name]): if rank == 0: logger.info("*** Reading coordinates for population %s" % pop_name) cell_attributes_iter = read_cell_attribute_selection(data_file_path, pop_name, selection=gid_range, \ namespace='Coordinates', comm=env.comm) if rank == 0: logger.info("*** Done reading coordinates for population %s" % pop_name) for i, (gid, coords) in enumerate(cell_attributes_iter): coords_output_dict[gid] = coords num_cells += 1 if rank == 0: logger.info( "*** Writing cell selection for population %s to file %s" % (pop_name, write_selection_file_path)) append_cell_trees(write_selection_file_path, pop_name, trees_output_dict, create_index=True, **write_kwds) write_cell_attributes(write_selection_file_path, pop_name, coords_output_dict, namespace='Coordinates', **write_kwds)
def main(population, forest_path, output_path, index_path, types_path, index_namespace, coords_namespace, sample_count, io_size, chunk_size, value_chunk_size, verbose): """ :param population: str :param forest_path: str (path) :param output_path: str (path) :param index_path: str (path) :param io_size: int :param chunk_size: int :param value_chunk_size: int :param verbose: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) random.seed(13) if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(types_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (forest_pop_ranges, _) = read_population_ranges(forest_path) (forest_population_start, forest_population_count) = forest_pop_ranges[population] (pop_ranges, _) = read_population_ranges(output_path) (population_start, population_count) = pop_ranges[population] if rank == 0: logger.info('reading new cell index map...') reindex_map1 = {} reindex_map_gen = bcast_cell_attributes(index_path, population, namespace=index_namespace, root=0, comm=comm) for gid, attr_dict in reindex_map_gen: reindex_map1[gid] = attr_dict['New Cell Index'][0] if rank == 0: logger.info('reading cell coordinates...') old_coords_dict = {} coords_map_gen = bcast_cell_attributes(index_path, population, namespace=coords_namespace, root=0, comm=comm) for gid, attr_dict in coords_map_gen: old_coords_dict[gid] = attr_dict gc.collect() if rank == 0: logger.info('sampling cell population reindex...') N = len(reindex_map1) if sample_count is None: sample_count = min(population_count, N) else: sample_count = min(sample_count, N) reindex_map = None if rank == 0: reindex_map = {} reindex_map = dict( random_subset(utils.viewitems(reindex_map1), sample_count)) reindex_map = comm.bcast(reindex_map, root=0) if rank == 0: logger.info('computing new population index...') gid_map = { k: i + population_start for i, k in enumerate(sorted(reindex_map.keys())) } new_coords_dict = {} new_trees_dict = {} for gid, old_trees_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=False): if gid is not None and gid in reindex_map: reindex_gid = reindex_map[gid] new_gid = gid_map[gid] new_trees_dict[new_gid] = old_trees_dict new_coords_dict[new_gid] = old_coords_dict[gid] logger.info('Rank: %i mapping old gid: %i to new gid: %i' % (rank, gid, new_gid)) append_cell_trees(output_path, population, new_trees_dict, io_size=io_size, comm=comm) append_cell_attributes(output_path, population, new_coords_dict, \ namespace=coords_namespace, io_size=io_size, comm=comm) comm.barrier() if rank == 0: logger.info('Appended reindexed trees to %s' % output_path)