def main(population, forest_path, forest_measurement_namespace, attr_name, selection_path): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size population_ranges = read_population_ranges(forest_path)[0] selection = [] f = open(selection_path, 'r') for line in f.readlines(): selection.append(int(line)) f.close() columns = [attr_name] df_dict = {} it = read_cell_attribute_selection(forest_path, population, namespace=forest_measurement_namespace, selection=selection) for cell_gid, meas_dict in it: cell_attr = meas_dict[attr_name] df_dict[cell_gid] = [np.sum(cell_attr)] df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns) df = df.reindex(selection) df.to_csv('tree.%s.%s.csv' % (attr_name, population))
def main(coords_path, coords_namespace): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print('Allocated %i ranks' % size) population_ranges = read_population_ranges(coords_path)[0] print(population_ranges) soma_coords = {} populations = ['GC'] for population in population_ranges.keys(): print('Population %s' % population) it = read_cell_attributes(coords_path, population, namespace=coords_namespace) print('it = %s' % str(it)) for cell_gid, coords_dict in it: print(hasattr(coords_dict, 'U Coordinate')) cell_u = getattr(coords_dict, 'U Coordinate') cell_u = getattr(coords_dict, 'U Coordinate') cell_v = getattr(coords_dict, 'V Coordinate') print('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v))
def main(coords_path, coords_namespace): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print ('Allocated %i ranks' % size) sys.stdout.flush() population_ranges = read_population_ranges(coords_path)[0] soma_coords = {} for population in ['GC']: attr_iter = bcast_cell_attributes(coords_path, population, namespace=coords_namespace, root=0, mask=set(['U Coordinate', 'V Coordinate', 'L Coordinate']), comm=comm) for cell_gid, coords_dict in attr_iter: cell_u = coords_dict['U Coordinate'] cell_v = coords_dict['V Coordinate'] print ('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v))
def main(config, config_prefix, population, gid, ref_axis, input_file, template_name, output_file, dry_run, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) h.load_file("nrngui.hoc") h.load_file("import3d.hoc") env = Env(config_file=config, config_prefix=config_prefix) swc_type_defs = env.SWC_Types if not os.path.isfile(output_file): io_utils.make_h5types(env, output_file) (forest_pop_ranges, _) = read_population_ranges(output_file) (forest_population_start, forest_population_count) = forest_pop_ranges[population] forest_population_end = forest_population_start + forest_population_count h.load_file(input_file) cell = getattr(h, template_name)(0, 0) if verbose: h.topology() tree_dict = export_swc_dict(cell, ref_axis=ref_axis) if (gid < forest_population_start) or (gid > forest_population_end): gid = forest_population_start trees_dict = { gid : tree_dict } logger.info(pprint.pformat(trees_dict)) if not dry_run: append_cell_trees(output_file, population, trees_dict)
def main(population, features_path, features_namespace, extra_columns): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size population_ranges = read_population_ranges(features_path)[0] soma_coords = {} extra_columns_list = extra_columns.split(",") columns = ['Field Width', 'X Offset', 'Y Offset'] + extra_columns_list df_dict = {} it = read_cell_attributes(features_path, population, namespace=features_namespace) for cell_gid, features_dict in it: cell_field_width = features_dict['Field Width'][0] cell_xoffset = features_dict['X Offset'][0] cell_yoffset = features_dict['Y Offset'][0] df_dict[cell_gid] = [cell_field_width, cell_xoffset, cell_yoffset] df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns) df = df.reindex(sorted(df_dict.keys())) df.to_csv('features.%s.csv' % population)
def main(population, features_path, features_namespace, selection_path): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size population_ranges = read_population_ranges(features_path)[0] soma_coords = {} selection = [] f = open(selection_path, 'r') for line in f.readlines(): selection.append(int(line)) f.close() columns = ['Field Width', 'X Offset', 'Y Offset'] df_dict = {} it = read_cell_attribute_selection(features_path, population, namespace=features_namespace, selection=selection) for cell_gid, features_dict in it: cell_field_width = features_dict['Field Width'][0] cell_xoffset = features_dict['X Offset'][0] cell_yoffset = features_dict['Y Offset'][0] df_dict[cell_gid] = [cell_field_width, cell_xoffset, cell_yoffset] df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns) df = df.reindex(selection) df.to_csv('features.%s.csv' % population)
def main(config, config_prefix, population, input_file, output_file, dry_run, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) h.load_file("nrngui.hoc") h.load_file("import3d.hoc") env = Env(config_file=config, config_prefix=config_prefix) swc_type_defs = env.SWC_Types if not os.path.isfile(output_file): io_utils.make_h5types(env, output_file) (forest_pop_ranges, _) = read_population_ranges(output_file) (forest_population_start, forest_population_count) = forest_pop_ranges[population] h.load_file(input_file) if verbose: h.topology() tree_dict = export_swc_dict() trees_dict = {0: tree_dict} logger.info(pprint.pformat(trees_dict)) if not dry_run: append_cell_trees(output_file, population, trees_dict)
def main(coords_path, coords_namespace): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print('Allocated %i ranks' % size) population_ranges = read_population_ranges(coords_path)[0] print(population_ranges) soma_coords = {} for population in sorted(population_ranges.keys()): print('Population %s' % population) it, tuple_info = read_cell_attributes(coords_path, population, namespace=coords_namespace, return_type='tuple') u_index = tuple_info['U Coordinate'] v_index = tuple_info['V Coordinate'] for cell_gid, coords_tuple in it: cell_u = coords_tuple[u_index] cell_v = coords_tuple[v_index] print('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v))
def load_celltypes(self): """ :return: """ rank = self.comm.Get_rank() size = self.comm.Get_size() celltypes = self.celltypes typenames = sorted(celltypes.keys()) if rank == 0: self.logger.info('env.data_file_path = %s' % str(self.data_file_path)) (population_ranges, _) = read_population_ranges(self.data_file_path, self.comm) if rank == 0: self.logger.info('population_ranges = %s' % str(population_ranges)) for k in typenames: population_range = population_ranges.get(k, None) if population_range is not None: celltypes[k]['start'] = population_ranges[k][0] celltypes[k]['num'] = population_ranges[k][1] if 'mechanism file' in celltypes[k]: celltypes[k]['mech_file_path'] = '%s/%s' % ( self.config_prefix, celltypes[k]['mechanism file']) mech_dict = read_from_yaml(celltypes[k]['mech_file_path']) celltypes[k]['mech_dict'] = mech_dict if 'synapses' in celltypes[k]: synapses_dict = celltypes[k]['synapses'] if 'weights' in synapses_dict: weights_config = synapses_dict['weights'] if isinstance(weights_config, list): weights_dicts = weights_config else: weights_dicts = [weights_config] for weights_dict in weights_dicts: if 'expr' in weights_dict: expr = weights_dict['expr'] parameter = weights_dict['parameter'] const = weights_dict.get('const', {}) clos = ExprClosure(parameter, expr, const) weights_dict['closure'] = clos synapses_dict['weights'] = weights_dicts population_names = read_population_names(self.data_file_path, self.comm) if rank == 0: self.logger.info('population_names = %s' % str(population_names)) self.cell_attribute_info = read_cell_attribute_info( self.data_file_path, population_names, comm=self.comm) if rank == 0: self.logger.info('attribute info: %s' % str(self.cell_attribute_info))
def make_node_rank_map(comm, filepath, iosize): size = comm.Get_size() (_, n_nodes) = read_population_ranges(filepath, comm=comm) node_rank_map = {} for i in range(0, n_nodes): node_rank_map[i] = i % size return (node_rank_map, n_nodes)
def main(config, template_path, prototype_gid, prototype_path, forest_path, population, io_size, verbose): """ :param config: :param template_path: :param prototype_gid: :param prototype_path: :param forest_path: :param population: :param io_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) configure_hoc_env(env) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) layers = env.layers layer_idx_dict = { layers[layer_name]: layer_name for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus'] } (tree_iter, _) = read_tree_selection(prototype_path, population, selection=[prototype_gid]) (_, prototype_morph_dict) = next(tree_iter) prototype_x = prototype_morph_dict['x'] prototype_y = prototype_morph_dict['y'] prototype_z = prototype_morph_dict['z'] prototype_xyz = (prototype_x, prototype_y, prototype_z) (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() (population_start, _) = pop_ranges[population] template_class = load_cell_template(env, population, bcast_template=True) for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, cache_size=1, comm=comm, topology=True): # trees, _ = scatter_read_trees(forest_path, population, io_size=io_size, comm=comm, topology=True) # for gid, morph_dict in trees: if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) secnodes_dict = morph_dict['section_topology']['nodes'] vx = morph_dict['x'] vy = morph_dict['y'] vz = morph_dict['z'] if compare_points((vx,vy,vz), prototype_xyz): logger.info('Possible match: gid %i' % gid) MPI.Finalize()
def load_graph_networkit(comm, input_file): (_, n_nodes) = read_population_ranges(input_file, comm=comm) nhg = read_graph(input_file, comm=comm) g = Graph(n_nodes, False, True) for (presyn, prjs) in list(nhg.items()): for (postsyn, edges) in list(prjs.items()): sources = edges[0] destinations = edges[1] for (src, dst) in zip(sources, destinations): g.addEdge(src, dst) return g
def main(config, stimulus_id, template_path, coords_path, output_path, distances_namespace, io_size, chunk_size, value_chunk_size, cache_size, write_size, verbose, dry_run): """ :param config: :param coords_path: :param distances_namespace: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: :param write_size: :param dry_run: """ comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config, template_paths=template_path) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if (not dry_run) and (rank == 0): if not os.path.isfile(output_path): input_file = h5py.File(coords_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() population_ranges = read_population_ranges(coords_path, comm)[0] context.update(locals()) gid_normed_distances = assign_cells_to_normalized_position( ) # Assign normalized u,v coordinates gid_module_assignments = assign_cells_to_module( gid_normed_distances, p_width=0.75, displace=0.0 ) # Determine which module a cell is in based on normalized u position total_num_fields, gid_attributes = determine_cell_participation( gid_module_assignments ) # Determine if a cell is 1) active and; 2) how many fields? cell_attributes = build_cell_attributes( gid_attributes, gid_normed_distances, total_num_fields ) # Determine additional cell properties (lambda, field_width, orientation, jitter, and rate map. This will also build the data structure ({<pop>: {<cell type>: <cells>}}) containing all cells. if not dry_run and rank == 0: save_to_h5(cell_attributes)
def main(config, coords_path, coords_namespace, resample, resolution, populations, projection_depth, io_size, chunk_size, value_chunk_size, cache_size, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config) soma_coords = {} if rank == 0: logger.info('Reading population coordinates...') rotate = env.geometry['Parametric Surface']['Rotation'] min_l = float('inf') max_l = 0.0 population_ranges = read_population_ranges(coords_path)[0] population_extents = {} for population in population_ranges: min_extent = env.geometry['Cell Layers']['Minimum Extent'][population] max_extent = env.geometry['Cell Layers']['Maximum Extent'][population] min_l = min(min_extent[2], min_l) max_l = max(max_extent[2], max_l) population_extents[population] = (min_extent, max_extent) for population in populations: coords = bcast_cell_attributes(coords_path, population, 0, \ namespace=coords_namespace) soma_coords[population] = { k: (v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0]) for (k,v) in coords } del coords gc.collect() output_path = coords_path soma_coords = icp_transform(comm, soma_coords, projection_depth, population_extents, \ populations=populations, rotate=rotate, verbose=verbose) for population in populations: if rank == 0: logger.info('Writing transformed coordinates for population %s...' % population) append_cell_attributes(output_path, population, soma_coords[population], namespace='Soma Projections', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size)
def main(population, coords_path, coords_namespace, distances_namespace, selection_path): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size population_ranges = read_population_ranges(coords_path)[0] soma_coords = {} selection = [] f = open(selection_path, 'r') for line in f.readlines(): selection.append(int(line)) f.close() columns = ['U', 'V', 'L'] df_dict = {} it = read_cell_attribute_selection(coords_path, population, namespace=coords_namespace, selection=selection) for cell_gid, coords_dict in it: cell_u = coords_dict['U Coordinate'][0] cell_v = coords_dict['V Coordinate'][0] cell_l = coords_dict['L Coordinate'][0] df_dict[cell_gid] = [cell_u, cell_v, cell_l] if distances_namespace is not None: columns.extend(['U Distance', 'V Distance']) it = read_cell_attribute_selection(coords_path, population, namespace=distances_namespace, selection=selection) for cell_gid, distances_dict in it: cell_ud = distances_dict['U Distance'][0] cell_vd = distances_dict['V Distance'][0] df_dict[cell_gid].extend([cell_ud, cell_vd]) df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns) df = df.reindex(selection) df.to_csv('coords.%s.csv' % population)
def main(coords_path, io_size, chunk_size, value_chunk_size): utils.config_logging(verbose) logger = utils.get_script_logger(__file__) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config) output_path = coords_path if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) source_population_ranges = read_population_ranges(coords_path) source_populations = list(source_population_ranges.keys()) for population in source_populations: if rank == 0: logger.info('population: ',population) soma_coords = bcast_cell_attributes(0, coords_path, population, namespace='Interpolated Coordinates', comm=comm) #print soma_coords.keys() u_coords = [] gids = [] for gid, attrs in viewitems(soma_coords): u_coords.append(attrs['U Coordinate']) gids.append(gid) u_coordv = np.asarray(u_coords, dtype=np.float32) gidv = np.asarray(gids, dtype=np.uint32) sort_idx = np.argsort(u_coordv, axis=0) offset = source_population_ranges[population][0] sorted_coords_dict = {} for i in range(0,sort_idx.size): sorted_coords_dict[offset+i] = soma_coords[gidv[sort_idx[i][0]]] append_cell_attributes(coords_path, population, sorted_coords_dict, namespace='Sorted Coordinates', io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, comm=comm)
def main(coords_path, coords_namespace, io_size, cache_size): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print('Allocated %i ranks' % size) population_ranges = read_population_ranges(coords_path)[0] print(population_ranges) soma_coords = {} for population in population_ranges.keys(): attr_iter = NeuroH5CellAttrGen(coords_path, population, namespace=coords_namespace, \ comm=comm, io_size=io_size, cache_size=cache_size) i = 0 for cell_gid, coords_dict in attr_iter: if cell_gid is not None: print('coords_dict: ', coords_dict) cell_u = coords_dict['U Coordinate'] cell_v = coords_dict['V Coordinate'] print('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v)) if i > 10: break i = i + 1 if rank == 0: import h5py count = 0 f = h5py.File(coords_path, 'r+') if 'test' in f: count = f['test'][()] del (f['test']) f['test'] = count + 1 comm.barrier()
def main(coords_path, coords_namespace, io_size): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print ('Allocated %i ranks' % size) population_ranges = read_population_ranges(coords_path)[0] soma_coords = {} for population in ['GC']: attr_dict = scatter_read_cell_attributes(coords_path, population, namespaces=[coords_namespace], io_size=io_size) attr_iter = attr_dict[coords_namespace] for cell_gid, coords_dict in attr_iter: cell_u = coords_dict['U Coordinate'] cell_v = coords_dict['V Coordinate'] print ('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v))
def main(syn_path, syn_namespace, io_size): comm = MPI.COMM_WORLD rank = comm.rank size = comm.size print('Allocated %i ranks' % size) population_ranges = read_population_ranges(syn_path)[0] print(population_ranges) for population in population_ranges.keys(): attr_dict = scatter_read_cell_attributes(syn_path, population, namespaces=[syn_namespace], io_size=io_size) attr_iter = attr_dict[syn_namespace] for cell_gid, attr_dict in attr_iter: print('Rank %i: gid = %i syn attrs:' % (rank, cell_gid)) pprint.pprint(attr_dict)
def main(config, template_path, output_path, forest_path, populations, io_size, chunk_size, value_chunk_size, cache_size, verbose): """ :param config: :param template_path: :param forest_path: :param populations: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) h('objref nil, pc, templatePaths') h.load_file("nrngui.hoc") h.load_file("./templates/Value.hoc") h.xopen("./lib.hoc") h.pc = h.ParallelContext() if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) h.templatePaths = h.List() for path in env.templatePaths: h.templatePaths.append(h.Value(1, path)) if output_path is None: output_path = forest_path if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() for population in populations: logger.info('Rank %i population: %s' % (rank, population)) count = 0 (population_start, _) = pop_ranges[population] template_name = env.celltypes[population]['template'] h.find_template(h.pc, h.templatePaths, template_name) template_class = eval('h.%s' % template_name) measures_dict = {} for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=True): if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) cell = cells.make_neurotree_cell(template_class, neurotree_dict=morph_dict, gid=gid) secnodes_dict = morph_dict['section_topology']['nodes'] apicalidx = set(cell.apicalidx) basalidx = set(cell.basalidx) dendrite_area_dict = {k + 1: 0.0 for k in range(0, 4)} dendrite_length_dict = {k + 1: 0.0 for k in range(0, 4)} for (i, sec) in enumerate(cell.sections): if (i in apicalidx) or (i in basalidx): secnodes = secnodes_dict[i] prev_layer = None for seg in sec.allseg(): L = seg.sec.L nseg = seg.sec.nseg seg_l = old_div(L, nseg) seg_area = h.area(seg.x) layer = cells.get_node_attribute( 'layer', morph_dict, seg.sec, secnodes, seg.x) layer = layer if layer > 0 else ( prev_layer if prev_layer is not None else 1) prev_layer = layer dendrite_length_dict[layer] += seg_l dendrite_area_dict[layer] += seg_area measures_dict[gid] = { 'dendrite_area': np.asarray([ dendrite_area_dict[k] for k in sorted(dendrite_area_dict.keys()) ], dtype=np.float32), \ 'dendrite_length': np.asarray([ dendrite_length_dict[k] for k in sorted(dendrite_length_dict.keys()) ], dtype=np.float32) } del cell count += 1 else: logger.info('Rank %i gid is None' % rank) append_cell_attributes(output_path, population, measures_dict, namespace='Tree Measurements', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size) MPI.Finalize()
def main(config, config_prefix, selectivity_path, selectivity_namespace, arena_id, populations, n_trials, io_size, chunk_size, value_chunk_size, cache_size, write_size, output_path, spikes_namespace, spike_train_attr_name, gather, debug, plot, show_fig, save_fig, save_fig_dir, font_size, fig_format, verbose, dry_run): """ :param config: str (.yaml file name) :param config_prefix: str (path to dir) :param selectivity_path: str (path to file) :param selectivity_namespace: str :param arena_id: str :param populations: str :param n_trials: int :param io_size: int :param chunk_size: int :param value_chunk_size: int :param cache_size: int :param write_size: int :param output_path: str (path to file) :param spikes_namespace: str :param spike_train_attr_name: str :param gather: bool :param debug: bool :param plot: bool :param show_fig: bool :param save_fig: str (base file name) :param save_fig_dir: str (path to dir) :param font_size: float :param fig_format: str :param verbose: bool :param dry_run: bool """ comm = MPI.COMM_WORLD rank = comm.rank config_logging(verbose) env = Env(comm=comm, config_file=config, config_prefix=config_prefix, template_paths=None) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if save_fig is not None: plot = True if plot: from dentate.plot import default_fig_options fig_options = copy.copy(default_fig_options) fig_options.saveFigDir = save_fig_dir fig_options.fontSize = font_size fig_options.figFormat = fig_format fig_options.showFig = show_fig population_ranges = read_population_ranges(selectivity_path, comm)[0] if len(populations) == 0: populations = sorted(population_ranges.keys()) if arena_id not in env.stimulus_config['Arena']: raise RuntimeError( 'Arena with ID: %s not specified by configuration at file path: %s' % (arena_id, config_prefix + '/' + config)) arena = env.stimulus_config['Arena'][arena_id] valid_selectivity_namespaces = dict() if rank == 0: for population in populations: if population not in population_ranges: raise RuntimeError( 'generate_input_spike_trains: specified population: %s not found in ' 'provided selectivity_path: %s' % (population, selectivity_path)) if population not in env.stimulus_config[ 'Selectivity Type Probabilities']: raise RuntimeError( 'generate_input_spike_trains: selectivity type not specified for ' 'population: %s' % population) valid_selectivity_namespaces[population] = [] with h5py.File(selectivity_path, 'r') as selectivity_f: for this_namespace in selectivity_f['Populations'][population]: if 'Selectivity %s' % arena_id in this_namespace: valid_selectivity_namespaces[population].append( this_namespace) if len(valid_selectivity_namespaces[population]) == 0: raise RuntimeError( 'generate_input_spike_trains: no selectivity data in arena: %s found ' 'for specified population: %s in provided selectivity_path: %s' % (arena_id, population, selectivity_path)) comm.barrier() valid_selectivity_namespaces = comm.bcast(valid_selectivity_namespaces, root=0) selectivity_type_names = dict( (val, key) for (key, val) in viewitems(env.selectivity_types)) equilibrate = get_equilibration(env) for trajectory_id in sorted(arena.trajectories.keys()): trajectory = arena.trajectories[trajectory_id] t, x, y, d = None, None, None, None if rank == 0: t, x, y, d = generate_linear_trajectory( trajectory, temporal_resolution=env.stimulus_config['Temporal Resolution'], equilibration_duration=env. stimulus_config['Equilibration Duration']) t = comm.bcast(t, root=0) x = comm.bcast(x, root=0) y = comm.bcast(y, root=0) d = comm.bcast(d, root=0) trajectory = t, x, y, d trajectory_namespace = 'Trajectory %s %s' % (arena_id, trajectory_id) output_namespace = '%s %s %s' % (spikes_namespace, arena_id, trajectory_id) if not dry_run and rank == 0: if output_path is None: raise RuntimeError( 'generate_input_spike_trains: missing output_path') if not os.path.isfile(output_path): with h5py.File(output_path, 'w') as output_file: input_file = h5py.File(selectivity_path, 'r') input_file.copy('/H5Types', output_file) input_file.close() with h5py.File(output_path, 'a') as f: if trajectory_namespace not in f: logger.info('Appending %s datasets to file at path: %s' % (trajectory_namespace, output_path)) group = f.create_group(trajectory_namespace) for key, value in zip(['t', 'x', 'y', 'd'], [t, x, y, d]): dataset = group.create_dataset(key, data=value, dtype='float32') else: loaded_t = f[trajectory_namespace]['t'][:] if len(t) != len(loaded_t): raise RuntimeError( 'generate_input_spike_trains: file at path: %s already contains the ' 'namespace: %s, but the dataset sizes are inconsistent with the provided input' 'configuration' % (output_path, trajectory_namespace)) comm.barrier() if rank == 0: context.update(locals()) spike_hist_sum_dict = {} spike_hist_resolution = 1000 write_every = max(1, int(math.floor(write_size / comm.size))) for population in populations: this_spike_hist_sum = defaultdict( lambda: np.zeros(spike_hist_resolution)) process_time = dict() for this_selectivity_namespace in sorted( valid_selectivity_namespaces[population]): if rank == 0: logger.info( 'Generating input source spike trains for population %s [%s]...' % (population, this_selectivity_namespace)) start_time = time.time() selectivity_attr_gen = NeuroH5CellAttrGen( selectivity_path, population, namespace=this_selectivity_namespace, comm=comm, io_size=io_size, cache_size=cache_size) spikes_attr_dict = dict() gid_count = 0 for iter_count, (gid, selectivity_attr_dict ) in enumerate(selectivity_attr_gen): if gid is not None: context.update(locals()) spikes_attr_dict[gid] = \ generate_input_spike_trains(env, selectivity_type_names, trajectory, gid, selectivity_attr_dict, n_trials=n_trials, spike_train_attr_name=spike_train_attr_name, spike_hist_resolution=spike_hist_resolution, equilibrate=equilibrate, spike_hist_sum=this_spike_hist_sum, debug= (debug_callback, context) if debug else False) gid_count += 1 if (iter_count > 0 and iter_count % write_every == 0) or (debug and iter_count == 10): total_gid_count = comm.reduce(gid_count, root=0, op=MPI.SUM) if rank == 0: logger.info( 'generated spike trains for %i %s cells' % (total_gid_count, population)) if not dry_run: append_cell_attributes( output_path, population, spikes_attr_dict, namespace=output_namespace, comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) del spikes_attr_dict spikes_attr_dict = dict() if debug and iter_count == 10: break if not dry_run: append_cell_attributes(output_path, population, spikes_attr_dict, namespace=output_namespace, comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) del spikes_attr_dict spikes_attr_dict = dict() process_time = time.time() - start_time total_gid_count = comm.reduce(gid_count, root=0, op=MPI.SUM) if rank == 0: logger.info( 'generated spike trains for %i %s cells in %.2f s' % (total_gid_count, population, process_time)) if gather: spike_hist_sum_dict[population] = this_spike_hist_sum if gather: this_spike_hist_sum = dict([ (key, dict(val.items())) for key, val in viewitems(spike_hist_sum_dict) ]) spike_hist_sum = comm.gather(this_spike_hist_sum, root=0) if rank == 0: merged_spike_hist_sum = defaultdict(lambda: defaultdict( lambda: np.zeros(spike_hist_resolution))) for each_spike_hist_sum in spike_hist_sum: for population in each_spike_hist_sum: for selectivity_type_name in each_spike_hist_sum[ population]: merged_spike_hist_sum[population][selectivity_type_name] = \ np.add(merged_spike_hist_sum[population][selectivity_type_name], each_spike_hist_sum[population][selectivity_type_name]) if plot: if save_fig is not None: fig_options.saveFig = save_fig plot_summed_spike_psth(t, trajectory_id, selectivity_type_name, merged_spike_hist_sum, spike_hist_resolution, fig_options) comm.barrier() if is_interactive and rank == 0: context.update(locals())
def main(stimulus_path, input_stimulus_namespace, output_stimulus_namespace, io_size, chunk_size, value_chunk_size, cache_size, seed_offset, trajectory_id, debug): """ :param stimulus_path: str :param input_stimulus_namespace: str :param output_stimulus_namespace: str :param io_size: int :param chunk_size: int :param value_chunk_size: int :param cache_size: int :param seed_offset: int :param trajectory_id: int :param debug: bool """ comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: print('%i ranks have been allocated' % comm.size) sys.stdout.flush() seed_offset *= 2e6 np.random.seed(int(seed_offset)) population_ranges = read_population_ranges(comm, stimulus_path)[0] input_stimulus_namespace += ' ' + str(trajectory_id) output_stimulus_namespace += ' ' + str(trajectory_id) for population in ['LPP']: population_start = population_ranges[population][0] population_count = population_ranges[population][1] if rank == 0: random_gids = np.arange(0, population_count) np.random.shuffle(random_gids) else: random_gids = None random_gids = comm.bcast(random_gids, root=0) count = 0 start_time = time.time() attr_gen = NeuroH5CellAttrGen(comm, stimulus_path, population, io_size=io_size, cache_size=cache_size, namespace=input_stimulus_namespace) if debug: attr_gen_wrapper = (next(attr_gen) for i in range(2)) else: attr_gen_wrapper = attr_gen for gid, stimulus_dict in attr_gen_wrapper: local_time = time.time() new_response_dict = {} if gid is not None: random_gid = random_gids[gid-population_start] new_response_dict[random_gid] = {'rate': stimulus_dict['rate'], 'spiketrain': np.asarray(stimulus_dict['spiketrain'], dtype=np.float32), 'modulation': stimulus_dict['modulation'], 'peak_index': stimulus_dict['peak_index'] } print('Rank %i; source: %s; assigned spike trains for gid %i to gid %i in %.2f s' % \ (rank, population, gid, random_gid+population_start, time.time() - local_time)) count += 1 if not debug: append_cell_attributes(comm, stimulus_path, population, new_response_dict, namespace=output_stimulus_namespace, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) sys.stdout.flush() del new_response_dict gc.collect() global_count = comm.gather(count, root=0) if rank == 0: print('%i ranks randomized spike trains for %i cells in %.2f s' % (comm.size, np.sum(global_count), time.time() - start_time))
def main(config, forest_path, connectivity_namespace, coords_path, coords_namespace, io_size, chunk_size, value_chunk_size, cache_size, debug): """ :param forest_path: :param connectivity_namespace: :param coords_path: :param coords_namespace: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ # troubleshooting if False: forest_path = '../morphologies/DGC_forest_connectivity_20170508.h5' coords_path = '../morphologies/dentate_Full_Scale_Control_coords_selectivity_20170615.h5' coords_namespace = 'Coordinates' io_size = -1 chunk_size = 1000 value_chunk_size = 1000 cache_size = 50 comm = MPI.COMM_WORLD rank = comm.rank # The process ID (integer 0-3 for 4-process run) env = Env(comm=comm, config_file=config) connection_config = env.connection_config proportions = connection_config.synapse_proportions layers = connection_config.synapse_layers syn_types = connection_config.synapse_types swc_types = connection_config.synapse_locations if io_size == -1: io_size = comm.size if rank == 0: print('%i ranks have been allocated' % comm.size) sys.stdout.flush() start_time = time.time() soma_coords = {} source_populations = list(read_population_ranges(comm, coords_path).keys()) for population in source_populations: soma_coords[population] = bcast_cell_attributes(comm, 0, coords_path, population, namespace=coords_namespace) target = 'GC' layer_set, swc_type_set, syn_type_set = set(), set(), set() for source in layers[target]: layer_set.update(layers[target][source]) swc_type_set.update(swc_types[target][source]) syn_type_set.update(syn_types[target][source]) count = 0 attr_gen = NeuroH5CellAttrGen(comm, forest_path, target, io_size=io_size, cache_size=cache_size, namespace='Synapse Attributes') if debug: attr_gen_wrapper = (next(attr_gen) for i in range(2)) else: attr_gen_wrapper = attr_gen for target_gid, attributes_dict in attr_gen_wrapper: last_time = time.time() connection_dict = {} p_dict = {} source_gid_dict = {} if target_gid is None: print('Rank %i target gid is None' % rank) else: print('Rank %i received attributes for target: %s, gid: %i' % (rank, target, target_gid)) synapse_dict = attributes_dict['Synapse_Attributes'] connection_dict[target_gid] = {} local_np_random.seed(target_gid + connectivity_seed_offset) connection_dict[target_gid]['source_gid'] = np.array([], dtype='uint32') connection_dict[target_gid]['syn_id'] = np.array([], dtype='uint32') for layer in layer_set: for swc_type in swc_type_set: for syn_type in syn_type_set: sources, this_proportions = filter_sources(target, layer, swc_type, syn_type, connection_config) if sources: if rank == 0 and count == 0: source_list_str = '[' + ', '.join(['%s' % xi for xi in sources]) + ']' print('Connections to target: %s in layer: %i ' \ '(swc_type: %i, syn_type: %i): %s' % \ (target, layer, swc_type, syn_type, source_list_str)) p, source_gid = np.array([]), np.array([]) for source, this_proportion in zip(sources, this_proportions): if source not in source_gid_dict: this_source_gid = list(soma_coords[source].keys()) this_p = np.ones(len(this_source_gid)) / float(len(this_source_gid)) source_gid_dict[source] = this_source_gid p_dict[source] = this_p else: this_source_gid = source_gid_dict[source] this_p = p_dict[source] p = np.append(p, this_p * this_proportion) source_gid = np.append(source_gid, this_source_gid) syn_indexes = filter_synapses(synapse_dict, layer, swc_type, syn_type) connection_dict[target_gid]['syn_id'] = \ np.append(connection_dict[target_gid]['syn_id'], synapse_dict['syn_id'][syn_indexes]).astype('uint32', copy=False) this_source_gid = local_np_random.choice(source_gid, len(syn_indexes), p=p) connection_dict[target_gid]['source_gid'] = \ np.append(connection_dict[target_gid]['source_gid'], this_source_gid).astype('uint32', copy=False) count += 1 print('Rank %i took %.2f s to compute connectivity for target: %s, gid: %i' % (rank, time.time() - last_time, target, target_gid)) sys.stdout.flush() if not debug: append_cell_attributes(comm, forest_path, target, connection_dict, namespace=connectivity_namespace, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) sys.stdout.flush() del connection_dict del p_dict del source_gid_dict gc.collect() global_count = comm.gather(count, root=0) if rank == 0: print('%i ranks took took %.2f s to compute connectivity for %i cells' % (comm.size, time.time() - start_time, np.sum(global_count)))
def main(arena_id, bin_sample_count, config, config_prefix, dataset_prefix, distances_namespace, distance_bin_extent, input_features_path, input_features_namespaces, populations, spike_input_path, spike_input_namespace, spike_input_attr, output_path, io_size, trajectory_id, write_selection, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config, config_prefix=config_prefix, dataset_prefix=dataset_prefix, results_path=output_path, spike_input_path=spike_input_path, spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr, arena_id=arena_id, trajectory_id=trajectory_id) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm) distance_U_dict = {} distance_V_dict = {} range_U_dict = {} range_V_dict = {} selection_dict = defaultdict(set) comm0 = env.comm.Split(2 if rank == 0 else 0, 0) local_random = np.random.RandomState() local_random.seed(1000) if len(populations) == 0: populations = sorted(pop_ranges.keys()) if rank == 0: for population in populations: distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0) soma_distances = {} if input_features_path is not None: num_fields_dict = {} for input_features_namespace in input_features_namespaces: if arena_id is not None: this_features_namespace = '%s %s' % ( input_features_namespace, arena_id) else: this_features_namespace = input_features_namespace input_features_iter = read_cell_attributes( input_features_path, population, namespace=this_features_namespace, mask=set(['Num Fields']), comm=comm0) count = 0 for gid, attr_dict in input_features_iter: num_fields_dict[gid] = attr_dict['Num Fields'] count += 1 logger.info( 'Read feature data from namespace %s for %i cells in population %s' % (this_features_namespace, count, population)) for (gid, v) in distances: num_fields = num_fields_dict.get(gid, 0) if num_fields > 0: soma_distances[gid] = (v['U Distance'][0], v['V Distance'][0]) else: for (gid, v) in distances: soma_distances[gid] = (v['U Distance'][0], v['V Distance'][0]) numitems = len(list(soma_distances.keys())) logger.info('read %s distances (%i elements)' % (population, numitems)) if numitems == 0: continue gid_array = np.asarray([gid for gid in soma_distances]) distance_U_array = np.asarray( [soma_distances[gid][0] for gid in gid_array]) distance_V_array = np.asarray( [soma_distances[gid][1] for gid in gid_array]) U_min = np.min(distance_U_array) U_max = np.max(distance_U_array) V_min = np.min(distance_V_array) V_max = np.max(distance_V_array) range_U_dict[population] = (U_min, U_max) range_V_dict[population] = (V_min, V_max) distance_U = { gid: soma_distances[gid][0] for gid in soma_distances } distance_V = { gid: soma_distances[gid][1] for gid in soma_distances } distance_U_dict[population] = distance_U distance_V_dict[population] = distance_V min_dist = U_min max_dist = U_max distance_bins = np.arange(U_min, U_max, distance_bin_extent) distance_bin_array = np.digitize(distance_U_array, distance_bins) selection_set = set([]) for bin_index in range(len(distance_bins) + 1): bin_gids = gid_array[np.where( distance_bin_array == bin_index)[0]] if len(bin_gids) > 0: selected_bin_gids = local_random.choice( bin_gids, replace=False, size=bin_sample_count) for gid in selected_bin_gids: selection_set.add(int(gid)) selection_dict[population] = selection_set yaml_output_dict = {} for k, v in utils.viewitems(selection_dict): yaml_output_dict[k] = list(sorted(v)) yaml_output_path = '%s/DG_slice.yaml' % output_path with open(yaml_output_path, 'w') as outfile: yaml.dump(yaml_output_dict, outfile) del (yaml_output_dict) env.comm.barrier() write_selection_file_path = None if write_selection: write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName) if write_selection_file_path is not None: if rank == 0: io_utils.mkout(env, write_selection_file_path) env.comm.barrier() selection_dict = env.comm.bcast(dict(selection_dict), root=0) env.cell_selection = selection_dict io_utils.write_cell_selection(env, write_selection_file_path, populations=populations) input_selection = io_utils.write_connection_selection( env, write_selection_file_path, populations=populations) if env.spike_input_ns is not None: io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path, populations=populations) env.comm.barrier() MPI.Finalize()
def main(config, template_path, output_path, forest_path, populations, distance_bin_size, io_size, chunk_size, value_chunk_size, cache_size, verbose): """ :param config: :param template_path: :param forest_path: :param populations: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) configure_hoc_env(env) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if output_path is None: output_path = forest_path if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() layers = env.layers layer_idx_dict = { layers[layer_name]: layer_name for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus'] } (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() for population in populations: logger.info('Rank %i population: %s' % (rank, population)) count = 0 (population_start, _) = pop_ranges[population] template_class = load_cell_template(env, population, bcast_template=True) measures_dict = {} for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=True): if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) cell = cells.make_neurotree_cell(template_class, neurotree_dict=morph_dict, gid=gid) secnodes_dict = morph_dict['section_topology']['nodes'] apicalidx = set(cell.apicalidx) basalidx = set(cell.basalidx) dendrite_area_dict = {k: 0.0 for k in layer_idx_dict} dendrite_length_dict = {k: 0.0 for k in layer_idx_dict} dendrite_distances = [] dendrite_diams = [] for (i, sec) in enumerate(cell.sections): if (i in apicalidx) or (i in basalidx): secnodes = secnodes_dict[i] for seg in sec.allseg(): L = seg.sec.L nseg = seg.sec.nseg seg_l = L / nseg seg_area = h.area(seg.x) seg_diam = seg.diam seg_distance = get_distance_to_node( cell, list(cell.soma)[0], seg.sec, seg.x) dendrite_diams.append(seg_diam) dendrite_distances.append(seg_distance) layer = synapses.get_node_attribute( 'layer', morph_dict, seg.sec, secnodes, seg.x) dendrite_length_dict[layer] += seg_l dendrite_area_dict[layer] += seg_area dendrite_distance_array = np.asarray(dendrite_distances) dendrite_diam_array = np.asarray(dendrite_diams) dendrite_distance_bin_range = int( ((np.max(dendrite_distance_array)) - np.min(dendrite_distance_array)) / distance_bin_size) + 1 dendrite_distance_counts, dendrite_distance_edges = np.histogram( dendrite_distance_array, bins=dendrite_distance_bin_range, density=False) dendrite_diam_sums, _ = np.histogram( dendrite_distance_array, weights=dendrite_diam_array, bins=dendrite_distance_bin_range, density=False) dendrite_mean_diam_hist = np.zeros_like(dendrite_diam_sums) np.divide(dendrite_diam_sums, dendrite_distance_counts, where=dendrite_distance_counts > 0, out=dendrite_mean_diam_hist) dendrite_area_per_layer = np.asarray([ dendrite_area_dict[k] for k in sorted(dendrite_area_dict.keys()) ], dtype=np.float32) dendrite_length_per_layer = np.asarray([ dendrite_length_dict[k] for k in sorted(dendrite_length_dict.keys()) ], dtype=np.float32) measures_dict[gid] = { 'dendrite_distance_hist_edges': np.asarray(dendrite_distance_edges, dtype=np.float32), 'dendrite_distance_counts': np.asarray(dendrite_distance_counts, dtype=np.int32), 'dendrite_mean_diam_hist': np.asarray(dendrite_mean_diam_hist, dtype=np.float32), 'dendrite_area_per_layer': dendrite_area_per_layer, 'dendrite_length_per_layer': dendrite_length_per_layer } del cell count += 1 else: logger.info('Rank %i gid is None' % rank) append_cell_attributes(output_path, population, measures_dict, namespace='Tree Measurements', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size) MPI.Finalize()
def main(config, config_prefix, max_section_length, population, forest_path, template_path, output_path, io_size, chunk_size, value_chunk_size, dry_run, verbose): """ :param population: str :param forest_path: str (path) :param output_path: str (path) :param io_size: int :param chunk_size: int :param value_chunk_size: int :param verbose: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) env = Env(comm=comm, config_file=config, config_prefix=config_prefix, template_paths=template_path) if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (forest_pop_ranges, _) = read_population_ranges(forest_path) (forest_population_start, forest_population_count) = forest_pop_ranges[population] (pop_ranges, _) = read_population_ranges(output_path) (population_start, population_count) = pop_ranges[population] new_trees_dict = {} for gid, tree_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=False): if gid is not None: logger.info("Rank %d received gid %d" % (rank, gid)) logger.info(pprint.pformat(tree_dict)) new_tree_dict = cells.resize_tree_sections(tree_dict, max_section_length) logger.info(pprint.pformat(new_tree_dict)) new_trees_dict[gid] = new_tree_dict if not dry_run: append_cell_trees(output_path, population, new_trees_dict, io_size=io_size, comm=comm) comm.barrier() if (not dry_run) and (rank == 0): logger.info('Appended resized trees to %s' % output_path)
def main(arena_id, config, config_prefix, dataset_prefix, distances_namespace, spike_input_path, spike_input_namespace, spike_input_attr, input_features_namespaces, input_features_path, selection_path, output_path, io_size, trajectory_id, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size env = Env(comm=comm, config_file=config, config_prefix=config_prefix, dataset_prefix=dataset_prefix, results_path=output_path, spike_input_path=spike_input_path, spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr, arena_id=arena_id, trajectory_id=trajectory_id, io_size=io_size) selection = [] f = open(selection_path, 'r') for line in f.readlines(): selection.append(int(line)) f.close() selection = set(selection) pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm) distance_U_dict = {} distance_V_dict = {} range_U_dict = {} range_V_dict = {} selection_dict = defaultdict(set) comm0 = env.comm.Split(2 if rank == 0 else 0, 0) if rank == 0: for population in pop_ranges: distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0) soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k,v) in distances } del distances numitems = len(list(soma_distances.keys())) if numitems == 0: continue distance_U_array = np.asarray([soma_distances[gid][0] for gid in soma_distances]) distance_V_array = np.asarray([soma_distances[gid][1] for gid in soma_distances]) U_min = np.min(distance_U_array) U_max = np.max(distance_U_array) V_min = np.min(distance_V_array) V_max = np.max(distance_V_array) range_U_dict[population] = (U_min, U_max) range_V_dict[population] = (V_min, V_max) distance_U = { gid: soma_distances[gid][0] for gid in soma_distances } distance_V = { gid: soma_distances[gid][1] for gid in soma_distances } distance_U_dict[population] = distance_U distance_V_dict[population] = distance_V min_dist = U_min max_dist = U_max selection_dict[population] = set([ k for k in distance_U if k in selection ]) env.comm.barrier() write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName) if rank == 0: io_utils.mkout(env, write_selection_file_path) env.comm.barrier() selection_dict = env.comm.bcast(dict(selection_dict), root=0) env.cell_selection = selection_dict io_utils.write_cell_selection(env, write_selection_file_path) input_selection = io_utils.write_connection_selection(env, write_selection_file_path) if spike_input_path: io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path) if input_features_path: for this_input_features_namespace in sorted(input_features_namespaces): for population in sorted(input_selection): logger.info(f"Extracting input features {this_input_features_namespace} for population {population}...") it = read_cell_attribute_selection(input_features_path, population, namespace=f"{this_input_features_namespace} {arena_id}", selection=input_selection[population], comm=env.comm) output_features_dict = { cell_gid : cell_features_dict for cell_gid, cell_features_dict in it } append_cell_attributes(write_selection_file_path, population, output_features_dict, namespace=f"{this_input_features_namespace} {arena_id}", io_size=io_size, comm=env.comm) env.comm.barrier()
def main(config, config_prefix, types_path, geometry_path, output_path, output_namespace, populations, resolution, alpha_radius, nodeiter, dispersion_delta, snap_delta, io_size, chunk_size, value_chunk_size, verbose): config_logging(verbose) logger = get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank size = comm.size np.seterr(all='raise') if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(types_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() env = Env(comm=comm, config_file=config, config_prefix=config_prefix) random_seed = int(env.model_config['Random Seeds']['Soma Locations']) random.seed(random_seed) layer_extents = env.geometry['Parametric Surface']['Layer Extents'] rotate = env.geometry['Parametric Surface']['Rotation'] (extent_u, extent_v, extent_l) = get_total_extents(layer_extents) vol = make_CA1_volume(extent_u, extent_v, extent_l, rotate=rotate, resolution=resolution) layer_alpha_shape_path = 'Layer Alpha Shape/%d/%d/%d' % resolution if rank == 0: logger.info("Constructing alpha shape for volume: extents: %s..." % str((extent_u, extent_v, extent_l))) vol_alpha_shape_path = '%s/all' % (layer_alpha_shape_path) if geometry_path: vol_alpha_shape = load_alpha_shape(geometry_path, vol_alpha_shape_path) else: vol_alpha_shape = make_alpha_shape(vol, alpha_radius=alpha_radius) if geometry_path: save_alpha_shape(geometry_path, vol_alpha_shape_path, vol_alpha_shape) vert = vol_alpha_shape.points smp = np.asarray(vol_alpha_shape.bounds, dtype=np.int64) vol_domain = (vert, smp) layer_alpha_shapes = {} layer_extent_vals = {} layer_extent_transformed_vals = {} if rank == 0: for layer, extents in viewitems(layer_extents): (extent_u, extent_v, extent_l) = get_layer_extents(layer_extents, layer) layer_extent_vals[layer] = (extent_u, extent_v, extent_l) layer_extent_transformed_vals[layer] = CA1_volume_transform( extent_u, extent_v, extent_l) has_layer_alpha_shape = False if geometry_path: this_layer_alpha_shape_path = '%s/%s' % ( layer_alpha_shape_path, layer) this_layer_alpha_shape = load_alpha_shape( geometry_path, this_layer_alpha_shape_path) layer_alpha_shapes[layer] = this_layer_alpha_shape if this_layer_alpha_shape is not None: has_layer_alpha_shape = True if not has_layer_alpha_shape: logger.info( "Constructing alpha shape for layers %s: extents: %s..." % (layer, str(extents))) layer_vol = make_CA1_volume(extent_u, extent_v, extent_l, rotate=rotate, resolution=resolution) this_layer_alpha_shape = make_alpha_shape( layer_vol, alpha_radius=alpha_radius) layer_alpha_shapes[layer] = this_layer_alpha_shape if geometry_path: save_alpha_shape(geometry_path, this_layer_alpha_shape_path, this_layer_alpha_shape) comm.barrier() population_ranges = read_population_ranges(output_path, comm)[0] if len(populations) == 0: populations = sorted(population_ranges.keys()) total_count = 0 for population in populations: (population_start, population_count) = population_ranges[population] total_count += population_count all_xyz_coords1 = None generated_coords_count_dict = defaultdict(int) if rank == 0: all_xyz_coords_lst = [] for population in populations: gc.collect() (population_start, population_count) = population_ranges[population] pop_layers = env.geometry['Cell Distribution'][population] pop_constraint = None if 'Cell Constraints' in env.geometry: if population in env.geometry['Cell Constraints']: pop_constraint = env.geometry['Cell Constraints'][ population] if rank == 0: logger.info("Population %s: layer distribution is %s" % (population, str(pop_layers))) pop_layer_count = 0 for layer, count in viewitems(pop_layers): pop_layer_count += count assert (population_count == pop_layer_count) xyz_coords_lst = [] for layer, count in viewitems(pop_layers): if count <= 0: continue alpha = layer_alpha_shapes[layer] vert = alpha.points smp = np.asarray(alpha.bounds, dtype=np.int64) extents_xyz = layer_extent_transformed_vals[layer] for (vvi, vv) in enumerate(vert): for (vi, v) in enumerate(vv): if v < extents_xyz[vi][0]: vert[vvi][vi] = extents_xyz[vi][0] elif v > extents_xyz[vi][1]: vert[vvi][vi] = extents_xyz[vi][1] N = int(count * 2) # layer-specific number of nodes node_count = 0 logger.info( "Generating %i nodes in layer %s for population %s..." % (N, layer, population)) if verbose: rbf_logger = logging.Logger.manager.loggerDict[ 'rbf.pde.nodes'] rbf_logger.setLevel(logging.DEBUG) min_energy_constraint = None if pop_constraint is not None and layer in pop_constraint: min_energy_constraint = pop_constraint[layer] nodes = gen_min_energy_nodes(count, (vert, smp), min_energy_constraint, nodeiter, dispersion_delta, snap_delta) #nodes = gen_min_energy_nodes(count, (vert, smp), # pop_constraint[layer] if pop_constraint is not None else None, # nodeiter, dispersion_delta, snap_delta) xyz_coords_lst.append(nodes.reshape(-1, 3)) for this_xyz_coords in xyz_coords_lst: all_xyz_coords_lst.append(this_xyz_coords) generated_coords_count_dict[population] += len(this_xyz_coords) # Additional dispersion step to ensure no overlapping cell positions all_xyz_coords = np.row_stack(all_xyz_coords_lst) mask = np.ones((all_xyz_coords.shape[0], ), dtype=np.bool) # distance to nearest neighbor while True: kdt = cKDTree(all_xyz_coords[mask, :]) nndist, nnindices = kdt.query(all_xyz_coords[mask, :], k=2) nndist, nnindices = nndist[:, 1:], nnindices[:, 1:] zindices = nnindices[np.argwhere( np.isclose(nndist, 0.0, atol=1e-3, rtol=1e-3))] if len(zindices) > 0: mask[np.argwhere(mask)[zindices]] = False else: break coords_offset = 0 for population in populations: pop_coords_count = generated_coords_count_dict[population] pop_mask = mask[coords_offset:coords_offset + pop_coords_count] generated_coords_count_dict[population] = np.count_nonzero( pop_mask) coords_offset += pop_coords_count logger.info("Dispersion of %i nodes..." % np.count_nonzero(mask)) all_xyz_coords1 = disperse(all_xyz_coords[mask, :], vol_domain, delta=dispersion_delta) if rank == 0: logger.info("Computing UVL coordinates of %i nodes..." % len(all_xyz_coords1)) all_xyz_coords_interp = None all_uvl_coords_interp = None if rank == 0: all_uvl_coords_interp = vol.inverse(all_xyz_coords1) all_xyz_coords_interp = vol(all_uvl_coords_interp[:, 0], all_uvl_coords_interp[:, 1], all_uvl_coords_interp[:, 2], mesh=False).reshape(3, -1).T if rank == 0: logger.info("Broadcasting generated nodes...") xyz_coords = comm.bcast(all_xyz_coords1, root=0) all_xyz_coords_interp = comm.bcast(all_xyz_coords_interp, root=0) all_uvl_coords_interp = comm.bcast(all_uvl_coords_interp, root=0) generated_coords_count_dict = comm.bcast(dict(generated_coords_count_dict), root=0) coords_offset = 0 pop_coords_dict = {} for population in populations: xyz_error = np.asarray([0.0, 0.0, 0.0]) pop_layers = env.geometry['Cell Distribution'][population] pop_start, pop_count = population_ranges[population] coords = [] gen_coords_count = generated_coords_count_dict[population] for i, coord_ind in enumerate( range(coords_offset, coords_offset + gen_coords_count)): if i % size == rank: uvl_coords = all_uvl_coords_interp[coord_ind, :].ravel() xyz_coords1 = all_xyz_coords_interp[coord_ind, :].ravel() if uvl_in_bounds(all_uvl_coords_interp[coord_ind, :], layer_extents, pop_layers): xyz_error = np.add( xyz_error, np.abs( np.subtract(xyz_coords[coord_ind, :], xyz_coords1))) logger.info('Rank %i: %s cell %i: %f %f %f' % (rank, population, i, uvl_coords[0], uvl_coords[1], uvl_coords[2])) coords.append( (xyz_coords1[0], xyz_coords1[1], xyz_coords1[2], uvl_coords[0], uvl_coords[1], uvl_coords[2])) else: logger.debug( 'Rank %i: %s cell %i not in bounds: %f %f %f' % (rank, population, i, uvl_coords[0], uvl_coords[1], uvl_coords[2])) uvl_coords = None xyz_coords1 = None total_xyz_error = np.zeros((3, )) comm.Allreduce(xyz_error, total_xyz_error, op=MPI.SUM) coords_count = 0 coords_count = np.sum(np.asarray(comm.allgather(len(coords)))) mean_xyz_error = np.asarray([(total_xyz_error[0] / coords_count), \ (total_xyz_error[1] / coords_count), \ (total_xyz_error[2] / coords_count)]) pop_coords_dict[population] = coords coords_offset += gen_coords_count if rank == 0: logger.info( 'Total %i coordinates generated for population %s: mean XYZ error: %f %f %f' % (coords_count, population, mean_xyz_error[0], mean_xyz_error[1], mean_xyz_error[2])) if rank == 0: color = 1 else: color = 0 ## comm0 includes only rank 0 comm0 = comm.Split(color, 0) for population in populations: pop_start, pop_count = population_ranges[population] pop_layers = env.geometry['Cell Distribution'][population] pop_constraint = None if 'Cell Constraints' in env.geometry: if population in env.geometry['Cell Constraints']: pop_constraint = env.geometry['Cell Constraints'][population] coords_lst = comm.gather(pop_coords_dict[population], root=0) if rank == 0: all_coords = [] for sublist in coords_lst: for item in sublist: all_coords.append(item) coords_count = len(all_coords) if coords_count < pop_count: logger.warning( "Generating additional %i coordinates for population %s..." % (pop_count - len(all_coords), population)) safety = 0.01 delta = pop_count - len(all_coords) for i in range(delta): for layer, count in viewitems(pop_layers): if count > 0: min_extent = layer_extents[layer][0] max_extent = layer_extents[layer][1] coord_u = np.random.uniform( min_extent[0] + safety, max_extent[0] - safety) coord_v = np.random.uniform( min_extent[1] + safety, max_extent[1] - safety) if pop_constraint is None: coord_l = np.random.uniform( min_extent[2] + safety, max_extent[2] - safety) else: coord_l = np.random.uniform( pop_constraint[layer][0] + safety, pop_constraint[layer][1] - safety) xyz_coords = CA1_volume(coord_u, coord_v, coord_l, rotate=rotate).ravel() all_coords.append( (xyz_coords[0], xyz_coords[1], xyz_coords[2], coord_u, coord_v, coord_l)) sampled_coords = random_subset(all_coords, int(pop_count)) sampled_coords.sort( key=lambda coord: coord[3]) ## sort on U coordinate coords_dict = { pop_start + i: { 'X Coordinate': np.asarray([x_coord], dtype=np.float32), 'Y Coordinate': np.asarray([y_coord], dtype=np.float32), 'Z Coordinate': np.asarray([z_coord], dtype=np.float32), 'U Coordinate': np.asarray([u_coord], dtype=np.float32), 'V Coordinate': np.asarray([v_coord], dtype=np.float32), 'L Coordinate': np.asarray([l_coord], dtype=np.float32) } for (i, (x_coord, y_coord, z_coord, u_coord, v_coord, l_coord)) in enumerate(sampled_coords) } append_cell_attributes(output_path, population, coords_dict, namespace=output_namespace, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, comm=comm0) comm.barrier() comm0.Free()
def main(connectivity_path, output_path, coords_path, distances_namespace, destination, bin_size, cache_size, verbose): """ Measures vertex distribution with respect to septo-temporal distance :param connectivity_path: :param coords_path: :param distances_namespace: :param destination: :param source: """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.Get_rank() (population_ranges, _) = read_population_ranges(coords_path) destination_start = population_ranges[destination][0] destination_count = population_ranges[destination][1] if rank == 0: logger.info('reading %s distances...' % destination) destination_soma_distances = bcast_cell_attributes( coords_path, destination, namespace=distances_namespace, comm=comm, root=0) destination_soma_distance_U = {} destination_soma_distance_V = {} for k, v in destination_soma_distances: destination_soma_distance_U[k] = v['U Distance'][0] destination_soma_distance_V[k] = v['V Distance'][0] del (destination_soma_distances) sources = [] for (src, dst) in read_projection_names(connectivity_path): if dst == destination: sources.append(src) source_soma_distances = {} for s in sources: if rank == 0: logger.info('reading %s distances...' % s) source_soma_distances[s] = bcast_cell_attributes( coords_path, s, namespace=distances_namespace, comm=comm, root=0) source_soma_distance_U = {} source_soma_distance_V = {} for s in sources: this_source_soma_distance_U = {} this_source_soma_distance_V = {} for k, v in source_soma_distances[s]: this_source_soma_distance_U[k] = v['U Distance'][0] this_source_soma_distance_V[k] = v['V Distance'][0] source_soma_distance_U[s] = this_source_soma_distance_U source_soma_distance_V[s] = this_source_soma_distance_V del (source_soma_distances) logger.info('reading connections %s -> %s...' % (str(sources), destination)) gg = [ NeuroH5ProjectionGen(connectivity_path, source, destination, cache_size=cache_size, comm=comm) for source in sources ] dist_bins = defaultdict(dict) dist_u_bins = defaultdict(dict) dist_v_bins = defaultdict(dict) for prj_gen_tuple in utils.zip_longest(*gg): destination_gid = prj_gen_tuple[0][0] if not all([ prj_gen_elt[0] == destination_gid for prj_gen_elt in prj_gen_tuple ]): raise Exception( 'destination %s: destination_gid %i not matched across multiple projection generators: %s' % (destination, destination_gid, [prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple])) if destination_gid is not None: logger.info('reading connections of gid %i' % destination_gid) for (source, (this_destination_gid, rest)) in zip(sources, prj_gen_tuple): this_source_soma_distance_U = source_soma_distance_U[source] this_source_soma_distance_V = source_soma_distance_V[source] this_dist_bins = dist_bins[source] this_dist_u_bins = dist_u_bins[source] this_dist_v_bins = dist_v_bins[source] (source_indexes, attr_dict) = rest dst_U = destination_soma_distance_U[destination_gid] dst_V = destination_soma_distance_V[destination_gid] for source_gid in source_indexes: dist_u = dst_U - this_source_soma_distance_U[source_gid] dist_v = dst_V - this_source_soma_distance_V[source_gid] dist = abs(dist_u) + abs(dist_v) update_bins(this_dist_bins, bin_size, dist) update_bins(this_dist_u_bins, bin_size, dist_u) update_bins(this_dist_v_bins, bin_size, dist_v) comm.barrier() logger.info('merging distance dictionaries...') add_bins_op = MPI.Op.Create(add_bins, commute=True) for source in sources: dist_bins[source] = comm.reduce(dist_bins[source], op=add_bins_op, root=0) dist_u_bins[source] = comm.reduce(dist_u_bins[source], op=add_bins_op, root=0) dist_v_bins[source] = comm.reduce(dist_v_bins[source], op=add_bins_op, root=0) comm.barrier() if rank == 0: color = 1 else: color = 0 ## comm0 includes only rank 0 comm0 = comm.Split(color, 0) if rank == 0: if output_path is None: output_path = connectivity_path logger.info('writing output to %s...' % output_path) #f = h5py.File(output_path, 'a', driver='mpio', comm=comm0) #if 'Nodes' in f: # nodes_grp = f['Nodes'] #else: # nodes_grp = f.create_group('Nodes') #grp = nodes_grp.create_group('Connectivity Distance Histogram') #dst_grp = grp.create_group(destination) for source in sources: dist_histoCount, dist_bin_edges = finalize_bins( dist_bins[source], bin_size) dist_u_histoCount, dist_u_bin_edges = finalize_bins( dist_u_bins[source], bin_size) dist_v_histoCount, dist_v_bin_edges = finalize_bins( dist_v_bins[source], bin_size) np.savetxt('%s Distance U Bin Count.dat' % source, dist_u_histoCount) np.savetxt('%s Distance U Bin Edges.dat' % source, dist_u_bin_edges) np.savetxt('%s Distance V Bin Count.dat' % source, dist_v_histoCount) np.savetxt('%s Distance V Bin Edges.dat' % source, dist_v_bin_edges) np.savetxt('%s Distance Bin Count.dat' % source, dist_histoCount) np.savetxt('%s Distance Bin Edges.dat' % source, dist_bin_edges) #f.close() comm.barrier()
def main(forest_path, connectivity_namespace, coords_path, coords_namespace, io_size, chunk_size, value_chunk_size, cache_size): """ :param forest_path: :param connectivity_namespace: :param coords_path: :param coords_namespace: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ comm = MPI.COMM_WORLD rank = comm.rank # The process ID (integer 0-3 for 4-process run) if io_size == -1: io_size = comm.size if rank == 0: print('%i ranks have been allocated' % comm.size) sys.stdout.flush() start_time = time.time() soma_coords = {} source_populations = list(read_population_ranges(MPI._addressof(comm), coords_path).keys()) for population in source_populations: soma_coords[population] = bcast_cell_attributes(MPI._addressof(comm), 0, coords_path, population, namespace=coords_namespace) for population in soma_coords: for cell in viewvalues(soma_coords[population]): cell['u_index'] = get_array_index(u, cell['U Coordinate'][0]) cell['v_index'] = get_array_index(v, cell['V Coordinate'][0]) target = 'GC' layer_set, swc_type_set, syn_type_set = set(), set(), set() for source in layers[target]: layer_set.update(layers[target][source]) swc_type_set.update(swc_types[target][source]) syn_type_set.update(syn_types[target][source]) count = 0 for target_gid, attributes_dict in NeuroH5CellAttrGen(MPI._addressof(comm), forest_path, target, io_size=io_size, cache_size=cache_size, namespace='Synapse_Attributes'): last_time = time.time() connection_dict = {} p_dict = {} source_gid_dict = {} if target_gid is None: print('Rank %i target gid is None' % rank) else: print('Rank %i received attributes for target: %s, gid: %i' % (rank, target, target_gid)) synapse_dict = attributes_dict['Synapse_Attributes'] connection_dict[target_gid] = {} local_np_random.seed(target_gid + connectivity_seed_offset) connection_dict[target_gid]['source_gid'] = np.array([], dtype='uint32') connection_dict[target_gid]['syn_id'] = np.array([], dtype='uint32') for layer in layer_set: for swc_type in swc_type_set: for syn_type in syn_type_set: sources, this_proportions = filter_sources(target, layer, swc_type, syn_type) if sources: if rank == 0 and count == 0: source_list_str = '[' + ', '.join(['%s' % xi for xi in sources]) + ']' print('Connections to target: %s in layer: %i ' \ '(swc_type: %i, syn_type: %i): %s' % \ (target, layer, swc_type, syn_type, source_list_str)) p, source_gid = np.array([]), np.array([]) for source, this_proportion in zip(sources, this_proportions): if source not in source_gid_dict: this_p, this_source_gid = p_connect.get_p(target, source, target_gid, soma_coords, distance_U, distance_V) source_gid_dict[source] = this_source_gid p_dict[source] = this_p else: this_source_gid = source_gid_dict[source] this_p = p_dict[source] p = np.append(p, this_p * this_proportion) source_gid = np.append(source_gid, this_source_gid) syn_indexes = filter_synapses(synapse_dict, layer, swc_type, syn_type) connection_dict[target_gid]['syn_id'] = \ np.append(connection_dict[target_gid]['syn_id'], synapse_dict['syn_id'][syn_indexes]).astype('uint32', copy=False) this_source_gid = local_np_random.choice(source_gid, len(syn_indexes), p=p) connection_dict[target_gid]['source_gid'] = \ np.append(connection_dict[target_gid]['source_gid'], this_source_gid).astype('uint32', copy=False) count += 1 print('Rank %i took %i s to compute connectivity for target: %s, gid: %i' % (rank, time.time() - last_time, target, target_gid)) sys.stdout.flush() last_time = time.time() append_cell_attributes(MPI._addressof(comm), forest_path, target, connection_dict, namespace=connectivity_namespace, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) if rank == 0: print('Appending connectivity attributes for target: %s took %i s' % (target, time.time() - last_time)) sys.stdout.flush() del connection_dict del p_dict del source_gid_dict gc.collect() global_count = comm.gather(count, root=0) if rank == 0: print('%i ranks took took %i s to compute connectivity for %i cells' % (comm.size, time.time() - start_time, np.sum(global_count)))