def config_controller(): """ """ utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) try: context.env = Env(comm=context.controller_comm, **context.kwargs) except Exception as err: context.logger.exception(err) raise err opt_param_config = optimization_params( context.env.netclamp_config.optimize_parameters, context.target_populations, context.param_config_name) param_bounds = opt_param_config.param_bounds param_names = opt_param_config.param_bounds param_initial_dict = opt_param_config.param_initial_dict param_tuples = opt_param_config.param_tuples opt_targets = opt_param_config.opt_targets context.param_names = param_names context.bounds = [param_bounds[key] for key in param_names] context.x0 = param_initial_dict context.target_val = opt_targets context.target_range = opt_targets context.param_tuples = param_tuples # These kwargs will be sent from the controller to each worker context context.kwargs['param_tuples'] = param_tuples
def main(inst_rates_path, inst_rates_namespace, include, bin_size, nstdev, baseline_fraction, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) if not include: population_names = read_population_names(inst_rates_path) for pop in population_names: include.append(pop) for i, population in enumerate(include): rate_inst_iter = read_cell_attributes(inst_rates_path, population, namespace=inst_rates_namespace) rate_inst_dict = dict(rate_inst_iter) spikedata.place_fields(population, bin_size, rate_inst_dict, nstdev, baseline_fraction=baseline_fraction, saveData=inst_rates_path)
def main(coords_path, population, distances_namespace, reindex_namespace, reindex_attribute, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(script_name)) env = Env(config_file=config, config_prefix=config_prefix) plot.plot_reindex_positions (env, coords_path, population, distances_namespace, reindex_namespace, reindex_attribute)
def main(gid, pop_name, config_file, template_paths, hoc_lib_path, dataset_prefix, config_prefix, mech_file, load_edges, load_weights, correct_for_spines, verbose): """ :param gid: int :param pop_name: str :param config_file: str; model configuration file name :param template_paths: str; colon-separated list of paths to directories containing hoc cell templates :param hoc_lib_path: str; path to directory containing required hoc libraries :param dataset_prefix: str; path to directory containing required neuroh5 data files :param config_prefix: str; path to directory containing network and cell mechanism config files :param mech_file: str; cell mechanism config file name :param load_edges: bool; whether to attempt to load connections from a neuroh5 file :param load_weights: bool; whether to attempt to load connections from a neuroh5 file :param correct_for_spines: bool :param verbose: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD np.seterr(all='raise') env = Env(comm=comm, config_file=config_file, template_paths=template_paths, hoc_lib_path=hoc_lib_path, dataset_prefix=dataset_prefix, config_prefix=config_prefix, verbose=verbose) configure_hoc_env(env) mech_file_path = config_prefix + '/' + mech_file template_name = env.celltypes[pop_name]['template'] if template_name.lower() == 'izhikevich': cell = make_izhikevich_cell(env, pop_name=pop_name, gid=gid, load_synapses=True, load_connections=True, load_edges=load_edges, load_weights=load_weights, mech_file_path=mech_file_path) elif template_name.lower() == 'pr_nrn': cell = make_PR_cell(env, pop_name=pop_name, gid=gid, load_synapses=True, load_connections=True, load_edges=load_edges, load_weights=load_weights, mech_file_path=mech_file_path) else: cell = make_biophys_cell(env, pop_name=pop_name, gid=gid, load_synapses=True, load_connections=True, load_edges=load_edges, load_weights=load_weights, mech_file_path=mech_file_path) context.update(locals()) init_biophysics(cell, reset_cable=True, correct_cm=correct_for_spines, correct_g_pas=correct_for_spines, env=env, verbose=verbose) init_syn_mech_attrs(cell, env) config_biophys_cell_syns(env, gid, pop_name, insert=True, insert_netcons=True, insert_vecstims=True, verbose=verbose) if verbose: for sec in list(cell.hoc_cell.all if hasattr(cell, 'hoc_cell') else cell.all): h.psection(sec=sec) report_topology(cell, env)
def main(config, template_path, prototype_gid, prototype_path, forest_path, population, io_size, verbose): """ :param config: :param template_path: :param prototype_gid: :param prototype_path: :param forest_path: :param population: :param io_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) configure_hoc_env(env) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) layers = env.layers layer_idx_dict = { layers[layer_name]: layer_name for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus'] } (tree_iter, _) = read_tree_selection(prototype_path, population, selection=[prototype_gid]) (_, prototype_morph_dict) = next(tree_iter) prototype_x = prototype_morph_dict['x'] prototype_y = prototype_morph_dict['y'] prototype_z = prototype_morph_dict['z'] prototype_xyz = (prototype_x, prototype_y, prototype_z) (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() (population_start, _) = pop_ranges[population] template_class = load_cell_template(env, population, bcast_template=True) for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, cache_size=1, comm=comm, topology=True): # trees, _ = scatter_read_trees(forest_path, population, io_size=io_size, comm=comm, topology=True) # for gid, morph_dict in trees: if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) secnodes_dict = morph_dict['section_topology']['nodes'] vx = morph_dict['x'] vy = morph_dict['y'] vz = morph_dict['z'] if compare_points((vx,vy,vz), prototype_xyz): logger.info('Possible match: gid %i' % gid) MPI.Finalize()
def main(config_path, params_id, output_file_name, verbose): config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] network_param_values = eval_config['param_values'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) def from_param_list(x): result = [] for i, (param_name, param_tuple) in enumerate( zip(network_param_spec.param_names, network_param_spec.param_tuples)): param_range = param_tuple.param_range # assert((x[i] >= param_range[0]) and (x[i] <= param_range[1])) result.append((param_tuple, x[i])) return result x = network_param_values[params_id] param_tuple_values = from_param_list(x) def rec_dd(): return defaultdict(rec_dd) def dd2dict(d): for k, v in d.items(): if isinstance(v, dict): d[k] = dd2dict(v) return dict(d) param_output_ddict = rec_dd() for param_tuple, param_value in param_tuple_values: if isinstance(param_tuple.param_path, tuple): param_output_ddict[param_tuple.population][param_tuple.source][ param_tuple.sec_type][param_tuple.syn_name][ param_tuple.param_path[0]][ param_tuple.param_path[1]] = param_value else: param_output_ddict[param_tuple.population][param_tuple.source][ param_tuple.sec_type][param_tuple.syn_name][ param_tuple.param_path] = param_value param_output_dict = dd2dict(param_output_ddict) pprint.pprint(param_output_dict) write_to_yaml(output_file_name, param_output_dict)
def main(config, config_prefix, resolution, resample, alpha_radius, graph_type, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) env = Env(config_file=config, config_prefix=config_prefix) layers = env.layers rotate = env.geometry['Parametric Surface']['Rotation'] min_u = float('inf') max_u = 0.0 min_v = float('inf') max_v = 0.0 min_l = float('inf') max_l = 0.0 for layer in list(layers.keys()): min_extent = env.geometry['Parametric Surface']['Minimum Extent'][ layer] max_extent = env.geometry['Parametric Surface']['Maximum Extent'][ layer] min_u = min(min_extent[0], min_u) max_u = max(max_extent[0], max_u) min_v = min(min_extent[1], min_v) max_v = max(max_extent[1], max_v) min_l = min(min_extent[2], min_l) max_l = max(max_extent[2], max_l) logger.info('Creating volume: min_l = %f max_l = %f...' % (min_l, max_l)) ip_volume = make_volume((min_u, max_u), \ (min_v, max_v), \ (min_l, max_l), \ resolution=resolution, \ rotate=rotate) logger.info('Computing volume distances...') vol_dist = get_volume_distances(ip_volume, res=resample, alpha_radius=alpha_radius) (obs_uv, dist_u, dist_v) = vol_dist dist_dict = {} for i in range(0, len(dist_u)): dist_dict[i] = { 'U Distance': np.asarray([dist_u[i]], dtype=np.float32), \ 'V Distance': np.asarray([dist_v[i]], dtype=np.float32) } plot.plot_positions(env, "DG Volume", dist_dict, verbose=verbose, saveFig=True, graphType=graph_type)
def main(config, config_prefix, connectivity_path, coords_path, distances_namespace, target_gid, destination, source, extent_type, direction, normed, bin_size, font_size, save_format, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(script_name)) env = Env(config_file=config, config_prefix=config_prefix) plot.plot_single_vertex_dist (env, connectivity_path, coords_path, distances_namespace, \ target_gid, destination, source, direction=direction, \ normed=normed, extent_type=extent_type, bin_size=bin_size, \ fontSize=font_size, saveFig=True, figFormat=save_format)
def init_network_objfun(operational_config, opt_targets, param_names, param_tuples, worker, **kwargs): param_tuples = [ syn_param_from_dict(param_tuple) for param_tuple in param_tuples ] objective_names = operational_config['objective_names'] target_populations = operational_config['target_populations'] target_features_path = operational_config['target_features_path'] target_features_namespace = operational_config['target_features_namespace'] kwargs['results_file_id'] = 'DG_optimize_network_%d_%s' % \ (worker.worker_id, operational_config['run_ts']) logger = utils.get_script_logger(os.path.basename(__file__)) env = init_network(comm=MPI.COMM_WORLD, kwargs=kwargs) gc.collect() t_start = 50. t_stop = env.tstop time_range = (t_start, t_stop) target_trj_rate_map_dict = {} target_features_arena = env.arena_id target_features_trajectory = env.trajectory_id for pop_name in target_populations: if ('%s target rate dist residual' % pop_name) not in objective_names: continue my_cell_index_set = set(env.biophys_cells[pop_name].keys()) trj_rate_maps = {} trj_rate_maps = rate_maps_from_features( env, pop_name, cell_index_set=list(my_cell_index_set), input_features_path=target_features_path, input_features_namespace=target_features_namespace, time_range=time_range) target_trj_rate_map_dict[pop_name] = trj_rate_maps def from_param_dict(params_dict): result = [] for param_name, param_tuple in zip(param_names, param_tuples): result.append((param_tuple, params_dict[param_name])) return result return partial(network_objfun, env, operational_config, opt_targets, target_trj_rate_map_dict, from_param_dict, t_start, t_stop, target_populations)
def main(config, coords_path, coords_namespace, resample, resolution, populations, projection_depth, io_size, chunk_size, value_chunk_size, cache_size, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config) soma_coords = {} if rank == 0: logger.info('Reading population coordinates...') rotate = env.geometry['Parametric Surface']['Rotation'] min_l = float('inf') max_l = 0.0 population_ranges = read_population_ranges(coords_path)[0] population_extents = {} for population in population_ranges: min_extent = env.geometry['Cell Layers']['Minimum Extent'][population] max_extent = env.geometry['Cell Layers']['Maximum Extent'][population] min_l = min(min_extent[2], min_l) max_l = max(max_extent[2], max_l) population_extents[population] = (min_extent, max_extent) for population in populations: coords = bcast_cell_attributes(coords_path, population, 0, \ namespace=coords_namespace) soma_coords[population] = { k: (v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0]) for (k,v) in coords } del coords gc.collect() output_path = coords_path soma_coords = icp_transform(comm, soma_coords, projection_depth, population_extents, \ populations=populations, rotate=rotate, verbose=verbose) for population in populations: if rank == 0: logger.info('Writing transformed coordinates for population %s...' % population) append_cell_attributes(output_path, population, soma_coords[population], namespace='Soma Projections', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size)
def config_worker(): """ """ utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) if 'results_id' not in context(): context.results_id = 'DG_test_network_subworlds_%s_%s' % \ (context.interface.worker_id, datetime.datetime.today().strftime('%Y%m%d_%H%M')) if 'env' not in context(): try: init_network() except Exception as err: context.logger.exception(err) raise err context.bin_size = 5.0
def config_controller(): """ """ utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) if 'results_file_id' not in context(): context.results_file_id = 'DG_optimize_network_subworlds_%s_%s' % \ (context.interface.worker_id, datetime.datetime.today().strftime('%Y%m%d_%H%M')) if 'env' not in context(): try: context.comm = MPI.COMM_WORLD #init_env() except Exception as err: context.logger.exception(err) raise err
def config_controller(): """ """ utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) context.init_params = context.kwargs context.init_params['target_rate_map_arena'] = context.init_params['arena_id'] context.init_params['target_rate_map_trajectory'] = context.init_params['trajectory_id'] context.gid = int(context.init_params['gid']) context.target_val = {} if 'results_file_id' not in context(): context.results_file_id = 'DG_optimize_pf_%s_%s' % \ (context.interface.worker_id, datetime.datetime.today().strftime('%Y%m%d_%H%M'))
def main(config_file, population, gid, template_paths, dataset_prefix, config_prefix, load_synapses, syn_types, syn_sources, syn_source_threshold, font_size, bgcolor, colormap, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) params = dict(locals()) env = Env(**params) configure_hoc_env(env) ## Determine if a mechanism configuration file exists for this cell type if 'mech_file_path' in env.celltypes[population]: mech_file_path = env.celltypes[population]['mech_file_path'] else: mech_file_path = None logger.info('loading cell %i' % gid) load_weights = False biophys_cell = get_biophys_cell(env, population, gid, load_synapses=load_synapses, load_weights=load_weights, load_edges=load_synapses, mech_file_path=mech_file_path) if len(syn_types) == 0: syn_types = None else: syn_types = list(syn_types) if len(syn_sources) == 0: syn_sources = None else: syn_sources = list(syn_sources) plot.plot_biophys_cell_tree(env, biophys_cell, saveFig=True, syn_source_threshold=syn_source_threshold, synapse_filters={ 'syn_types': syn_types, 'sources': syn_sources }, bgcolor=bgcolor, colormap=colormap)
def main(config_path, params_id, output_file_name, verbose): config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] network_param_values = eval_config['param_values'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) def from_param_list(x): result = [] for i, (param_name, param_tuple) in enumerate( zip(network_param_spec.param_names, network_param_spec.param_tuples)): param_range = param_tuple.param_range # assert((x[i] >= param_range[0]) and (x[i] <= param_range[1])) result.append((param_tuple, x[i])) return result params_id_list = [] if params_id is None: params_id_list = list(network_param_values.keys()) else: params_id_list = [params_id] param_output_dict = dict() for this_params_id in params_id_list: x = network_param_values[this_params_id] param_tuple_values = from_param_list(x) this_param_list = [] for param_tuple, param_value in param_tuple_values: this_param_list.append((param_tuple.population, param_tuple.source, param_tuple.sec_type, param_tuple.syn_name, param_tuple.param_path, param_value)) param_output_dict[this_params_id] = this_param_list pprint.pprint(param_output_dict) if output_file_name is not None: write_to_yaml(output_file_name, param_output_dict)
def main(config, coords_path, coords_namespace, distances_namespace, populations, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(__file__) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config) output_path = coords_path soma_coords = {} soma_distances = {} if rank == 0: logger.info('Reading population coordinates and distances...') for population in populations: coords = bcast_cell_attributes(coords_path, population, 0, namespace=coords_namespace, comm=comm) soma_coords[population] = { k: (v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0]) for (k, v) in coords } del coords gc.collect() distances = bcast_cell_attributes(coords_path, population, 0, namespace=distances_namespace, comm=comm) soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k, v) in distances } del distances gc.collect()
def generate_param_lattice(config_path, n_samples, output_file_dir, output_file_name, maxiter=5, verbose=False): from dmosopt import sampling logger = utils.get_script_logger(os.path.basename(__file__)) output_path = None if output_file_name is not None: output_path = f'{output_file_dir}/{output_file_name}' eval_config = read_from_yaml(config_path, include_loader=utils.IncludeLoader) network_param_spec_src = eval_config['param_spec'] target_populations = eval_config['target_populations'] network_param_spec = make_param_spec(target_populations, network_param_spec_src) param_tuples = network_param_spec.param_tuples param_names = network_param_spec.param_names n_params = len(param_tuples) n_init = n_params * n_samples Xinit = sampling.glp(n_init, n_params, maxiter=maxiter) ub = [] lb = [] for param_name, param_tuple in zip(param_names, param_tuples): param_range = param_tuple.param_range ub.append(param_range[1]) lb.append(param_range[0]) ub = np.asarray(ub) lb = np.asarray(lb) for i in range(n_init): Xinit[i,:] = Xinit[i,:] * (ub - lb) + lb output_dict = {} for i in range(Xinit.shape[0]): output_dict[i] = list([float(x) for x in Xinit[i, :]]) if output_path is not None: write_to_yaml(output_path, output_dict) else: pprint.pprint(output_dict)
def main(coords_path, io_size, chunk_size, value_chunk_size): utils.config_logging(verbose) logger = utils.get_script_logger(__file__) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config) output_path = coords_path if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) source_population_ranges = read_population_ranges(coords_path) source_populations = list(source_population_ranges.keys()) for population in source_populations: if rank == 0: logger.info('population: ',population) soma_coords = bcast_cell_attributes(0, coords_path, population, namespace='Interpolated Coordinates', comm=comm) #print soma_coords.keys() u_coords = [] gids = [] for gid, attrs in viewitems(soma_coords): u_coords.append(attrs['U Coordinate']) gids.append(gid) u_coordv = np.asarray(u_coords, dtype=np.float32) gidv = np.asarray(gids, dtype=np.uint32) sort_idx = np.argsort(u_coordv, axis=0) offset = source_population_ranges[population][0] sorted_coords_dict = {} for i in range(0,sort_idx.size): sorted_coords_dict[offset+i] = soma_coords[gidv[sort_idx[i][0]]] append_cell_attributes(coords_path, population, sorted_coords_dict, namespace='Sorted Coordinates', io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, comm=comm)
def main(config, config_prefix, connectivity_path, coords_path, vertex_metrics_namespace, distances_namespace, destination, sources, normed, metric, graph_type, bin_size, font_size, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(script_name) env = Env(config_file=config, config_prefix=config_prefix) plot.plot_vertex_metrics(env, connectivity_path, coords_path, vertex_metrics_namespace, distances_namespace, destination, sources, metric=metric, normed=normed, bin_size=bin_size, fontSize=font_size, graph_type=graph_type, saveFig=True)
def main(config, config_prefix, features_path, coords_path, features_namespace, arena_id, trajectory_id, distances_namespace, include, bin_size, from_spikes, normed, font_size, verbose, save_fig): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(script_name)) env = Env(config_file=config, config_prefix=config_prefix) plot.plot_stimulus_spatial_rate_map(env, features_path, coords_path, arena_id, trajectory_id, features_namespace, distances_namespace, include, bin_size=bin_size, from_spikes=from_spikes, normed=normed, fontSize=font_size, saveFig=save_fig, verbose=verbose)
def main(config, config_prefix, max_section_length, population, forest_path, template_path, output_path, io_size, chunk_size, value_chunk_size, dry_run, verbose): """ :param population: str :param forest_path: str (path) :param output_path: str (path) :param io_size: int :param chunk_size: int :param value_chunk_size: int :param verbose: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) env = Env(comm=comm, config_file=config, config_prefix=config_prefix, template_paths=template_path) if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (forest_pop_ranges, _) = read_population_ranges(forest_path) (forest_population_start, forest_population_count) = forest_pop_ranges[population] (pop_ranges, _) = read_population_ranges(output_path) (population_start, population_count) = pop_ranges[population] new_trees_dict = {} for gid, tree_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=False): if gid is not None: logger.info("Rank %d received gid %d" % (rank, gid)) logger.info(pprint.pformat(tree_dict)) new_tree_dict = cells.resize_tree_sections(tree_dict, max_section_length) logger.info(pprint.pformat(new_tree_dict)) new_trees_dict[gid] = new_tree_dict if not dry_run: append_cell_trees(output_path, population, new_trees_dict, io_size=io_size, comm=comm) comm.barrier() if (not dry_run) and (rank == 0): logger.info('Appended resized trees to %s' % output_path)
def main(arena_id, config, config_prefix, dataset_prefix, distances_namespace, spike_input_path, spike_input_namespace, spike_input_attr, input_features_namespaces, input_features_path, selection_path, output_path, io_size, trajectory_id, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if io_size == -1: io_size = comm.size env = Env(comm=comm, config_file=config, config_prefix=config_prefix, dataset_prefix=dataset_prefix, results_path=output_path, spike_input_path=spike_input_path, spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr, arena_id=arena_id, trajectory_id=trajectory_id, io_size=io_size) selection = [] f = open(selection_path, 'r') for line in f.readlines(): selection.append(int(line)) f.close() selection = set(selection) pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm) distance_U_dict = {} distance_V_dict = {} range_U_dict = {} range_V_dict = {} selection_dict = defaultdict(set) comm0 = env.comm.Split(2 if rank == 0 else 0, 0) if rank == 0: for population in pop_ranges: distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0) soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k,v) in distances } del distances numitems = len(list(soma_distances.keys())) if numitems == 0: continue distance_U_array = np.asarray([soma_distances[gid][0] for gid in soma_distances]) distance_V_array = np.asarray([soma_distances[gid][1] for gid in soma_distances]) U_min = np.min(distance_U_array) U_max = np.max(distance_U_array) V_min = np.min(distance_V_array) V_max = np.max(distance_V_array) range_U_dict[population] = (U_min, U_max) range_V_dict[population] = (V_min, V_max) distance_U = { gid: soma_distances[gid][0] for gid in soma_distances } distance_V = { gid: soma_distances[gid][1] for gid in soma_distances } distance_U_dict[population] = distance_U distance_V_dict[population] = distance_V min_dist = U_min max_dist = U_max selection_dict[population] = set([ k for k in distance_U if k in selection ]) env.comm.barrier() write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName) if rank == 0: io_utils.mkout(env, write_selection_file_path) env.comm.barrier() selection_dict = env.comm.bcast(dict(selection_dict), root=0) env.cell_selection = selection_dict io_utils.write_cell_selection(env, write_selection_file_path) input_selection = io_utils.write_connection_selection(env, write_selection_file_path) if spike_input_path: io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path) if input_features_path: for this_input_features_namespace in sorted(input_features_namespaces): for population in sorted(input_selection): logger.info(f"Extracting input features {this_input_features_namespace} for population {population}...") it = read_cell_attribute_selection(input_features_path, population, namespace=f"{this_input_features_namespace} {arena_id}", selection=input_selection[population], comm=env.comm) output_features_dict = { cell_gid : cell_features_dict for cell_gid, cell_features_dict in it } append_cell_attributes(write_selection_file_path, population, output_features_dict, namespace=f"{this_input_features_namespace} {arena_id}", io_size=io_size, comm=env.comm) env.comm.barrier()
def config_worker(): """ """ utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) if 'results_file_id' not in context(): context.results_file_id = 'DG_optimize_network_subworlds_%s_%s' % \ (context.interface.worker_id, datetime.datetime.today().strftime('%Y%m%d_%H%M')) if 'env' not in context(): try: context.comm = MPI.COMM_WORLD init_network() except Exception as err: context.logger.exception(err) raise err context.bin_size = 5.0 param_bounds = {} param_names = [] param_initial_dict = {} param_range_tuples = [] opt_targets = {} for pop_name in context.target_populations: if (pop_name in context.env.netclamp_config.optimize_parameters): opt_params = context.env.netclamp_config.optimize_parameters[pop_name] param_ranges = opt_params['Parameter ranges'] else: raise RuntimeError( "optimize_network_subworlds: population %s does not have optimization configuration" % pop_name) for target_name, target_val in viewitems(opt_params['Targets']): opt_targets['%s %s' % (pop_name, target_name)] = target_val for source, source_dict in sorted(viewitems(param_ranges), key=lambda k_v3: k_v3[0]): for sec_type, sec_type_dict in sorted(viewitems(source_dict), key=lambda k_v2: k_v2[0]): for syn_name, syn_mech_dict in sorted(viewitems(sec_type_dict), key=lambda k_v1: k_v1[0]): for param_fst, param_rst in sorted(viewitems(syn_mech_dict), key=lambda k_v: k_v[0]): if isinstance(param_rst, dict): for const_name, const_range in sorted(viewitems(param_rst)): param_path = (param_fst, const_name) param_range_tuples.append((pop_name, source, sec_type, syn_name, param_path, const_range)) param_key = '%s.%s.%s.%s.%s.%s' % (pop_name, source, sec_type, syn_name, param_fst, const_name) param_initial_value = (const_range[1] - const_range[0]) / 2.0 param_initial_dict[param_key] = param_initial_value param_bounds[param_key] = const_range param_names.append(param_key) else: param_name = param_fst param_range = param_rst param_range_tuples.append((pop_name, source, sec_type, syn_name, param_name, param_range)) param_key = '%s.%s.%s.%s.%s' % (pop_name, source, sec_type, syn_name, param_name) param_initial_value = (param_range[1] - param_range[0]) / 2.0 param_initial_dict[param_key] = param_initial_value param_bounds[param_key] = param_range param_names.append(param_key) def from_param_vector(params): result = [] assert (len(params) == len(param_range_tuples)) for i, (pop_name, source, sec_type, syn_name, param_name, param_range) in enumerate(param_range_tuples): result.append((pop_name, source, sec_type, syn_name, param_name, params[i])) return result def to_param_vector(params): result = [] for (source, sec_type, syn_name, param_name, param_value) in params: result.append(param_value) return result context.param_names = param_names context.bounds = [ param_bounds[key] for key in param_names ] context.x0 = param_initial_dict context.from_param_vector = from_param_vector context.to_param_vector = to_param_vector context.target_val = opt_targets context.target_range = opt_targets
def main(config, coordinates, gid, field_width, peak_rate, input_features_path, input_features_namespaces, output_features_namespace, output_weights_path, output_features_path, initial_weights_path, reference_weights_path, h5types_path, synapse_name, initial_weights_namespace, reference_weights_namespace, output_weights_namespace, reference_weights_are_delta, connections_path, optimize_method, destination, sources, arena_id, max_delta_weight, field_width_scale, max_iter, verbose, dry_run, plot): """ :param config: str (path to .yaml file) :param coordinates: tuple of float :param gid: int :param field_width: float :param peak_rate: float :param input_features_path: str (path to .h5 file) :param input_features_namespaces: str :param output_features_namespace: str :param output_weights_path: str (path to .h5 file) :param output_features_path: str (path to .h5 file) :param initial_weights_path: str (path to .h5 file) :param reference_weights_path: str (path to .h5 file) :param h5types_path: str (path to .h5 file) :param synapse_name: str :param initial_weights_namespace: str :param output_weights_namespace: str :param reference_weights_are_delta: bool :param connections_path: str (path to .h5 file) :param destination: str (population name) :param sources: list of str (population name) :param arena_id: str :param max_delta_weight: float :param field_width_scale: float :param max_iter: int :param verbose: bool :param dry_run: bool :param interactive: bool :param plot: bool """ utils.config_logging(verbose) logger = utils.get_script_logger(__file__) env = Env(config_file=config) if not dry_run: if output_weights_path is None: raise RuntimeError( 'Missing required argument: output_weights_path.') if not os.path.isfile(output_weights_path): if initial_weights_path is not None and os.path.isfile( initial_weights_path): input_file_path = initial_weights_path elif h5types_path is not None and os.path.isfile(h5types_path): input_file_path = h5types_path else: raise RuntimeError( 'Missing required source for h5types: either an initial_weights_path or an ' 'h5types_path must be provided.') with h5py.File(output_weights_path, 'a') as output_file: with h5py.File(input_file_path, 'r') as input_file: input_file.copy('/H5Types', output_file) this_input_features_namespaces = [ '%s %s' % (input_features_namespace, arena_id) for input_features_namespace in input_features_namespaces ] features_attr_names = ['Arena Rate Map'] spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm arena = env.stimulus_config['Arena'][arena_id] default_run_vel = arena.properties['default run velocity'] # cm/s arena_x, arena_y = stimulus.get_2D_arena_spatial_mesh( arena, spatial_resolution) dim_x = len(arena_x) dim_y = len(arena_y) if gid is None: target_gids = [] else: target_gids = [gid] dst_input_features = defaultdict(dict) num_fields = len(coordinates) this_field_width = np.array([field_width] * num_fields, dtype=np.float32) this_scaled_field_width = np.array([field_width * field_width_scale] * num_fields, dtype=np.float32) this_peak_rate = np.array([peak_rate] * num_fields, dtype=np.float32) this_x0 = np.array([x for x, y in coordinates], dtype=np.float32) this_y0 = np.array([y for x, y in coordinates], dtype=np.float32) this_rate_map = np.asarray(get_rate_map(this_x0, this_y0, this_field_width, this_peak_rate, arena_x, arena_y), dtype=np.float32) target_map = np.asarray(get_rate_map(this_x0, this_y0, this_scaled_field_width, this_peak_rate, arena_x, arena_y), dtype=np.float32) selectivity_type = env.selectivity_types['place'] dst_input_features[destination][target_gid] = { 'Selectivity Type': np.array([selectivity_type], dtype=np.uint8), 'Num Fields': np.array([num_fields], dtype=np.uint8), 'Field Width': this_field_width, 'Peak Rate': this_peak_rate, 'X Offset': this_x0, 'Y Offset': this_y0, 'Arena Rate Map': this_rate_map.ravel() } initial_weights_by_syn_id_dict = dict() selection = [target_gid] if initial_weights_path is not None: initial_weights_iter = \ read_cell_attribute_selection(initial_weights_path, destination, namespace=initial_weights_namespace, selection=selection) syn_weight_attr_dict = dict(initial_weights_iter) syn_ids = syn_weight_attr_dict[target_gid]['syn_id'] weights = syn_weight_attr_dict[target_gid][synapse_name] for (syn_id, weight) in zip(syn_ids, weights): initial_weights_by_syn_id_dict[int(syn_id)] = float(weight) logger.info( 'destination: %s; gid %i; read initial synaptic weights for %i synapses' % (destination, target_gid, len(initial_weights_by_syn_id_dict))) reference_weights_by_syn_id_dict = None if reference_weights_path is not None: reference_weights_by_syn_id_dict = dict() reference_weights_iter = \ read_cell_attribute_selection(reference_weights_path, destination, namespace=reference_weights_namespace, selection=selection) syn_weight_attr_dict = dict(reference_weights_iter) syn_ids = syn_weight_attr_dict[target_gid]['syn_id'] weights = syn_weight_attr_dict[target_gid][synapse_name] for (syn_id, weight) in zip(syn_ids, weights): reference_weights_by_syn_id_dict[int(syn_id)] = float(weight) logger.info( 'destination: %s; gid %i; read reference synaptic weights for %i synapses' % (destination, target_gid, len(reference_weights_by_syn_id_dict))) source_gid_set_dict = defaultdict(set) syn_ids_by_source_gid_dict = defaultdict(list) initial_weights_by_source_gid_dict = dict() if reference_weights_by_syn_id_dict is None: reference_weights_by_source_gid_dict = None else: reference_weights_by_source_gid_dict = dict() (graph, edge_attr_info) = read_graph_selection(file_name=connections_path, selection=[target_gid], namespaces=['Synapses']) syn_id_attr_index = None for source, edge_iter in viewitems(graph[destination]): if source not in sources: continue this_edge_attr_info = edge_attr_info[destination][source] if 'Synapses' in this_edge_attr_info and \ 'syn_id' in this_edge_attr_info['Synapses']: syn_id_attr_index = this_edge_attr_info['Synapses']['syn_id'] for (destination_gid, edges) in edge_iter: assert destination_gid == target_gid source_gids, edge_attrs = edges syn_ids = edge_attrs['Synapses'][syn_id_attr_index] count = 0 for i in range(len(source_gids)): this_source_gid = int(source_gids[i]) source_gid_set_dict[source].add(this_source_gid) this_syn_id = int(syn_ids[i]) if this_syn_id not in initial_weights_by_syn_id_dict: this_weight = \ env.connection_config[destination][source].mechanisms['default'][synapse_name]['weight'] initial_weights_by_syn_id_dict[this_syn_id] = this_weight syn_ids_by_source_gid_dict[this_source_gid].append(this_syn_id) if this_source_gid not in initial_weights_by_source_gid_dict: initial_weights_by_source_gid_dict[this_source_gid] = \ initial_weights_by_syn_id_dict[this_syn_id] if reference_weights_by_source_gid_dict is not None: reference_weights_by_source_gid_dict[this_source_gid] = \ reference_weights_by_syn_id_dict[this_syn_id] count += 1 logger.info( 'destination: %s; gid %i; set initial synaptic weights for %d inputs from source population ' '%s' % (destination, destination_gid, count, source)) syn_count_by_source_gid_dict = dict() for source_gid in syn_ids_by_source_gid_dict: syn_count_by_source_gid_dict[source_gid] = len( syn_ids_by_source_gid_dict[source_gid]) input_rate_maps_by_source_gid_dict = dict() for source in sources: source_gids = list(source_gid_set_dict[source]) for input_features_namespace in this_input_features_namespaces: input_features_iter = read_cell_attribute_selection( input_features_path, source, namespace=input_features_namespace, mask=set(features_attr_names), selection=source_gids) count = 0 for gid, attr_dict in input_features_iter: input_rate_maps_by_source_gid_dict[gid] = attr_dict[ 'Arena Rate Map'].reshape((dim_x, dim_y)) count += 1 logger.info('Read %s feature data for %i cells in population %s' % (input_features_namespace, count, source)) if is_interactive: context.update(locals()) normalized_delta_weights_dict, arena_LS_map = \ synapses.generate_structured_weights(target_map=target_map, initial_weight_dict=initial_weights_by_source_gid_dict, input_rate_map_dict=input_rate_maps_by_source_gid_dict, syn_count_dict=syn_count_by_source_gid_dict, max_delta_weight=max_delta_weight, arena_x=arena_x, arena_y=arena_y, reference_weight_dict=reference_weights_by_source_gid_dict, reference_weights_are_delta=reference_weights_are_delta, reference_weights_namespace=reference_weights_namespace, optimize_method=optimize_method, verbose=verbose, plot=plot) output_syn_ids = np.empty(len(initial_weights_by_syn_id_dict), dtype='uint32') output_weights = np.empty(len(initial_weights_by_syn_id_dict), dtype='float32') i = 0 for source_gid, this_weight in viewitems(normalized_delta_weights_dict): for syn_id in syn_ids_by_source_gid_dict[source_gid]: output_syn_ids[i] = syn_id output_weights[i] = this_weight i += 1 output_weights_dict = { target_gid: { 'syn_id': output_syn_ids, synapse_name: output_weights } } logger.info('destination: %s; gid %i; generated %s for %i synapses' % (destination, target_gid, output_weights_namespace, len(output_weights))) if not dry_run: this_output_weights_namespace = '%s %s' % (output_weights_namespace, arena_id) logger.info('Destination: %s; appending %s ...' % (destination, this_output_weights_namespace)) append_cell_attributes(output_weights_path, destination, output_weights_dict, namespace=this_output_weights_namespace) logger.info('Destination: %s; appended %s' % (destination, this_output_weights_namespace)) output_weights_dict.clear() if output_features_path is not None: this_output_features_namespace = '%s %s' % ( output_features_namespace, arena_id) cell_attr_dict = dst_input_features[destination] cell_attr_dict[target_gid]['Arena State Map'] = np.asarray( arena_LS_map.ravel(), dtype=np.float32) logger.info('Destination: %s; appending %s ...' % (destination, this_output_features_namespace)) append_cell_attributes(output_features_path, destination, cell_attr_dict, namespace=this_output_features_namespace) if is_interactive: context.update(locals())
def main(config_file, config_prefix, input_path, population, template_paths, dataset_prefix, results_path, results_file_id, results_namespace_id, v_init, io_size, chunk_size, value_chunk_size, write_size, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if io_size == -1: io_size = comm.size if results_file_id is None: if rank == 0: result_file_id = uuid.uuid4() results_file_id = comm.bcast(results_file_id, root=0) if results_namespace_id is None: results_namespace_id = 'Cell Clamp Results' comm = MPI.COMM_WORLD np.seterr(all='raise') verbose = True params = dict(locals()) env = Env(**params) configure_hoc_env(env) if rank == 0: io_utils.mkout(env, env.results_file_path) env.comm.barrier() env.cell_selection = {} template_class = load_cell_template(env, population) if input_path is not None: env.data_file_path = input_path env.load_celltypes() synapse_config = env.celltypes[population]['synapses'] weights_namespaces = [] if 'weights' in synapse_config: has_weights = synapse_config['weights'] if has_weights: if 'weights namespace' in synapse_config: weights_namespaces.append(synapse_config['weights namespace']) elif 'weights namespaces' in synapse_config: weights_namespaces.extend(synapse_config['weights namespaces']) else: weights_namespaces.append('Weights') else: has_weights = False start_time = time.time() count = 0 gid_count = 0 attr_dict = {} if input_path is None: cell_path = env.data_file_path connectivity_path = env.connectivity_file_path else: cell_path = input_path connectivity_path = input_path for gid, morph_dict in NeuroH5TreeGen(cell_path, population, io_size=io_size, comm=env.comm, topology=True): local_time = time.time() if gid is not None: color = 0 comm0 = comm.Split(color, 0) logger.info('Rank %i gid: %i' % (rank, gid)) cell_dict = {'morph': morph_dict} synapses_iter = read_cell_attribute_selection(cell_path, population, [gid], 'Synapse Attributes', comm=comm0) _, synapse_dict = next(synapses_iter) cell_dict['synapse'] = synapse_dict if has_weights: cell_weights_iters = [ read_cell_attribute_selection(cell_path, population, [gid], weights_namespace, comm=comm0) for weights_namespace in weights_namespaces ] weight_dict = dict( zip_longest(weights_namespaces, cell_weights_iters)) cell_dict['weight'] = weight_dict (graph, a) = read_graph_selection(file_name=connectivity_path, selection=[gid], namespaces=['Synapses', 'Connections'], comm=comm0) cell_dict['connectivity'] = (graph, a) gid_count += 1 attr_dict[gid] = {} attr_dict[gid].update( cell_clamp.measure_passive(gid, population, v_init, env, cell_dict=cell_dict)) attr_dict[gid].update( cell_clamp.measure_ap(gid, population, v_init, env, cell_dict=cell_dict)) attr_dict[gid].update( cell_clamp.measure_ap_rate(gid, population, v_init, env, cell_dict=cell_dict)) attr_dict[gid].update( cell_clamp.measure_fi(gid, population, v_init, env, cell_dict=cell_dict)) else: color = 1 comm0 = comm.Split(color, 0) logger.info('Rank %i gid is None' % (rank)) comm0.Free() count += 1 if (results_path is not None) and (count % write_size == 0): append_cell_attributes(env.results_file_path, population, attr_dict, namespace=env.results_namespace_id, comm=env.comm, io_size=env.io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) attr_dict = {} env.comm.barrier() if results_path is not None: append_cell_attributes(env.results_file_path, population, attr_dict, namespace=env.results_namespace_id, comm=env.comm, io_size=env.io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size) global_count = env.comm.gather(gid_count, root=0) MPI.Finalize()
def main(config, template_path, output_path, forest_path, populations, distance_bin_size, io_size, chunk_size, value_chunk_size, cache_size, verbose): """ :param config: :param template_path: :param forest_path: :param populations: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) configure_hoc_env(env) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) if output_path is None: output_path = forest_path if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() layers = env.layers layer_idx_dict = { layers[layer_name]: layer_name for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus'] } (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() for population in populations: logger.info('Rank %i population: %s' % (rank, population)) count = 0 (population_start, _) = pop_ranges[population] template_class = load_cell_template(env, population, bcast_template=True) measures_dict = {} for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=True): if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) cell = cells.make_neurotree_cell(template_class, neurotree_dict=morph_dict, gid=gid) secnodes_dict = morph_dict['section_topology']['nodes'] apicalidx = set(cell.apicalidx) basalidx = set(cell.basalidx) dendrite_area_dict = {k: 0.0 for k in layer_idx_dict} dendrite_length_dict = {k: 0.0 for k in layer_idx_dict} dendrite_distances = [] dendrite_diams = [] for (i, sec) in enumerate(cell.sections): if (i in apicalidx) or (i in basalidx): secnodes = secnodes_dict[i] for seg in sec.allseg(): L = seg.sec.L nseg = seg.sec.nseg seg_l = L / nseg seg_area = h.area(seg.x) seg_diam = seg.diam seg_distance = get_distance_to_node( cell, list(cell.soma)[0], seg.sec, seg.x) dendrite_diams.append(seg_diam) dendrite_distances.append(seg_distance) layer = synapses.get_node_attribute( 'layer', morph_dict, seg.sec, secnodes, seg.x) dendrite_length_dict[layer] += seg_l dendrite_area_dict[layer] += seg_area dendrite_distance_array = np.asarray(dendrite_distances) dendrite_diam_array = np.asarray(dendrite_diams) dendrite_distance_bin_range = int( ((np.max(dendrite_distance_array)) - np.min(dendrite_distance_array)) / distance_bin_size) + 1 dendrite_distance_counts, dendrite_distance_edges = np.histogram( dendrite_distance_array, bins=dendrite_distance_bin_range, density=False) dendrite_diam_sums, _ = np.histogram( dendrite_distance_array, weights=dendrite_diam_array, bins=dendrite_distance_bin_range, density=False) dendrite_mean_diam_hist = np.zeros_like(dendrite_diam_sums) np.divide(dendrite_diam_sums, dendrite_distance_counts, where=dendrite_distance_counts > 0, out=dendrite_mean_diam_hist) dendrite_area_per_layer = np.asarray([ dendrite_area_dict[k] for k in sorted(dendrite_area_dict.keys()) ], dtype=np.float32) dendrite_length_per_layer = np.asarray([ dendrite_length_dict[k] for k in sorted(dendrite_length_dict.keys()) ], dtype=np.float32) measures_dict[gid] = { 'dendrite_distance_hist_edges': np.asarray(dendrite_distance_edges, dtype=np.float32), 'dendrite_distance_counts': np.asarray(dendrite_distance_counts, dtype=np.int32), 'dendrite_mean_diam_hist': np.asarray(dendrite_mean_diam_hist, dtype=np.float32), 'dendrite_area_per_layer': dendrite_area_per_layer, 'dendrite_length_per_layer': dendrite_length_per_layer } del cell count += 1 else: logger.info('Rank %i gid is None' % rank) append_cell_attributes(output_path, population, measures_dict, namespace='Tree Measurements', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size) MPI.Finalize()
def main(arena_id, bin_sample_count, config, config_prefix, dataset_prefix, distances_namespace, distance_bin_extent, input_features_path, input_features_namespaces, populations, spike_input_path, spike_input_namespace, spike_input_attr, output_path, io_size, trajectory_id, write_selection, verbose): utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=comm, config_file=config, config_prefix=config_prefix, dataset_prefix=dataset_prefix, results_path=output_path, spike_input_path=spike_input_path, spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr, arena_id=arena_id, trajectory_id=trajectory_id) if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm) distance_U_dict = {} distance_V_dict = {} range_U_dict = {} range_V_dict = {} selection_dict = defaultdict(set) comm0 = env.comm.Split(2 if rank == 0 else 0, 0) local_random = np.random.RandomState() local_random.seed(1000) if len(populations) == 0: populations = sorted(pop_ranges.keys()) if rank == 0: for population in populations: distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0) soma_distances = {} if input_features_path is not None: num_fields_dict = {} for input_features_namespace in input_features_namespaces: if arena_id is not None: this_features_namespace = '%s %s' % ( input_features_namespace, arena_id) else: this_features_namespace = input_features_namespace input_features_iter = read_cell_attributes( input_features_path, population, namespace=this_features_namespace, mask=set(['Num Fields']), comm=comm0) count = 0 for gid, attr_dict in input_features_iter: num_fields_dict[gid] = attr_dict['Num Fields'] count += 1 logger.info( 'Read feature data from namespace %s for %i cells in population %s' % (this_features_namespace, count, population)) for (gid, v) in distances: num_fields = num_fields_dict.get(gid, 0) if num_fields > 0: soma_distances[gid] = (v['U Distance'][0], v['V Distance'][0]) else: for (gid, v) in distances: soma_distances[gid] = (v['U Distance'][0], v['V Distance'][0]) numitems = len(list(soma_distances.keys())) logger.info('read %s distances (%i elements)' % (population, numitems)) if numitems == 0: continue gid_array = np.asarray([gid for gid in soma_distances]) distance_U_array = np.asarray( [soma_distances[gid][0] for gid in gid_array]) distance_V_array = np.asarray( [soma_distances[gid][1] for gid in gid_array]) U_min = np.min(distance_U_array) U_max = np.max(distance_U_array) V_min = np.min(distance_V_array) V_max = np.max(distance_V_array) range_U_dict[population] = (U_min, U_max) range_V_dict[population] = (V_min, V_max) distance_U = { gid: soma_distances[gid][0] for gid in soma_distances } distance_V = { gid: soma_distances[gid][1] for gid in soma_distances } distance_U_dict[population] = distance_U distance_V_dict[population] = distance_V min_dist = U_min max_dist = U_max distance_bins = np.arange(U_min, U_max, distance_bin_extent) distance_bin_array = np.digitize(distance_U_array, distance_bins) selection_set = set([]) for bin_index in range(len(distance_bins) + 1): bin_gids = gid_array[np.where( distance_bin_array == bin_index)[0]] if len(bin_gids) > 0: selected_bin_gids = local_random.choice( bin_gids, replace=False, size=bin_sample_count) for gid in selected_bin_gids: selection_set.add(int(gid)) selection_dict[population] = selection_set yaml_output_dict = {} for k, v in utils.viewitems(selection_dict): yaml_output_dict[k] = list(sorted(v)) yaml_output_path = '%s/DG_slice.yaml' % output_path with open(yaml_output_path, 'w') as outfile: yaml.dump(yaml_output_dict, outfile) del (yaml_output_dict) env.comm.barrier() write_selection_file_path = None if write_selection: write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName) if write_selection_file_path is not None: if rank == 0: io_utils.mkout(env, write_selection_file_path) env.comm.barrier() selection_dict = env.comm.bcast(dict(selection_dict), root=0) env.cell_selection = selection_dict io_utils.write_cell_selection(env, write_selection_file_path, populations=populations) input_selection = io_utils.write_connection_selection( env, write_selection_file_path, populations=populations) if env.spike_input_ns is not None: io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path, populations=populations) env.comm.barrier() MPI.Finalize()
def config_worker(): """ """ if 'debug' not in context(): context.debug = False if context.debug: if context.comm.rank == 1: print('# of parameters: %i' % len(context.param_names)) print('param_names: ', context.param_names) print('target_val: ', context.target_val) print('target_range: ', context.target_range) print('param_tuples: ', context.param_tuples) sys.stdout.flush() utils.config_logging(context.verbose) context.logger = utils.get_script_logger(os.path.basename(__file__)) # TODO: Do you want this to be identical on all ranks in a subworld? You can use context.comm.bcast if 'results_file_id' not in context(): context.results_file_id = 'DG_optimize_network_subworlds_%s_%s' % \ (context.interface.worker_id, datetime.datetime.today().strftime('%Y%m%d_%H%M')) # 'env' might be in context on controller, but it needs to be re-built when the controller is in a worker subworld try: if context.debug: print( 'debug: config_worker; local_comm.rank: %i/%i; global_comm.rank: %i/%i' % (context.comm.rank, context.comm.size, context.global_comm.rank, context.global_comm.size)) if context.global_comm.rank == 0: print('t_start: %.1f, t_stop: %.1f' % (context.t_start, context.t_stop)) sys.stdout.flush() if context.debug: raise RuntimeError('config_worker: debug') init_network() except Exception as err: context.logger.exception(err) raise err if 't_start' not in context(): context.t_start = 50. else: context.t_start = float(context.t_start) if 't_stop' not in context(): context.t_stop = context.env.tstop else: context.t_stop = float(context.t_stop) time_range = (context.t_start, context.t_stop) context.target_trj_rate_map_dict = {} target_rate_map_path = context.target_rate_map_path target_rate_map_namespace = context.target_rate_map_namespace target_rate_map_arena = context.env.arena_id target_rate_map_trajectory = context.env.trajectory_id for pop_name in context.target_populations: my_cell_index_set = set(context.env.biophys_cells[pop_name].keys()) trj_rate_maps = rate_maps_from_features( context.env, pop_name, cell_index_set=list(my_cell_index_set), input_features_path=target_rate_map_path, input_features_namespace=target_rate_map_namespace, time_range=time_range) if len(trj_rate_maps) > 0: context.target_trj_rate_map_dict[pop_name] = trj_rate_maps # TODO: This is not put in context or used elsewhere target_rate_vector_dict = { gid: trj_rate_maps[gid] for gid in trj_rate_maps } for gid, target_rate_vector in viewitems(target_rate_vector_dict): idxs = np.where( np.isclose(target_rate_vector, 0., atol=1e-4, rtol=1e-4))[0] target_rate_vector[idxs] = 0.
def main(config, template_path, output_path, forest_path, populations, io_size, chunk_size, value_chunk_size, cache_size, verbose): """ :param config: :param template_path: :param forest_path: :param populations: :param io_size: :param chunk_size: :param value_chunk_size: :param cache_size: """ utils.config_logging(verbose) logger = utils.get_script_logger(script_name) comm = MPI.COMM_WORLD rank = comm.rank env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path) h('objref nil, pc, templatePaths') h.load_file("nrngui.hoc") h.load_file("./templates/Value.hoc") h.xopen("./lib.hoc") h.pc = h.ParallelContext() if io_size == -1: io_size = comm.size if rank == 0: logger.info('%i ranks have been allocated' % comm.size) h.templatePaths = h.List() for path in env.templatePaths: h.templatePaths.append(h.Value(1, path)) if output_path is None: output_path = forest_path if rank == 0: if not os.path.isfile(output_path): input_file = h5py.File(forest_path, 'r') output_file = h5py.File(output_path, 'w') input_file.copy('/H5Types', output_file) input_file.close() output_file.close() comm.barrier() (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) start_time = time.time() for population in populations: logger.info('Rank %i population: %s' % (rank, population)) count = 0 (population_start, _) = pop_ranges[population] template_name = env.celltypes[population]['template'] h.find_template(h.pc, h.templatePaths, template_name) template_class = eval('h.%s' % template_name) measures_dict = {} for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, comm=comm, topology=True): if gid is not None: logger.info('Rank %i gid: %i' % (rank, gid)) cell = cells.make_neurotree_cell(template_class, neurotree_dict=morph_dict, gid=gid) secnodes_dict = morph_dict['section_topology']['nodes'] apicalidx = set(cell.apicalidx) basalidx = set(cell.basalidx) dendrite_area_dict = {k + 1: 0.0 for k in range(0, 4)} dendrite_length_dict = {k + 1: 0.0 for k in range(0, 4)} for (i, sec) in enumerate(cell.sections): if (i in apicalidx) or (i in basalidx): secnodes = secnodes_dict[i] prev_layer = None for seg in sec.allseg(): L = seg.sec.L nseg = seg.sec.nseg seg_l = old_div(L, nseg) seg_area = h.area(seg.x) layer = cells.get_node_attribute( 'layer', morph_dict, seg.sec, secnodes, seg.x) layer = layer if layer > 0 else ( prev_layer if prev_layer is not None else 1) prev_layer = layer dendrite_length_dict[layer] += seg_l dendrite_area_dict[layer] += seg_area measures_dict[gid] = { 'dendrite_area': np.asarray([ dendrite_area_dict[k] for k in sorted(dendrite_area_dict.keys()) ], dtype=np.float32), \ 'dendrite_length': np.asarray([ dendrite_length_dict[k] for k in sorted(dendrite_length_dict.keys()) ], dtype=np.float32) } del cell count += 1 else: logger.info('Rank %i gid is None' % rank) append_cell_attributes(output_path, population, measures_dict, namespace='Tree Measurements', comm=comm, io_size=io_size, chunk_size=chunk_size, value_chunk_size=value_chunk_size, cache_size=cache_size) MPI.Finalize()
def main(connectivity_path, output_path, coords_path, distances_namespace, destination, bin_size, cache_size, verbose): """ Measures vertex distribution with respect to septo-temporal distance :param connectivity_path: :param coords_path: :param distances_namespace: :param destination: :param source: """ utils.config_logging(verbose) logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.Get_rank() (population_ranges, _) = read_population_ranges(coords_path) destination_start = population_ranges[destination][0] destination_count = population_ranges[destination][1] if rank == 0: logger.info('reading %s distances...' % destination) destination_soma_distances = bcast_cell_attributes( coords_path, destination, namespace=distances_namespace, comm=comm, root=0) destination_soma_distance_U = {} destination_soma_distance_V = {} for k, v in destination_soma_distances: destination_soma_distance_U[k] = v['U Distance'][0] destination_soma_distance_V[k] = v['V Distance'][0] del (destination_soma_distances) sources = [] for (src, dst) in read_projection_names(connectivity_path): if dst == destination: sources.append(src) source_soma_distances = {} for s in sources: if rank == 0: logger.info('reading %s distances...' % s) source_soma_distances[s] = bcast_cell_attributes( coords_path, s, namespace=distances_namespace, comm=comm, root=0) source_soma_distance_U = {} source_soma_distance_V = {} for s in sources: this_source_soma_distance_U = {} this_source_soma_distance_V = {} for k, v in source_soma_distances[s]: this_source_soma_distance_U[k] = v['U Distance'][0] this_source_soma_distance_V[k] = v['V Distance'][0] source_soma_distance_U[s] = this_source_soma_distance_U source_soma_distance_V[s] = this_source_soma_distance_V del (source_soma_distances) logger.info('reading connections %s -> %s...' % (str(sources), destination)) gg = [ NeuroH5ProjectionGen(connectivity_path, source, destination, cache_size=cache_size, comm=comm) for source in sources ] dist_bins = defaultdict(dict) dist_u_bins = defaultdict(dict) dist_v_bins = defaultdict(dict) for prj_gen_tuple in utils.zip_longest(*gg): destination_gid = prj_gen_tuple[0][0] if not all([ prj_gen_elt[0] == destination_gid for prj_gen_elt in prj_gen_tuple ]): raise Exception( 'destination %s: destination_gid %i not matched across multiple projection generators: %s' % (destination, destination_gid, [prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple])) if destination_gid is not None: logger.info('reading connections of gid %i' % destination_gid) for (source, (this_destination_gid, rest)) in zip(sources, prj_gen_tuple): this_source_soma_distance_U = source_soma_distance_U[source] this_source_soma_distance_V = source_soma_distance_V[source] this_dist_bins = dist_bins[source] this_dist_u_bins = dist_u_bins[source] this_dist_v_bins = dist_v_bins[source] (source_indexes, attr_dict) = rest dst_U = destination_soma_distance_U[destination_gid] dst_V = destination_soma_distance_V[destination_gid] for source_gid in source_indexes: dist_u = dst_U - this_source_soma_distance_U[source_gid] dist_v = dst_V - this_source_soma_distance_V[source_gid] dist = abs(dist_u) + abs(dist_v) update_bins(this_dist_bins, bin_size, dist) update_bins(this_dist_u_bins, bin_size, dist_u) update_bins(this_dist_v_bins, bin_size, dist_v) comm.barrier() logger.info('merging distance dictionaries...') add_bins_op = MPI.Op.Create(add_bins, commute=True) for source in sources: dist_bins[source] = comm.reduce(dist_bins[source], op=add_bins_op, root=0) dist_u_bins[source] = comm.reduce(dist_u_bins[source], op=add_bins_op, root=0) dist_v_bins[source] = comm.reduce(dist_v_bins[source], op=add_bins_op, root=0) comm.barrier() if rank == 0: color = 1 else: color = 0 ## comm0 includes only rank 0 comm0 = comm.Split(color, 0) if rank == 0: if output_path is None: output_path = connectivity_path logger.info('writing output to %s...' % output_path) #f = h5py.File(output_path, 'a', driver='mpio', comm=comm0) #if 'Nodes' in f: # nodes_grp = f['Nodes'] #else: # nodes_grp = f.create_group('Nodes') #grp = nodes_grp.create_group('Connectivity Distance Histogram') #dst_grp = grp.create_group(destination) for source in sources: dist_histoCount, dist_bin_edges = finalize_bins( dist_bins[source], bin_size) dist_u_histoCount, dist_u_bin_edges = finalize_bins( dist_u_bins[source], bin_size) dist_v_histoCount, dist_v_bin_edges = finalize_bins( dist_v_bins[source], bin_size) np.savetxt('%s Distance U Bin Count.dat' % source, dist_u_histoCount) np.savetxt('%s Distance U Bin Edges.dat' % source, dist_u_bin_edges) np.savetxt('%s Distance V Bin Count.dat' % source, dist_v_histoCount) np.savetxt('%s Distance V Bin Edges.dat' % source, dist_v_bin_edges) np.savetxt('%s Distance Bin Count.dat' % source, dist_histoCount) np.savetxt('%s Distance Bin Edges.dat' % source, dist_bin_edges) #f.close() comm.barrier()