def make_rotate3d(rotate): """Creates a rotation matrix based on angles in degrees.""" for i in range(0, 3): if rotate[i] != 0.: a = float(np.deg2rad(rotate[i])) rot = rotate3d([1 if i == j else 0 for j in range(0, 3)], a) return rot
def trial_state_residuals(gid, target_outfld, t_peak_idxs, t_trough_idxs, t_infld_idxs, t_outfld_idxs, state_values, masked_state_values): state_value_arrays = np.row_stack(state_values) masked_state_value_arrays = None if masked_state_values is not None: masked_state_value_arrays = np.row_stack(masked_state_values) residuals_outfld = [] peak_inflds = [] trough_inflds = [] mean_outflds = [] for i in range(state_value_arrays.shape[0]): state_value_array = state_value_arrays[i, :] peak_infld = np.mean(state_value_array[t_peak_idxs]) trough_infld = np.mean(state_value_array[t_trough_idxs]) mean_infld = np.mean(state_value_array[t_infld_idxs]) masked_state_value_array = masked_state_value_arrays[i, :] mean_masked = np.mean(masked_state_value_array) residual_masked = np.mean(masked_state_value_array) - target_outfld mean_outfld = mean_masked if t_outfld_idxs is not None: mean_outfld = np.mean(state_value_array[t_outfld_idxs]) peak_inflds.append(peak_infld) trough_inflds.append(trough_infld) mean_outflds.append(mean_outfld) residuals_outfld.append(residual_masked) logger.info(f'selectivity objective: state values of gid {gid}: ' f'peak/trough/mean in/mean out/masked: {peak_infld:.02f} / {trough_infld:.02f} / {mean_infld:.02f} / {mean_outfld:.02f} / residual masked: {residual_masked:.04f}') state_features = [np.mean(peak_inflds), np.mean(trough_inflds), np.mean(mean_outflds)] return (np.asarray(residuals_outfld), state_features)
def lpt(cx, npart): ''' From the list of (cx, gid) return a npart length list with each partition being a total_cx followed by a list of (cx, gid). ''' cx.sort(key=lambda x: x[0], reverse=True) # initialize a priority queue for fast determination of current # partition with least complexity. The priority queue always has # npart items in it. At this time we do not care which partition will # be associated with which rank so a partition on the heap is just # (totalcx, [list of (cx, gid)] h = [] for i in range(npart): heapq.heappush(h, (0.0, [])) # each cx item goes into the current least complex partition for c in cx: lp = heapq.heappop(h) # least partition lp[1].append(c) heapq.heappush(h, (lp[0] + c[0], lp[1])) parts = [heapq.heappop(h) for i in range(len(h))] return parts
def gid_firing_rate_vectors(spkdict, cell_index_set): rates_dict = defaultdict(list) for i in range(n_trials): spkdict1 = {} for gid in cell_index_set: if gid in spkdict[population]: spkdict1[gid] = spkdict[population][gid][i] else: spkdict1[gid] = np.asarray([], dtype=np.float32) spike_density_dict = spikedata.spike_density_estimate (population, spkdict1, time_bins) for gid in cell_index_set: rate_vector = spike_density_dict[gid]['rate'] rate_vector[np.isclose(rate_vector, 0., atol=1e-3, rtol=1e-3)] = 0. rates_dict[gid].append(rate_vector) logger.info(f'selectivity objective: trial {i} firing rate min/max of gid {gid}: ' f'{np.min(rates_dict[gid]):.02f} / {np.max(rates_dict[gid]):.02f} Hz') return rates_dict
def read_params(input_path): output_file = h5py.File(output_path, 'a') pop_params_dict = {} parameters_group = h5_get_group(output_file, 'Parameters') for population in parameters_group.keys(): this_pop_params_dict = {} pop_group = h5_get_group(parameters_group, population) for id_str in pop_group.keys(): params_data = pop_group[id_str][:] this_id_params_dict = {} for i in range(len(params_data)): this_id_params_dict[params_data[i] ["parameter"]] = params_data[i]["value"] this_pop_params_dict[int(id_str)] = this_id_params_dict pop_params_dict[population] = this_pop_params_dict output_file.close() return pop_params_dict
def trial_snr_residuals(gid, peak_idxs, trough_idxs, infld_idxs, outfld_idxs, rate_vectors, masked_rate_vectors, target_rate_vector): n_trials = len(rate_vectors) residual_inflds = [] trial_inflds = [] trial_outflds = [] target_infld = target_rate_vector[infld_idxs] target_max_infld = np.max(target_infld) target_mean_trough = np.mean(target_rate_vector[trough_idxs]) logger.info(f'selectivity objective: target max infld/mean trough of gid {gid}: ' f'{target_max_infld:.02f} {target_mean_trough:.02f}') for trial_i in range(n_trials): rate_vector = rate_vectors[trial_i] infld_rate_vector = rate_vector[infld_idxs] masked_rate_vector = masked_rate_vectors[trial_i] if outfld_idxs is None: outfld_rate_vector = masked_rate_vectors[trial_i] else: outfld_rate_vector = rate_vector[outfld_idxs] mean_peak = np.mean(rate_vector[peak_idxs]) mean_trough = np.mean(rate_vector[trough_idxs]) min_infld = np.min(infld_rate_vector) max_infld = np.max(infld_rate_vector) mean_infld = np.mean(infld_rate_vector) mean_outfld = np.mean(outfld_rate_vector) residual_infld = np.abs(np.sum(target_infld - infld_rate_vector)) logger.info(f'selectivity objective: max infld/mean infld/mean peak/trough/mean outfld/residual_infld of gid {gid} trial {trial_i}: ' f'{max_infld:.02f} {mean_infld:.02f} {mean_peak:.02f} {mean_trough:.02f} {mean_outfld:.02f} {residual_infld:.04f}') residual_inflds.append(residual_infld) trial_inflds.append(mean_infld) trial_outflds.append(mean_outfld) trial_rate_features = [np.asarray(trial_inflds, dtype=np.float32).reshape((1, n_trials)), np.asarray(trial_outflds, dtype=np.float32).reshape((1, n_trials))] rate_features = [mean_peak, mean_trough, max_infld, min_infld, mean_infld, mean_outfld, ] #rate_constr = [ mean_peak if max_infld > 0. else -1. ] rate_constr = [ mean_peak - mean_trough if max_infld > 0. else -1. ] return (np.asarray(residual_inflds), trial_rate_features, rate_features, rate_constr)
def main(config_file, population, dt, gid, gid_selection_file, arena_id, trajectory_id, generate_weights, t_max, t_min, nprocs_per_worker, n_epochs, n_initial, initial_maxiter, initial_method, optimizer_method, surrogate_method, population_size, num_generations, resample_fraction, mutation_rate, template_paths, dataset_prefix, config_prefix, param_config_name, selectivity_config_name, param_type, recording_profile, results_file, results_path, spike_events_path, spike_events_namespace, spike_events_t, input_features_path, input_features_namespaces, n_trials, trial_regime, problem_regime, target_features_path, target_features_namespace, target_state_variable, target_state_filter, use_coreneuron, cooperative_init, spawn_startup_wait): """ Optimize the input stimulus selectivity of the specified cell in a network clamp configuration. """ init_params = dict(locals()) comm = MPI.COMM_WORLD size = comm.Get_size() rank = comm.Get_rank() results_file_id = None if rank == 0: results_file_id = generate_results_file_id(population, gid) results_file_id = comm.bcast(results_file_id, root=0) comm.barrier() np.seterr(all='raise') verbose = True cache_queries = True config_logging(verbose) cell_index_set = set([]) if gid_selection_file is not None: with open(gid_selection_file, 'r') as f: lines = f.readlines() for line in lines: gid = int(line) cell_index_set.add(gid) elif gid is not None: cell_index_set.add(gid) else: comm.barrier() comm0 = comm.Split(2 if rank == 0 else 1, 0) if rank == 0: env = Env(**init_params, comm=comm0) attr_info_dict = read_cell_attribute_info(env.data_file_path, populations=[population], read_cell_index=True, comm=comm0) cell_index = None attr_name, attr_cell_index = next(iter(attr_info_dict[population]['Trees'])) cell_index_set = set(attr_cell_index) comm.barrier() cell_index_set = comm.bcast(cell_index_set, root=0) comm.barrier() comm0.Free() init_params['cell_index_set'] = cell_index_set del(init_params['gid']) params = dict(locals()) env = Env(**params) if size == 1: configure_hoc_env(env) init(env, population, cell_index_set, arena_id, trajectory_id, n_trials, spike_events_path, spike_events_namespace=spike_events_namespace, spike_train_attr_name=spike_events_t, input_features_path=input_features_path, input_features_namespaces=input_features_namespaces, generate_weights_pops=set(generate_weights), t_min=t_min, t_max=t_max) if (population in env.netclamp_config.optimize_parameters[param_type]): opt_params = env.netclamp_config.optimize_parameters[param_type][population] else: raise RuntimeError(f'optimize_selectivity: population {population} does not have optimization configuration') if target_state_variable is None: target_state_variable = 'v' init_params['target_features_arena'] = arena_id init_params['target_features_trajectory'] = trajectory_id opt_state_baseline = opt_params['Targets']['state'][target_state_variable]['baseline'] init_params['state_baseline'] = opt_state_baseline init_params['state_variable'] = target_state_variable init_params['state_filter'] = target_state_filter init_objfun_name = 'init_selectivity_objfun' best = optimize_run(env, population, param_config_name, selectivity_config_name, init_objfun_name, problem_regime=problem_regime, n_epochs=n_epochs, n_initial=n_initial, initial_maxiter=initial_maxiter, initial_method=initial_method, optimizer_method=optimizer_method, surrogate_method=surrogate_method, population_size=population_size, num_generations=num_generations, resample_fraction=resample_fraction, mutation_rate=mutation_rate, param_type=param_type, init_params=init_params, results_file=results_file, nprocs_per_worker=nprocs_per_worker, cooperative_init=cooperative_init, spawn_startup_wait=spawn_startup_wait, verbose=verbose) opt_param_config = optimization_params(env.netclamp_config.optimize_parameters, [population], param_config_name, param_type) if best is not None: if results_path is not None: run_ts = time.strftime("%Y%m%d_%H%M%S") file_path = f'{results_path}/optimize_selectivity.{run_ts}.yaml' param_names = opt_param_config.param_names param_tuples = opt_param_config.param_tuples if ProblemRegime[problem_regime] == ProblemRegime.every: results_config_dict = {} for gid, prms in viewitems(best): n_res = prms[0][1].shape[0] prms_dict = dict(prms) this_results_config_dict = {} for i in range(n_res): results_param_list = [] for param_pattern, param_tuple in zip(param_names, param_tuples): results_param_list.append((param_tuple.population, param_tuple.source, param_tuple.sec_type, param_tuple.syn_name, param_tuple.param_path, float(prms_dict[param_pattern][i]))) this_results_config_dict[i] = results_param_list results_config_dict[gid] = this_results_config_dict else: prms = best[0] n_res = prms[0][1].shape[0] prms_dict = dict(prms) results_config_dict = {} for i in range(n_res): results_param_list = [] for param_pattern, param_tuple in zip(param_names, param_tuples): results_param_list.append((param_tuple.population, param_tuple.source, param_tuple.sec_type, param_tuple.syn_name, param_tuple.param_path, float(prms_dict[param_pattern][i]))) results_config_dict[i] = results_param_list write_to_yaml(file_path, { population: results_config_dict } ) comm.barrier()
def recsout(env, output_path, t_start=None, clear_data=False, write_cell_location_data=False, write_trial_data=False): """ Writes intracellular state traces to specified NeuroH5 output file. :param env: :param output_path: :param clear_data: :param reduce_data: :return: """ t_rec = env.t_rec equilibration_duration = float( env.stimulus_config['Equilibration Duration']) reduce_data = env.recording_profile.get('reduce', None) n_trials = env.n_trials trial_time_ranges = get_trial_time_ranges(env.t_rec.to_python(), env.n_trials) trial_time_bins = [ t_trial_start for t_trial_start, t_trial_end in trial_time_ranges ] trial_dur = np.asarray([env.tstop + equilibration_duration] * n_trials, dtype=np.float32) for pop_name in sorted(env.celltypes.keys()): local_rec_types = list(env.recs_dict[pop_name].keys()) rec_types = sorted( set(env.comm.allreduce(local_rec_types, op=mpi_op_concat))) for rec_type in rec_types: recs = env.recs_dict[pop_name][rec_type] attr_dict = defaultdict(lambda: {}) for rec in recs: gid = rec['gid'] data_vec = np.array(rec['vec'], copy=clear_data, dtype=np.float32) time_vec = np.array(t_rec, copy=clear_data, dtype=np.float32) if t_start is not None: time_inds = np.where(time_vec >= t_start)[0] time_vec = time_vec[time_inds] data_vec = data_vec[time_inds] trial_bins = np.digitize(time_vec, trial_time_bins) - 1 for trial_i in range(n_trials): trial_inds = np.where(trial_bins == trial_i)[0] time_vec[trial_inds] -= np.sum( trial_dur[:(trial_i)]) + equilibration_duration label = rec['label'] if label in attr_dict[gid]: if reduce_data is None: raise RuntimeError( 'recsout: duplicate recorder labels and no reduce strategy specified' ) elif reduce_data is True: attr_dict[gid][label] += data_vec else: raise RuntimeError( 'recsout: unsupported reduce strategy specified') else: attr_dict[gid][label] = data_vec attr_dict[gid]['t'] = time_vec if write_trial_data: attr_dict[gid]['trial duration'] = trial_dur if write_cell_location_data: distance = rec.get('distance', None) if distance is not None: attr_dict[gid]['distance'] = np.asarray( [distance], dtype=np.float32) section = rec.get('section', None) if section is not None: attr_dict[gid]['section'] = np.asarray([section], dtype=np.int16) loc = rec.get('loc', None) if loc is not None: attr_dict[gid]['loc'] = np.asarray([loc], dtype=np.float32) if clear_data: rec['vec'].resize(0) if env.results_namespace_id is None: namespace_id = "Intracellular %s" % (rec_type) else: namespace_id = "Intracellular %s %s" % ( rec_type, str(env.results_namespace_id)) append_cell_attributes(output_path, pop_name, attr_dict, namespace=namespace_id, comm=env.comm, io_size=env.io_size) if clear_data: env.t_rec.resize(0) env.comm.barrier() if env.comm.Get_rank() == 0: logger.info("*** Output intracellular state results to file %s" % output_path)
def spikeout(env, output_path, t_start=None, clear_data=False): """ Writes spike times to specified NeuroH5 output file. :param env: :param output_path: :param clear_data: :return: """ equilibration_duration = float( env.stimulus_config['Equilibration Duration']) n_trials = env.n_trials t_vec = np.array(env.t_vec, dtype=np.float32) id_vec = np.array(env.id_vec, dtype=np.uint32) trial_time_ranges = get_trial_time_ranges(env.t_rec.to_python(), env.n_trials) trial_time_bins = [ t_trial_start for t_trial_start, t_trial_end in trial_time_ranges ] trial_dur = np.asarray([env.tstop + equilibration_duration] * n_trials, dtype=np.float32) binlst = [] typelst = sorted(env.celltypes.keys()) binvect = np.asarray([env.celltypes[k]['start'] for k in typelst]) sort_idx = np.argsort(binvect, axis=0) pop_names = [typelst[i] for i in sort_idx] bins = binvect[sort_idx][1:] inds = np.digitize(id_vec, bins) if env.results_namespace_id is None: namespace_id = "Spike Events" else: namespace_id = "Spike Events %s" % str(env.results_namespace_id) for i, pop_name in enumerate(pop_names): spkdict = {} sinds = np.where(inds == i) if len(sinds) > 0: ids = id_vec[sinds] ts = t_vec[sinds] for j in range(0, len(ids)): gid = ids[j] t = ts[j] if (t_start is None) or (t >= t_start): if gid in spkdict: spkdict[gid]['t'].append(t) else: spkdict[gid] = {'t': [t]} for gid in spkdict: spiketrain = np.array(spkdict[gid]['t'], dtype=np.float32) if gid in env.spike_onset_delay: spiketrain -= env.spike_onset_delay[gid] trial_bins = np.digitize(spiketrain, trial_time_bins) - 1 trial_spikes = [ np.copy(spiketrain[np.where(trial_bins == trial_i)[0]]) for trial_i in range(n_trials) ] for trial_i, trial_spiketrain in enumerate(trial_spikes): trial_spiketrain = trial_spikes[trial_i] trial_spiketrain -= np.sum( trial_dur[:(trial_i)]) + equilibration_duration spkdict[gid]['t'] = np.concatenate(trial_spikes) spkdict[gid]['Trial Duration'] = trial_dur spkdict[gid]['Trial Index'] = np.asarray(trial_bins, dtype=np.uint8) append_cell_attributes(output_path, pop_name, spkdict, namespace=namespace_id, comm=env.comm, io_size=env.io_size) del (spkdict) if clear_data: env.t_vec.resize(0) env.id_vec.resize(0) env.comm.barrier() if env.comm.Get_rank() == 0: logger.info("*** Output spike results to file %s" % output_path)
def spatial_bin_graph(connectivity_path, coords_path, distances_namespace, destination, sources, extents, bin_size=20.0, cache_size=100, comm=None): """ Obtain reduced graphs of the specified projections by binning nodes according to their spatial position. :param connectivity_path: :param coords_path: :param distances_namespace: :param destination: :param source: """ import networkx as nx if comm is None: comm = MPI.COMM_WORLD rank = comm.Get_rank() (population_ranges, _) = read_population_ranges(coords_path) destination_start = population_ranges[destination][0] destination_count = population_ranges[destination][1] if rank == 0: logger.info('reading %s distances...' % destination) destination_soma_distances = bcast_cell_attributes( coords_path, destination, namespace=distances_namespace, comm=comm, root=0) ((x_min, x_max), (y_min, y_max)) = extents u_bins = np.arange(x_min, x_max, bin_size) v_bins = np.arange(y_min, y_max, bin_size) dest_u_bins = {} dest_v_bins = {} destination_soma_distance_U = {} destination_soma_distance_V = {} for k, v in destination_soma_distances: dist_u = v['U Distance'][0] dist_v = v['V Distance'][0] dest_u_bins[k] = np.searchsorted(u_bins, dist_u, side='left') dest_v_bins[k] = np.searchsorted(v_bins, dist_v, side='left') destination_soma_distance_U[k] = dist_u destination_soma_distance_V[k] = dist_v del (destination_soma_distances) if (sources == ()) or (sources == []) or (sources is None): sources = [] for (src, dst) in read_projection_names(connectivity_path): if dst == destination: sources.append(src) source_soma_distances = {} for s in sources: if rank == 0: logger.info('reading %s distances...' % s) source_soma_distances[s] = bcast_cell_attributes( coords_path, s, namespace=distances_namespace, comm=comm, root=0) source_u_bins = {} source_v_bins = {} source_soma_distance_U = {} source_soma_distance_V = {} for s in sources: this_source_soma_distance_U = {} this_source_soma_distance_V = {} this_source_u_bins = {} this_source_v_bins = {} for k, v in source_soma_distances[s]: dist_u = v['U Distance'][0] dist_v = v['V Distance'][0] this_source_u_bins[k] = np.searchsorted(u_bins, dist_u, side='left') this_source_v_bins[k] = np.searchsorted(v_bins, dist_v, side='left') this_source_soma_distance_U[k] = dist_u this_source_soma_distance_V[k] = dist_v source_soma_distance_U[s] = this_source_soma_distance_U source_soma_distance_V[s] = this_source_soma_distance_V source_u_bins[s] = this_source_u_bins source_v_bins[s] = this_source_v_bins del (source_soma_distances) if rank == 0: logger.info('reading connections %s -> %s...' % (str(sources), destination)) gg = [ NeuroH5ProjectionGen(connectivity_path, source, destination, cache_size=cache_size, comm=comm) for source in sources ] dist_bins = defaultdict(dict) dist_u_bins = defaultdict(dict) dist_v_bins = defaultdict(dict) local_u_bin_graph = defaultdict(dict) local_v_bin_graph = defaultdict(dict) for prj_gen_tuple in zip_longest(*gg): destination_gid = prj_gen_tuple[0][0] if not all([ prj_gen_elt[0] == destination_gid for prj_gen_elt in prj_gen_tuple ]): raise RuntimeError( 'destination %s: destination_gid %i not matched across multiple projection generators: ' '%s' % (destination, destination_gid, [prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple])) if destination_gid is not None: dest_u_bin = dest_u_bins[destination_gid] dest_v_bin = dest_v_bins[destination_gid] for (source, (this_destination_gid, rest)) in zip_longest(sources, prj_gen_tuple): this_source_u_bins = source_u_bins[source] this_source_v_bins = source_v_bins[source] (source_indexes, attr_dict) = rest source_u_bin_dict = defaultdict(int) source_v_bin_dict = defaultdict(int) for source_gid in source_indexes: source_u_bin = this_source_u_bins[source_gid] source_v_bin = this_source_v_bins[source_gid] source_u_bin_dict[source_u_bin] += 1 source_v_bin_dict[source_v_bin] += 1 local_u_bin_graph[dest_u_bin][source] = source_u_bin_dict local_v_bin_graph[dest_v_bin][source] = source_v_bin_dict local_u_bin_graphs = comm.gather(dict(local_u_bin_graph), root=0) local_v_bin_graphs = comm.gather(dict(local_v_bin_graph), root=0) u_bin_graph = None v_bin_graph = None nu = None nv = None if rank == 0: u_bin_edges = {destination: dict(ChainMap(*local_u_bin_graphs))} v_bin_edges = {destination: dict(ChainMap(*local_v_bin_graphs))} nu = len(u_bins) u_bin_graph = nx.Graph() for pop in [destination] + list(sources): for i in range(nu): u_bin_graph.add_node((pop, i)) for i, ss in viewitems(u_bin_edges[destination]): for source, ids in viewitems(ss): u_bin_graph.add_weighted_edges_from([ ((source, j), (destination, i), count) for j, count in viewitems(ids) ]) nv = len(v_bins) v_bin_graph = nx.Graph() for pop in [destination] + list(sources): for i in range(nv): v_bin_graph.add_node((pop, i)) for i, ss in viewitems(v_bin_edges[destination]): for source, ids in viewitems(ss): v_bin_graph.add_weighted_edges_from([ ((source, j), (destination, i), count) for j, count in viewitems(ids) ]) label = '%s to %s' % (str(sources), destination) return { 'label': label, 'bin size': bin_size, 'destination': destination, 'sources': sources, 'U graph': u_bin_graph, 'V graph': v_bin_graph }
def generate_synaptic_connections(rank, gid, ranstream_syn, ranstream_con, cluster_seed, destination_gid, synapse_dict, population_dict, projection_synapse_dict, projection_prob_dict, connection_dict, random_choice=random_choice_w_replacement): """ Given a set of synapses for a particular gid, projection configuration, projection and connection probability dictionaries, generates a set of possible connections for each synapse. The procedure first assigns each synapse to a projection, using the given proportions of each synapse type, and then chooses source gids for each synapse using the given projection probability dictionary. :param ranstream_syn: random stream for the synapse partitioning step :param ranstream_con: random stream for the choosing source gids step :param destination_gid: destination gid :param synapse_dict: synapse configurations, a dictionary with fields: 1) syn_ids (synapse ids) 2) syn_types (excitatory, inhibitory, etc)., 3) swc_types (SWC types(s) of synapse location in the neuronal morphological structure 3) syn_layers (synapse layer placement) :param population_dict: mapping of population names to population indices :param projection_synapse_dict: mapping of projection names to a tuple of the form: <syn_layer, swc_type, syn_type, syn_proportion> :param projection_prob_dict: mapping of presynaptic population names to sets of source probabilities and source gids :param connection_dict: output connection dictionary :param random_choice: random choice procedure (default uses np.ranstream.multinomial) """ num_projections = len(projection_synapse_dict) prj_pop_index = { population: i for (i, population) in enumerate(projection_synapse_dict) } synapse_prj_counts = np.zeros((num_projections, )) synapse_prj_partition = defaultdict(lambda: defaultdict(list)) maxit = 10 it = 0 ## assign each synapse to a projection while (np.count_nonzero(synapse_prj_counts) < num_projections) and (it < maxit): log_flag = it > 1 if log_flag: logger.info("generate_synaptic_connections: gid %i: iteration %i" % (gid, it)) synapse_prj_counts.fill(0) synapse_prj_partition.clear() for (syn_id, syn_type, swc_type, syn_layer) in zip( synapse_dict['syn_ids'], synapse_dict['syn_types'], synapse_dict['swc_types'], synapse_dict['syn_layers']): projection = choose_synapse_projection(ranstream_syn, syn_layer, swc_type, syn_type, \ population_dict, projection_synapse_dict, log=log_flag) if log_flag: logger.info('generate_synaptic_connections: gid %i: ' \ 'syn_id = %i syn_type = %i swc_type = %i syn_layer = %i projection = %s' % \ (gid, syn_id, syn_type, swc_type, syn_layer, projection)) assert (projection is not None) synapse_prj_counts[prj_pop_index[projection]] += 1 synapse_prj_partition[projection][syn_layer].append(syn_id) it += 1 empty_projections = [] for projection in projection_synapse_dict: logger.debug('Rank %i: gid %i: projection %s has %i synapses' % (rank, destination_gid, projection, len(synapse_prj_partition[projection]))) if not (len(synapse_prj_partition[projection]) > 0): empty_projections.append(projection) if len(empty_projections) > 0: logger.warning('Rank %i: gid %i: projections %s have an empty synapse list; ' \ 'swc types are %s layers are %s' % \ (rank, destination_gid, str(empty_projections), str(set(synapse_dict['swc_types'].flat)), \ str(set(synapse_dict['syn_layers'].flat)))) assert (len(empty_projections) == 0) ## Choose source connections based on distance-weighted probability count = 0 for projection, prj_layer_dict in viewitems(synapse_prj_partition): (syn_config_type, syn_config_layers, syn_config_sections, syn_config_proportions, syn_config_contacts) = \ projection_synapse_dict[projection] gid_dict = connection_dict[projection] prj_source_vertices = [] prj_syn_ids = [] prj_distances = [] for prj_layer, syn_ids in viewitems(prj_layer_dict): source_probs, source_gids, distances_u, distances_v = \ projection_prob_dict[projection][prj_layer] distance_dict = {source_gid: distance_u + distance_v \ for (source_gid, distance_u, distance_v) in \ zip(source_gids, distances_u, distances_v)} if len(source_gids) > 0: n_syn_groups = int( math.ceil( float(len(syn_ids)) / float(syn_config_contacts))) source_gid_counts = random_choice(ranstream_con, n_syn_groups, source_probs) total_count = 0 if syn_config_contacts > 1: ncontacts = int(math.ceil(syn_config_contacts)) for i in range(0, len(source_gid_counts)): if source_gid_counts[i] > 0: source_gid_counts[i] *= ncontacts if len(source_gid_counts) == 0: logger.warning('Rank %i: source vertices list is empty for gid: %i projection: %s layer: %s ' \ 'source probs: %s distances_u: %s distances_v: %s' % \ (rank, destination_gid, projection, str(layer), \ str(source_probs), str(distances_u), str(distances_v))) uv_distance_sums = np.add(distances_u, distances_v, dtype=np.float32) source_vertices = np.asarray(random_clustered_shuffle(len(source_gids), \ source_gid_counts, \ center_ids=source_gids, \ cluster_std=2.0, \ random_seed=cluster_seed), \ dtype=np.uint32)[0:len(syn_ids)] assert (len(source_vertices) == len(syn_ids)) distances = np.asarray([distance_dict[gid] for gid in source_vertices], \ dtype=np.float32).reshape(-1, ) prj_source_vertices.append(source_vertices) prj_syn_ids.append(syn_ids) prj_distances.append(distances) gid_dict[destination_gid] = (np.asarray([], dtype=np.uint32), { 'Synapses': { 'syn_id': np.asarray([], dtype=np.uint32) }, 'Connections': { 'distance': np.asarray([], dtype=np.float32) } }) cluster_seed += 1 if len(prj_source_vertices) > 0: prj_source_vertices_array = np.concatenate(prj_source_vertices) else: prj_source_vertices_array = np.asarray([], dtype=np.uint32) del (prj_source_vertices) if len(prj_syn_ids) > 0: prj_syn_ids_array = np.concatenate(prj_syn_ids) else: prj_syn_ids_array = np.asarray([], dtype=np.uint32) del (prj_syn_ids) if len(prj_distances) > 0: prj_distances_array = np.concatenate(prj_distances) else: prj_distances_array = np.asarray([], dtype=np.float32) del (prj_distances) if len(prj_source_vertices_array) == 0: logger.warning( 'Rank %i: source gid list is empty for gid: %i projection: %s' % (rank, destination_gid, projection)) count += len(prj_source_vertices_array) gid_dict[destination_gid] = (prj_source_vertices_array, {'Synapses': {'syn_id': np.asarray(prj_syn_ids_array, \ dtype=np.uint32)}, 'Connections': {'distance': prj_distances_array} }) return count
def statistics(parts): npart = len(parts) total_cx = 0 max_part_cx = 0 ncx = 0 max_cx = 0 for part in parts: ncx += len(part[1]) total_cx += part[0] if part[0] > max_part_cx: max_part_cx = part[0] for cx in part[1]: if cx[0] > max_cx: max_cx = cx[0] avg_part_cx = total_cx / npart loadbal = 1.0 if max_part_cx > 0.: loadbal = avg_part_cx / max_part_cx logger.info( "*** loadbal=%g total_cx=%g npart=%d ncx=%d max_part_cx=%g max_cx=%g" % (loadbal, total_cx, npart, ncx, max_part_cx, max_cx)) if __name__ == '__main__': for cx in ([(i, i) for i in range(10)], []): logger.info('%i complexity items %s' % (len(cx), cx)) pinfo = lpt(cx, 3) logger.info('%i lpt partitions %s' % (len(pinfo), str(pinfo))) statistics(pinfo)
def icp_transform(comm, env, soma_coords, projection_ls, population_extents, rotate=None, populations=None, icp_iter=1000, opt_iter=100): """ Uses the iterative closest point (ICP) algorithm of the PCL library to transform soma coordinates onto a surface for a particular L value. http://pointclouds.org/documentation/tutorials/iterative_closest_point.php#iterative-closest-point """ import dlib, pcl rank = comm.rank size = comm.size if populations is None: populations = list(soma_coords.keys()) srf_resample = 25 layer_extents = env.geometry['Parametric Surface']['Layer Extents'] (extent_u, extent_v, extent_l) = get_total_extents(layer_extents) min_u, max_u = extent_u min_v, max_v = extent_v min_l, max_l = extent_l ## This parameter is used to expand the range of L and avoid ## situations where the endpoints of L end up outside of the range ## of the distance interpolant safety = 0.01 extent_u = (min_u - safety, max_u + safety) extent_v = (min_v - safety, max_v + safety) projection_ptclouds = [] for obs_l in projection_ls: srf = make_surface(extent_u, extent_v, obs_l, rotate=rotate) U, V = srf._resample_uv(srf_resample, srf_resample) meshpts = srf.ev(U, V) projection_ptcloud = pcl.PointCloud() projection_ptcloud.from_array(meshpts) projection_ptclouds.append(projection_ptcloud) soma_coords_dict = {} for pop in populations: coords_dict = soma_coords[pop] if rank == 0: logger.info('Computing point transformation for population %s...' % pop) count = 0 xyz_coords = [] gids = [] for gid, coords in viewitems(coords_dict): if gid % size == rank: soma_u, soma_v, soma_l = coords xyz_coords.append( DG_volume(soma_u, soma_v, soma_l, rotate=rotate)) gids.append(gid) xyz_pts = np.vstack(xyz_coords) cloud_in = pcl.PointCloud() cloud_in.from_array(xyz_pts) icp = cloud_in.make_IterativeClosestPoint() all_est_xyz_coords = [] all_est_uvl_coords = [] all_interp_err = [] for (k, cloud_prj) in enumerate(projection_ls): k_est_xyz_coords = np.zeros((len(gids), 3)) k_est_uvl_coords = np.zeros((len(gids), 3)) interp_err = np.zeros((len(gids), )) converged, transf, estimate, fitness = icp.icp(cloud_in, cloud_prj, max_iter=icp_iter) logger.info('Transformation of population %s has converged: ' % (pop) + str(converged) + ' score: %f' % (fitness)) for i, gid in zip(list(range(0, estimate.size)), gids): est_xyz_coords = estimate[i] k_est_xyz_coords[i, :] = est_xyz_coords f_uvl_distance = make_uvl_distance(est_xyz_coords, rotate=rotate) uvl_coords, err = dlib.find_min_global(f_uvl_distance, limits[0], limits[1], opt_iter) k_est_uvl_coords[i, :] = uvl_coords interp_err[i, ] = err if rank == 0: logger.info( 'gid %i: u: %f v: %f l: %f' % (gid, uvl_coords[0], uvl_coords[1], uvl_coords[2])) all_est_xyz_coords.append(k_est_xyz_coords) all_est_uvl_coords.append(k_est_uvl_coords) all_interp_err.append(interp_err) coords_dict = {} for (i, gid) in enumerate(gids): coords_dict[gid] = { 'X Coordinate': np.asarray([col[i, 0] for col in all_est_xyz_coords], dtype='float32'), 'Y Coordinate': np.asarray([col[i, 1] for col in all_est_xyz_coords], dtype='float32'), 'Z Coordinate': np.asarray([col[i, 2] for col in all_est_xyz_coords], dtype='float32'), 'U Coordinate': np.asarray([col[i, 0] for col in all_est_uvl_coords], dtype='float32'), 'V Coordinate': np.asarray([col[i, 1] for col in all_est_uvl_coords], dtype='float32'), 'L Coordinate': np.asarray([col[i, 2] for col in all_est_uvl_coords], dtype='float32'), 'Interpolation Error': np.asarray([err[i] for err in all_interp_err], dtype='float32') } soma_coords_dict[pop] = coords_dict return soma_coords_dict