def lfpout(env, output_path):
"""
Writes local field potential voltage traces to specified HDF5 output file.
:param env:
:param output_path:
:param clear_data:
:return:
"""
for lfp in list(env.lfp.values()):
if env.results_namespace_id is None:
namespace_id = "Local Field Potential %s" % str(lfp.label)
else:
namespace_id = "Local Field Potential %s %s" % (str(
lfp.label), str(env.results_namespace_id))
import h5py
output = h5py.File(output_path, 'a')
grp = output.create_group(namespace_id)
grp['t'] = np.asarray(lfp.t, dtype=np.float32)
grp['v'] = np.asarray(lfp.meanlfp, dtype=np.float32)
output.close()
if env.comm.Get_rank() == 0:
logger.info("*** Output LFP results to file %s" % output_path)
def load_celltypes(self):
"""
:return:
"""
rank = self.comm.Get_rank()
size = self.comm.Get_size()
celltypes = self.celltypes
typenames = sorted(celltypes.keys())
if rank == 0:
self.logger.info('env.data_file_path = %s' %
str(self.data_file_path))
(population_ranges,
_) = read_population_ranges(self.data_file_path, self.comm)
if rank == 0:
self.logger.info('population_ranges = %s' % str(population_ranges))
for k in typenames:
population_range = population_ranges.get(k, None)
if population_range is not None:
celltypes[k]['start'] = population_ranges[k][0]
celltypes[k]['num'] = population_ranges[k][1]
if 'mechanism file' in celltypes[k]:
celltypes[k]['mech_file_path'] = '%s/%s' % (
self.config_prefix, celltypes[k]['mechanism file'])
mech_dict = read_from_yaml(celltypes[k]['mech_file_path'])
celltypes[k]['mech_dict'] = mech_dict
if 'synapses' in celltypes[k]:
synapses_dict = celltypes[k]['synapses']
if 'weights' in synapses_dict:
weights_config = synapses_dict['weights']
if isinstance(weights_config, list):
weights_dicts = weights_config
else:
weights_dicts = [weights_config]
for weights_dict in weights_dicts:
if 'expr' in weights_dict:
expr = weights_dict['expr']
parameter = weights_dict['parameter']
const = weights_dict.get('const', {})
clos = ExprClosure(parameter, expr, const)
weights_dict['closure'] = clos
synapses_dict['weights'] = weights_dicts
population_names = read_population_names(self.data_file_path,
self.comm)
if rank == 0:
self.logger.info('population_names = %s' % str(population_names))
self.cell_attribute_info = read_cell_attribute_info(
self.data_file_path, population_names, comm=self.comm)
if rank == 0:
self.logger.info('attribute info: %s' %
str(self.cell_attribute_info))
def is_same_cell(self, cell1, cell2):
"""
:param cell1: :class:'BiophysCell' or :class:'h.hocObject'
:param cell2: :class:'BiophysCell' or :class:'h.hocObject'
:return: bool
"""
if cell1 == cell2:
return True
elif hasattr(cell1, 'tree'):
return cell1.tree.root.sec.cell() == cell2
else:
raise RuntimeError(
'QuickSim: problem comparing cell objects: %s and %s' %
(str(cell1), str(cell2)))
def get_prob(self, destination_gid, source, source_layers):
"""
Given the soma coordinates of a destination neuron and a
population source, return an array of connection probabilities
and an array of corresponding source gids.
:param destination_gid: int
:param source: string
:return: array of float, array of int
"""
prob_dict = {}
for layer in source_layers:
destination_u, destination_v, source_u, source_v, distance_u, distance_v, source_gid = \
self.filter_by_distance(destination_gid, source, layer)
if layer in self.p_dist[source]:
layer_key = layer
elif 'default' in self.p_dist[source]:
layer_key = 'default'
else:
raise RuntimeError('connection_generator.get_prob: missing configuration for layer %s' % \
str(layer))
p = self.p_dist[source][layer_key](distance_u, distance_v)
psum = np.sum(p)
assert ((p >= 0.).all() and (p <= 1.).all())
if psum > 0.:
pn = p / p.sum()
else:
pn = p
prob_dict[layer] = (pn.ravel(), source_gid.ravel(),
distance_u.ravel(), distance_v.ravel())
return prob_dict
def plot(self, axes=None, show=True):
"""
"""
import matplotlib.pyplot as plt
from dentate.plot import clean_axes
if len(self.recs) == 0:
return
if axes is None:
fig, axes = plt.subplots()
else:
fig = axes.get_figure()
for name, rec_dict in viewitems(self.recs):
description = str(rec_dict['description'])
axes.plot(self.tvec, rec_dict['vec'],
label='%s: %s(%.2f) %s' % (name, rec_dict['node'].name, rec_dict['loc'], description))
axes.set_xlabel('Time (ms)')
axes.set_ylabel('%s (%s)' % (rec_dict['ylabel'], rec_dict['units']))
axes.legend(loc='best', frameon=False, framealpha=0.5)
title = None
if 'title' in self.parameters:
title = self.parameters['title']
if 'description' in self.parameters:
if title is not None:
title = title + '; ' + self.parameters['description']
else:
title = self.parameters['description']
if title is not None:
axes.set_title(title)
clean_axes(axes)
if show:
fig.tight_layout()
fig.show()
else:
return axes
def choose_synapse_projection(ranstream_syn,
syn_layer,
swc_type,
syn_type,
population_dict,
projection_synapse_dict,
log=False):
"""
Given a synapse projection, SWC synapse location, and synapse type,
chooses a projection from the given projection dictionary based on
1) whether the projection properties match the given synapse
properties and 2) random choice between all the projections that
satisfy the given criteria.
:param ranstream_syn: random state object
:param syn_layer: synapse layer
:param swc_type: SWC location for synapse (soma, axon, apical, basal)
:param syn_type: synapse type (excitatory, inhibitory, neuromodulatory)
:param population_dict: mapping of population names to population indices
:param projection_synapse_dict: mapping of projection names to a tuple of the form: <type, layers, swc sections, proportions>
"""
ivd = {v: k for k, v in viewitems(population_dict)}
projection_lst = []
projection_prob_lst = []
for k, (syn_config_type, syn_config_layers, syn_config_sections,
syn_config_proportions,
syn_config_contacts) in viewitems(projection_synapse_dict):
if (syn_type == syn_config_type) and (swc_type in syn_config_sections):
ord_indices = list_find_all(lambda x: x == swc_type,
syn_config_sections)
for ord_index in ord_indices:
if syn_layer == syn_config_layers[ord_index]:
projection_lst.append(population_dict[k])
projection_prob_lst.append(
syn_config_proportions[ord_index])
if len(projection_lst) > 1:
candidate_projections = np.asarray(projection_lst)
candidate_probs = np.asarray(projection_prob_lst)
if log:
logger.info("candidate_projections: %s candidate_probs: %s" % \
(str(candidate_projections), str(candidate_probs)))
projection = ranstream_syn.choice(candidate_projections,
1,
p=candidate_probs)[0]
elif len(projection_lst) > 0:
projection = projection_lst[0]
else:
projection = None
if projection is None:
logger.error('Projection is none for syn_type = %s syn_layer = %s swc_type = %s' % \
(str(syn_type), str(syn_layer), str(swc_type)))
logger.error(str(projection_synapse_dict))
if projection is not None:
return ivd[projection]
else:
return None
def parse_syn_mechparams(self, mechparams_dict):
res = {}
for mech_name, mech_params in viewitems(mechparams_dict):
mech_params1 = {}
for k, v in viewitems(mech_params):
if isinstance(v, dict):
if 'expr' in v:
mech_params1[k] = ExprClosure([v['parameter']],
v['expr'],
v.get('const',
None), ['x'])
else:
raise RuntimeError(
'parse_syn_mechparams: unknown parameter type %s' %
str(v))
else:
mech_params1[k] = v
res[mech_name] = mech_params1
return res
def get_h5py_group(file, hierarchy, create=False):
"""
:param file: :class: in ['h5py.File', 'h5py.Group']
:param hierarchy: list of str
:param create: bool
:return: :class:'h5py.Group'
"""
target = file
for key in hierarchy:
if key is not None:
key = str(key)
if key not in target:
if create:
target = target.create_group(key)
else:
raise KeyError(
'get_h5py_group: target: %s does not contain key: %s; valid keys: %s'
% (target, key, list(target.keys())))
else:
target = target[key]
return target
def write_input_cell_selection(env,
input_sources,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Writes out predefined spike trains when only a subset of the network is instantiated.
:param env: an instance of the `dentate.Env` class
:param input_sources: a dictionary of the form { pop_name, gid_sources }
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
dataset_path = env.dataset_path
input_file_path = env.data_file_path
if populations is None:
pop_names = sorted(env.celltypes.keys())
else:
pop_names = populations
for pop_name, gid_range in sorted(viewitems(input_sources)):
if pop_name not in pop_names:
continue
spikes_output_dict = {}
if (env.cell_selection is not None) and (pop_name
in env.cell_selection):
local_gid_range = gid_range.difference(
set(env.cell_selection[pop_name]))
else:
local_gid_range = gid_range
gid_ranges = env.comm.allgather(local_gid_range)
this_gid_range = set([])
for gid_range in gid_ranges:
for gid in gid_range:
if gid % nhosts == rank:
this_gid_range.add(gid)
has_spike_train = False
spike_input_source_loc = []
if (env.spike_input_attribute_info is not None) and (env.spike_input_ns
is not None):
if (pop_name in env.spike_input_attribute_info) and \
(env.spike_input_ns in env.spike_input_attribute_info[pop_name]):
has_spike_train = True
spike_input_source_loc.append(
(env.spike_input_path, env.spike_input_ns))
if (env.cell_attribute_info is not None) and (env.spike_input_ns
is not None):
if (pop_name in env.cell_attribute_info) and \
(env.spike_input_ns in env.cell_attribute_info[pop_name]):
has_spike_train = True
spike_input_source_loc.append(
(input_file_path, env.spike_input_ns))
if rank == 0:
logger.info(
'*** Reading spike trains for population %s: %d cells: has_spike_train = %s'
% (pop_name, len(this_gid_range), str(has_spike_train)))
if has_spike_train:
vecstim_attr_set = set(['t'])
if env.spike_input_attr is not None:
vecstim_attr_set.add(env.spike_input_attr)
if 'spike train' in env.celltypes[pop_name]:
vecstim_attr_set.add(
env.celltypes[pop_name]['spike train']['attribute'])
cell_spikes_iters = [ read_cell_attribute_selection(input_path, pop_name, \
list(this_gid_range), \
namespace=input_ns, \
mask=vecstim_attr_set, \
comm=env.comm) for (input_path, input_ns) in spike_input_source_loc ]
for cell_spikes_iter in cell_spikes_iters:
spikes_output_dict.update(dict(list(cell_spikes_iter)))
if rank == 0:
logger.info('*** Writing spike trains for population %s: %s' %
(pop_name, str(spikes_output_dict)))
write_cell_attributes(write_selection_file_path, pop_name, spikes_output_dict, \
namespace=env.spike_input_ns, **write_kwds)
def write_connection_selection(env,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Loads NeuroH5 connectivity file, and writes the corresponding
synapse and network connection mechanisms for the selected postsynaptic cells.
:param env: an instance of the `dentate.Env` class
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
connectivity_file_path = env.connectivity_file_path
forest_file_path = env.forest_file_path
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
syn_attrs = env.synapse_attributes
if populations is None:
pop_names = sorted(env.cell_selection.keys())
else:
pop_names = populations
input_sources = {pop_name: set([]) for pop_name in env.celltypes}
for (postsyn_name, presyn_names) in sorted(viewitems(env.projection_dict)):
if rank == 0:
logger.info('*** Writing connection selection of population %s' %
(postsyn_name))
if postsyn_name not in pop_names:
continue
gid_range = [
gid for gid in env.cell_selection[postsyn_name]
if gid % nhosts == rank
]
synapse_config = env.celltypes[postsyn_name]['synapses']
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
else:
has_weights = False
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(
synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(
synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
if rank == 0:
logger.info('*** Reading synaptic attributes of population %s' %
(postsyn_name))
syn_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace='Synapse Attributes',
comm=env.comm)
syn_attributes_output_dict = dict(list(syn_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
syn_attributes_output_dict,
namespace='Synapse Attributes',
**write_kwds)
del syn_attributes_output_dict
del syn_attributes_iter
if has_weights:
for weights_namespace in sorted(weights_namespaces):
weight_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace=weights_namespace,
comm=env.comm)
weight_attributes_output_dict = dict(
list(weight_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
weight_attributes_output_dict,
namespace=weights_namespace,
**write_kwds)
del weight_attributes_output_dict
del weight_attributes_iter
logger.info(
'*** Rank %i: reading connectivity selection from file %s for postsynaptic population: %s: selection: %s'
% (rank, connectivity_file_path, postsyn_name, str(gid_range)))
(graph, attr_info) = read_graph_selection(connectivity_file_path, selection=gid_range, \
projections=[ (presyn_name, postsyn_name) for presyn_name in sorted(presyn_names) ], \
comm=env.comm, namespaces=['Synapses', 'Connections'])
for presyn_name in sorted(presyn_names):
gid_dict = {}
edge_count = 0
node_count = 0
if postsyn_name in graph:
if postsyn_name in attr_info and presyn_name in attr_info[
postsyn_name]:
edge_attr_info = attr_info[postsyn_name][presyn_name]
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
if 'Synapses' in edge_attr_info and \
'syn_id' in edge_attr_info['Synapses'] and \
'Connections' in edge_attr_info and \
'distance' in edge_attr_info['Connections']:
syn_id_attr_index = edge_attr_info['Synapses']['syn_id']
distance_attr_index = edge_attr_info['Connections'][
'distance']
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
edge_iter = compose_iter(lambda edgeset: input_sources[presyn_name].update(edgeset[1][0]), \
graph[postsyn_name][presyn_name])
for (postsyn_gid, edges) in edge_iter:
presyn_gids, edge_attrs = edges
edge_syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
edge_dists = edge_attrs['Connections'][distance_attr_index]
gid_dict[postsyn_gid] = (presyn_gids, {
'Synapses': {
'syn_id': edge_syn_ids
},
'Connections': {
'distance': edge_dists
}
})
edge_count += len(presyn_gids)
node_count += 1
logger.info(
'*** Rank %d: Writing projection %s -> %s selection: %d nodes, %d edges'
% (rank, presyn_name, postsyn_name, node_count, edge_count))
write_graph(write_selection_file_path, \
src_pop_name=presyn_name, dst_pop_name=postsyn_name, \
edges=gid_dict, comm=env.comm, io_size=env.io_size)
env.comm.barrier()
return input_sources
def statistics(parts):
npart = len(parts)
total_cx = 0
max_part_cx = 0
ncx = 0
max_cx = 0
for part in parts:
ncx += len(part[1])
total_cx += part[0]
if part[0] > max_part_cx:
max_part_cx = part[0]
for cx in part[1]:
if cx[0] > max_cx:
max_cx = cx[0]
avg_part_cx = total_cx / npart
loadbal = 1.0
if max_part_cx > 0.:
loadbal = avg_part_cx / max_part_cx
logger.info(
"*** loadbal=%g total_cx=%g npart=%d ncx=%d max_part_cx=%g max_cx=%g" %
(loadbal, total_cx, npart, ncx, max_part_cx, max_cx))
if __name__ == '__main__':
for cx in ([(i, i) for i in range(10)], []):
logger.info('%i complexity items %s' % (len(cx), cx))
pinfo = lpt(cx, 3)
logger.info('%i lpt partitions %s' % (len(pinfo), str(pinfo)))
statistics(pinfo)
def recsout(env,
output_path,
t_start=None,
clear_data=False,
write_cell_location_data=False,
write_trial_data=False):
"""
Writes intracellular state traces to specified NeuroH5 output file.
:param env:
:param output_path:
:param clear_data:
:param reduce_data:
:return:
"""
t_rec = env.t_rec
equilibration_duration = float(
env.stimulus_config['Equilibration Duration'])
reduce_data = env.recording_profile.get('reduce', None)
n_trials = env.n_trials
trial_time_ranges = get_trial_time_ranges(env.t_rec.to_python(),
env.n_trials)
trial_time_bins = [
t_trial_start for t_trial_start, t_trial_end in trial_time_ranges
]
trial_dur = np.asarray([env.tstop + equilibration_duration] * n_trials,
dtype=np.float32)
for pop_name in sorted(env.celltypes.keys()):
local_rec_types = list(env.recs_dict[pop_name].keys())
rec_types = sorted(
set(env.comm.allreduce(local_rec_types, op=mpi_op_concat)))
for rec_type in rec_types:
recs = env.recs_dict[pop_name][rec_type]
attr_dict = defaultdict(lambda: {})
for rec in recs:
gid = rec['gid']
data_vec = np.array(rec['vec'],
copy=clear_data,
dtype=np.float32)
time_vec = np.array(t_rec, copy=clear_data, dtype=np.float32)
if t_start is not None:
time_inds = np.where(time_vec >= t_start)[0]
time_vec = time_vec[time_inds]
data_vec = data_vec[time_inds]
trial_bins = np.digitize(time_vec, trial_time_bins) - 1
for trial_i in range(n_trials):
trial_inds = np.where(trial_bins == trial_i)[0]
time_vec[trial_inds] -= np.sum(
trial_dur[:(trial_i)]) + equilibration_duration
label = rec['label']
if label in attr_dict[gid]:
if reduce_data is None:
raise RuntimeError(
'recsout: duplicate recorder labels and no reduce strategy specified'
)
elif reduce_data is True:
attr_dict[gid][label] += data_vec
else:
raise RuntimeError(
'recsout: unsupported reduce strategy specified')
else:
attr_dict[gid][label] = data_vec
attr_dict[gid]['t'] = time_vec
if write_trial_data:
attr_dict[gid]['trial duration'] = trial_dur
if write_cell_location_data:
distance = rec.get('distance', None)
if distance is not None:
attr_dict[gid]['distance'] = np.asarray(
[distance], dtype=np.float32)
section = rec.get('section', None)
if section is not None:
attr_dict[gid]['section'] = np.asarray([section],
dtype=np.int16)
loc = rec.get('loc', None)
if loc is not None:
attr_dict[gid]['loc'] = np.asarray([loc],
dtype=np.float32)
if clear_data:
rec['vec'].resize(0)
if env.results_namespace_id is None:
namespace_id = "Intracellular %s" % (rec_type)
else:
namespace_id = "Intracellular %s %s" % (
rec_type, str(env.results_namespace_id))
append_cell_attributes(output_path,
pop_name,
attr_dict,
namespace=namespace_id,
comm=env.comm,
io_size=env.io_size)
if clear_data:
env.t_rec.resize(0)
env.comm.barrier()
if env.comm.Get_rank() == 0:
logger.info("*** Output intracellular state results to file %s" %
output_path)
def generate_synaptic_connections(rank,
gid,
ranstream_syn,
ranstream_con,
cluster_seed,
destination_gid,
synapse_dict,
population_dict,
projection_synapse_dict,
projection_prob_dict,
connection_dict,
random_choice=random_choice_w_replacement):
"""
Given a set of synapses for a particular gid, projection
configuration, projection and connection probability dictionaries,
generates a set of possible connections for each synapse. The
procedure first assigns each synapse to a projection, using the
given proportions of each synapse type, and then chooses source
gids for each synapse using the given projection probability
dictionary.
:param ranstream_syn: random stream for the synapse partitioning step
:param ranstream_con: random stream for the choosing source gids step
:param destination_gid: destination gid
:param synapse_dict: synapse configurations, a dictionary with fields: 1) syn_ids (synapse ids) 2) syn_types (excitatory, inhibitory, etc).,
3) swc_types (SWC types(s) of synapse location in the neuronal morphological structure 3) syn_layers (synapse layer placement)
:param population_dict: mapping of population names to population indices
:param projection_synapse_dict: mapping of projection names to a tuple of the form: <syn_layer, swc_type, syn_type, syn_proportion>
:param projection_prob_dict: mapping of presynaptic population names to sets of source probabilities and source gids
:param connection_dict: output connection dictionary
:param random_choice: random choice procedure (default uses np.ranstream.multinomial)
"""
num_projections = len(projection_synapse_dict)
prj_pop_index = {
population: i
for (i, population) in enumerate(projection_synapse_dict)
}
synapse_prj_counts = np.zeros((num_projections, ))
synapse_prj_partition = defaultdict(lambda: defaultdict(list))
maxit = 10
it = 0
## assign each synapse to a projection
while (np.count_nonzero(synapse_prj_counts) < num_projections) and (it <
maxit):
log_flag = it > 1
if log_flag:
logger.info("generate_synaptic_connections: gid %i: iteration %i" %
(gid, it))
synapse_prj_counts.fill(0)
synapse_prj_partition.clear()
for (syn_id, syn_type, swc_type, syn_layer) in zip(
synapse_dict['syn_ids'], synapse_dict['syn_types'],
synapse_dict['swc_types'], synapse_dict['syn_layers']):
projection = choose_synapse_projection(ranstream_syn, syn_layer, swc_type, syn_type, \
population_dict, projection_synapse_dict, log=log_flag)
if log_flag:
logger.info('generate_synaptic_connections: gid %i: ' \
'syn_id = %i syn_type = %i swc_type = %i syn_layer = %i projection = %s' % \
(gid, syn_id, syn_type, swc_type, syn_layer, projection))
assert (projection is not None)
synapse_prj_counts[prj_pop_index[projection]] += 1
synapse_prj_partition[projection][syn_layer].append(syn_id)
it += 1
empty_projections = []
for projection in projection_synapse_dict:
logger.debug('Rank %i: gid %i: projection %s has %i synapses' %
(rank, destination_gid, projection,
len(synapse_prj_partition[projection])))
if not (len(synapse_prj_partition[projection]) > 0):
empty_projections.append(projection)
if len(empty_projections) > 0:
logger.warning('Rank %i: gid %i: projections %s have an empty synapse list; ' \
'swc types are %s layers are %s' % \
(rank, destination_gid, str(empty_projections), str(set(synapse_dict['swc_types'].flat)), \
str(set(synapse_dict['syn_layers'].flat))))
assert (len(empty_projections) == 0)
## Choose source connections based on distance-weighted probability
count = 0
for projection, prj_layer_dict in viewitems(synapse_prj_partition):
(syn_config_type, syn_config_layers, syn_config_sections, syn_config_proportions, syn_config_contacts) = \
projection_synapse_dict[projection]
gid_dict = connection_dict[projection]
prj_source_vertices = []
prj_syn_ids = []
prj_distances = []
for prj_layer, syn_ids in viewitems(prj_layer_dict):
source_probs, source_gids, distances_u, distances_v = \
projection_prob_dict[projection][prj_layer]
distance_dict = {source_gid: distance_u + distance_v \
for (source_gid, distance_u, distance_v) in \
zip(source_gids, distances_u, distances_v)}
if len(source_gids) > 0:
n_syn_groups = int(
math.ceil(
float(len(syn_ids)) / float(syn_config_contacts)))
source_gid_counts = random_choice(ranstream_con, n_syn_groups,
source_probs)
total_count = 0
if syn_config_contacts > 1:
ncontacts = int(math.ceil(syn_config_contacts))
for i in range(0, len(source_gid_counts)):
if source_gid_counts[i] > 0:
source_gid_counts[i] *= ncontacts
if len(source_gid_counts) == 0:
logger.warning('Rank %i: source vertices list is empty for gid: %i projection: %s layer: %s ' \
'source probs: %s distances_u: %s distances_v: %s' % \
(rank, destination_gid, projection, str(layer), \
str(source_probs), str(distances_u), str(distances_v)))
uv_distance_sums = np.add(distances_u,
distances_v,
dtype=np.float32)
source_vertices = np.asarray(random_clustered_shuffle(len(source_gids), \
source_gid_counts, \
center_ids=source_gids, \
cluster_std=2.0, \
random_seed=cluster_seed), \
dtype=np.uint32)[0:len(syn_ids)]
assert (len(source_vertices) == len(syn_ids))
distances = np.asarray([distance_dict[gid] for gid in source_vertices], \
dtype=np.float32).reshape(-1, )
prj_source_vertices.append(source_vertices)
prj_syn_ids.append(syn_ids)
prj_distances.append(distances)
gid_dict[destination_gid] = (np.asarray([], dtype=np.uint32), {
'Synapses': {
'syn_id': np.asarray([], dtype=np.uint32)
},
'Connections': {
'distance': np.asarray([], dtype=np.float32)
}
})
cluster_seed += 1
if len(prj_source_vertices) > 0:
prj_source_vertices_array = np.concatenate(prj_source_vertices)
else:
prj_source_vertices_array = np.asarray([], dtype=np.uint32)
del (prj_source_vertices)
if len(prj_syn_ids) > 0:
prj_syn_ids_array = np.concatenate(prj_syn_ids)
else:
prj_syn_ids_array = np.asarray([], dtype=np.uint32)
del (prj_syn_ids)
if len(prj_distances) > 0:
prj_distances_array = np.concatenate(prj_distances)
else:
prj_distances_array = np.asarray([], dtype=np.float32)
del (prj_distances)
if len(prj_source_vertices_array) == 0:
logger.warning(
'Rank %i: source gid list is empty for gid: %i projection: %s'
% (rank, destination_gid, projection))
count += len(prj_source_vertices_array)
gid_dict[destination_gid] = (prj_source_vertices_array,
{'Synapses': {'syn_id': np.asarray(prj_syn_ids_array, \
dtype=np.uint32)},
'Connections': {'distance': prj_distances_array}
})
return count
def parse_connection_config(self):
"""
:return:
"""
connection_config = self.model_config['Connection Generator']
self.connection_velocity = connection_config['Connection Velocity']
syn_mech_names = connection_config['Synapse Mechanisms']
syn_param_rules = connection_config['Synapse Parameter Rules']
self.synapse_attributes = SynapseAttributes(self, syn_mech_names,
syn_param_rules)
extent_config = connection_config['Axon Extent']
self.connection_extents = {}
for population in extent_config:
pop_connection_extents = {}
for layer_name in extent_config[population]:
if layer_name == 'default':
pop_connection_extents[layer_name] = \
{'width': extent_config[population][layer_name]['width'], \
'offset': extent_config[population][layer_name]['offset']}
else:
layer_index = self.layers[layer_name]
pop_connection_extents[layer_index] = \
{'width': extent_config[population][layer_name]['width'], \
'offset': extent_config[population][layer_name]['offset']}
self.connection_extents[population] = pop_connection_extents
synapse_config = connection_config['Synapses']
connection_dict = {}
for (key_postsyn, val_syntypes) in viewitems(synapse_config):
connection_dict[key_postsyn] = {}
for (key_presyn, syn_dict) in viewitems(val_syntypes):
val_type = syn_dict['type']
val_synsections = syn_dict['sections']
val_synlayers = syn_dict['layers']
val_proportions = syn_dict['proportions']
if 'contacts' in syn_dict:
val_contacts = syn_dict['contacts']
else:
val_contacts = 1
mechparams_dict = None
swctype_mechparams_dict = None
if 'mechanisms' in syn_dict:
mechparams_dict = syn_dict['mechanisms']
else:
swctype_mechparams_dict = syn_dict['swctype mechanisms']
res_type = self.Synapse_Types[val_type]
res_synsections = []
res_synlayers = []
res_mechparams = {}
for name in val_synsections:
res_synsections.append(self.SWC_Types[name])
for name in val_synlayers:
res_synlayers.append(self.layers[name])
if swctype_mechparams_dict is not None:
for swc_type in swctype_mechparams_dict:
swc_type_index = self.SWC_Types[swc_type]
res_mechparams[
swc_type_index] = self.parse_syn_mechparams(
swctype_mechparams_dict[swc_type])
else:
res_mechparams['default'] = self.parse_syn_mechparams(
mechparams_dict)
connection_dict[key_postsyn][key_presyn] = \
SynapseConfig(res_type, res_synsections, res_synlayers, val_proportions, val_contacts, \
res_mechparams)
config_dict = defaultdict(lambda: 0.0)
for (key_presyn,
conn_config) in viewitems(connection_dict[key_postsyn]):
for (s, l, p) in zip(conn_config.sections, conn_config.layers,
conn_config.proportions):
config_dict[(conn_config.type, s, l)] += p
for (k, v) in viewitems(config_dict):
try:
assert (np.isclose(v, 1.0))
except Exception as e:
self.logger.error(
'Connection configuration: probabilities for %s do not sum to 1: %s = %f'
% (key_postsyn, str(k), v))
raise e
self.connection_config = connection_dict
def icp_transform(comm,
env,
soma_coords,
projection_ls,
population_extents,
rotate=None,
populations=None,
icp_iter=1000,
opt_iter=100):
"""
Uses the iterative closest point (ICP) algorithm of the PCL library to transform soma coordinates onto a surface for a particular L value.
http://pointclouds.org/documentation/tutorials/iterative_closest_point.php#iterative-closest-point
"""
import dlib, pcl
rank = comm.rank
size = comm.size
if populations is None:
populations = list(soma_coords.keys())
srf_resample = 25
layer_extents = env.geometry['Parametric Surface']['Layer Extents']
(extent_u, extent_v, extent_l) = get_total_extents(layer_extents)
min_u, max_u = extent_u
min_v, max_v = extent_v
min_l, max_l = extent_l
## This parameter is used to expand the range of L and avoid
## situations where the endpoints of L end up outside of the range
## of the distance interpolant
safety = 0.01
extent_u = (min_u - safety, max_u + safety)
extent_v = (min_v - safety, max_v + safety)
projection_ptclouds = []
for obs_l in projection_ls:
srf = make_surface(extent_u, extent_v, obs_l, rotate=rotate)
U, V = srf._resample_uv(srf_resample, srf_resample)
meshpts = srf.ev(U, V)
projection_ptcloud = pcl.PointCloud()
projection_ptcloud.from_array(meshpts)
projection_ptclouds.append(projection_ptcloud)
soma_coords_dict = {}
for pop in populations:
coords_dict = soma_coords[pop]
if rank == 0:
logger.info('Computing point transformation for population %s...' %
pop)
count = 0
xyz_coords = []
gids = []
for gid, coords in viewitems(coords_dict):
if gid % size == rank:
soma_u, soma_v, soma_l = coords
xyz_coords.append(
DG_volume(soma_u, soma_v, soma_l, rotate=rotate))
gids.append(gid)
xyz_pts = np.vstack(xyz_coords)
cloud_in = pcl.PointCloud()
cloud_in.from_array(xyz_pts)
icp = cloud_in.make_IterativeClosestPoint()
all_est_xyz_coords = []
all_est_uvl_coords = []
all_interp_err = []
for (k, cloud_prj) in enumerate(projection_ls):
k_est_xyz_coords = np.zeros((len(gids), 3))
k_est_uvl_coords = np.zeros((len(gids), 3))
interp_err = np.zeros((len(gids), ))
converged, transf, estimate, fitness = icp.icp(cloud_in,
cloud_prj,
max_iter=icp_iter)
logger.info('Transformation of population %s has converged: ' %
(pop) + str(converged) + ' score: %f' % (fitness))
for i, gid in zip(list(range(0, estimate.size)), gids):
est_xyz_coords = estimate[i]
k_est_xyz_coords[i, :] = est_xyz_coords
f_uvl_distance = make_uvl_distance(est_xyz_coords,
rotate=rotate)
uvl_coords, err = dlib.find_min_global(f_uvl_distance,
limits[0], limits[1],
opt_iter)
k_est_uvl_coords[i, :] = uvl_coords
interp_err[i, ] = err
if rank == 0:
logger.info(
'gid %i: u: %f v: %f l: %f' %
(gid, uvl_coords[0], uvl_coords[1], uvl_coords[2]))
all_est_xyz_coords.append(k_est_xyz_coords)
all_est_uvl_coords.append(k_est_uvl_coords)
all_interp_err.append(interp_err)
coords_dict = {}
for (i, gid) in enumerate(gids):
coords_dict[gid] = {
'X Coordinate':
np.asarray([col[i, 0] for col in all_est_xyz_coords],
dtype='float32'),
'Y Coordinate':
np.asarray([col[i, 1] for col in all_est_xyz_coords],
dtype='float32'),
'Z Coordinate':
np.asarray([col[i, 2] for col in all_est_xyz_coords],
dtype='float32'),
'U Coordinate':
np.asarray([col[i, 0] for col in all_est_uvl_coords],
dtype='float32'),
'V Coordinate':
np.asarray([col[i, 1] for col in all_est_uvl_coords],
dtype='float32'),
'L Coordinate':
np.asarray([col[i, 2] for col in all_est_uvl_coords],
dtype='float32'),
'Interpolation Error':
np.asarray([err[i] for err in all_interp_err], dtype='float32')
}
soma_coords_dict[pop] = coords_dict
return soma_coords_dict
def load_celltypes(self):
"""
:return:
"""
rank = self.comm.Get_rank()
size = self.comm.Get_size()
celltypes = self.celltypes
if rank == 0:
color = 1
else:
color = 0
## comm0 includes only rank 0
comm0 = self.comm.Split(color, 0)
if rank == 0:
self.logger.info('env.data_file_path = %s' %
str(self.data_file_path))
self.cell_attribute_info = None
population_ranges = None
population_names = None
type_names = None
if rank == 0:
population_names = read_population_names(self.data_file_path,
comm0)
(population_ranges,
_) = read_population_ranges(self.data_file_path, comm0)
type_names = sorted(population_ranges.keys())
self.cell_attribute_info = read_cell_attribute_info(
self.data_file_path, population_names, comm=comm0)
self.logger.info('population_names = %s' % str(population_names))
self.logger.info('population_ranges = %s' % str(population_ranges))
self.logger.info('attribute info: %s' %
str(self.cell_attribute_info))
population_ranges = self.comm.bcast(population_ranges, root=0)
population_names = self.comm.bcast(population_names, root=0)
type_names = self.comm.bcast(type_names, root=0)
self.cell_attribute_info = self.comm.bcast(self.cell_attribute_info,
root=0)
comm0.Free()
for k in type_names:
population_range = population_ranges.get(k, None)
if population_range is not None:
if k not in celltypes:
celltypes[k] = {}
celltypes[k]['start'] = population_ranges[k][0]
celltypes[k]['num'] = population_ranges[k][1]
if 'mechanism file' in celltypes[k]:
celltypes[k]['mech_file_path'] = '%s/%s' % (
self.config_prefix, celltypes[k]['mechanism file'])
mech_dict = None
if rank == 0:
mech_dict = read_from_yaml(
celltypes[k]['mech_file_path'])
mech_dict = self.comm.bcast(mech_dict, root=0)
celltypes[k]['mech_dict'] = mech_dict
if 'synapses' in celltypes[k]:
synapses_dict = celltypes[k]['synapses']
if 'weights' in synapses_dict:
weights_config = synapses_dict['weights']
if isinstance(weights_config, list):
weights_dicts = weights_config
else:
weights_dicts = [weights_config]
for weights_dict in weights_dicts:
if 'expr' in weights_dict:
expr = weights_dict['expr']
parameter = weights_dict['parameter']
const = weights_dict.get('const', None)
clos = ExprClosure(parameters=parameter,
expr=expr,
consts=const)
weights_dict['closure'] = clos
synapses_dict['weights'] = weights_dicts
def vertex_distribution(connectivity_path,
coords_path,
distances_namespace,
destination,
sources,
bin_size=20.0,
cache_size=100,
comm=None):
"""
Obtain spatial histograms of source vertices connecting to a given destination population.
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
if comm is None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
color = 0
if rank == 0:
color = 1
comm0 = comm.Split(color, 0)
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
destination_soma_distances = {}
if rank == 0:
logger.info(f'Reading {destination} distances...')
distances_iter = read_cell_attributes(
coords_path,
destination,
comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
destination_soma_distances = {
k: (float(v['U Distance'][0]), float(v['V Distance'][0]))
for (k, v) in distances_iter
}
gc.collect()
comm.barrier()
destination_soma_distances = comm.bcast(destination_soma_distances, root=0)
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in viewitems(destination_soma_distances):
destination_soma_distance_U[k] = v[0]
destination_soma_distance_V[k] = v[1]
del (destination_soma_distances)
if sources == ():
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
if rank == 0:
for s in sources:
logger.info(f'Reading {s} distances...')
distances_iter = read_cell_attributes(
coords_path,
s,
comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
source_soma_distances[s] = {
k: (float(v['U Distance'][0]), float(v['V Distance'][0]))
for (k, v) in distances_iter
}
gc.collect()
comm.barrier()
comm0.Free()
source_soma_distances = comm.bcast(source_soma_distances, root=0)
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
for k, v in viewitems(source_soma_distances[s]):
this_source_soma_distance_U[k] = v[0]
this_source_soma_distance_V[k] = v[1]
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
del (source_soma_distances)
if rank == 0:
logger.info('Reading connections %s -> %s...' %
(str(sources), destination))
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
for prj_gen_tuple in zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if rank == 0 and destination_gid is not None:
logger.info('%d' % destination_gid)
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise RuntimeError(
'destination %s: destination gid %i not matched across multiple projection generators: '
'%s' % (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
for (source, (this_destination_gid,
rest)) in zip_longest(sources, prj_gen_tuple):
this_source_soma_distance_U = source_soma_distance_U[source]
this_source_soma_distance_V = source_soma_distance_V[source]
this_dist_bins = dist_bins[source]
this_dist_u_bins = dist_u_bins[source]
this_dist_v_bins = dist_v_bins[source]
(source_indexes, attr_dict) = rest
dst_U = destination_soma_distance_U[destination_gid]
dst_V = destination_soma_distance_V[destination_gid]
for source_gid in source_indexes:
dist_u = dst_U - this_source_soma_distance_U[source_gid]
dist_v = dst_V - this_source_soma_distance_V[source_gid]
dist = abs(dist_u) + abs(dist_v)
update_bins(this_dist_bins, bin_size, dist)
update_bins(this_dist_u_bins, bin_size, dist_u)
update_bins(this_dist_v_bins, bin_size, dist_v)
add_bins_op = MPI.Op.Create(add_bins, commute=True)
for source in sources:
dist_bins[source] = comm.reduce(dist_bins[source], op=add_bins_op)
dist_u_bins[source] = comm.reduce(dist_u_bins[source], op=add_bins_op)
dist_v_bins[source] = comm.reduce(dist_v_bins[source], op=add_bins_op)
dist_hist_dict = defaultdict(dict)
dist_u_hist_dict = defaultdict(dict)
dist_v_hist_dict = defaultdict(dict)
if rank == 0:
for source in sources:
dist_hist_dict[destination][source] = finalize_bins(
dist_bins[source], bin_size)
dist_u_hist_dict[destination][source] = finalize_bins(
dist_u_bins[source], bin_size)
dist_v_hist_dict[destination][source] = finalize_bins(
dist_v_bins[source], bin_size)
return {
'Total distance': dist_hist_dict,
'U distance': dist_u_hist_dict,
'V distance': dist_v_hist_dict
}
def spatial_bin_graph(connectivity_path,
coords_path,
distances_namespace,
destination,
sources,
extents,
bin_size=20.0,
cache_size=100,
comm=None):
"""
Obtain reduced graphs of the specified projections by binning nodes according to their spatial position.
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
import networkx as nx
if comm is None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
if rank == 0:
logger.info('reading %s distances...' % destination)
destination_soma_distances = bcast_cell_attributes(
coords_path,
destination,
namespace=distances_namespace,
comm=comm,
root=0)
((x_min, x_max), (y_min, y_max)) = extents
u_bins = np.arange(x_min, x_max, bin_size)
v_bins = np.arange(y_min, y_max, bin_size)
dest_u_bins = {}
dest_v_bins = {}
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in destination_soma_distances:
dist_u = v['U Distance'][0]
dist_v = v['V Distance'][0]
dest_u_bins[k] = np.searchsorted(u_bins, dist_u, side='left')
dest_v_bins[k] = np.searchsorted(v_bins, dist_v, side='left')
destination_soma_distance_U[k] = dist_u
destination_soma_distance_V[k] = dist_v
del (destination_soma_distances)
if (sources == ()) or (sources == []) or (sources is None):
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
for s in sources:
if rank == 0:
logger.info('reading %s distances...' % s)
source_soma_distances[s] = bcast_cell_attributes(
coords_path, s, namespace=distances_namespace, comm=comm, root=0)
source_u_bins = {}
source_v_bins = {}
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
this_source_u_bins = {}
this_source_v_bins = {}
for k, v in source_soma_distances[s]:
dist_u = v['U Distance'][0]
dist_v = v['V Distance'][0]
this_source_u_bins[k] = np.searchsorted(u_bins,
dist_u,
side='left')
this_source_v_bins[k] = np.searchsorted(v_bins,
dist_v,
side='left')
this_source_soma_distance_U[k] = dist_u
this_source_soma_distance_V[k] = dist_v
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
source_u_bins[s] = this_source_u_bins
source_v_bins[s] = this_source_v_bins
del (source_soma_distances)
if rank == 0:
logger.info('reading connections %s -> %s...' %
(str(sources), destination))
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
local_u_bin_graph = defaultdict(dict)
local_v_bin_graph = defaultdict(dict)
for prj_gen_tuple in zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise RuntimeError(
'destination %s: destination_gid %i not matched across multiple projection generators: '
'%s' % (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
dest_u_bin = dest_u_bins[destination_gid]
dest_v_bin = dest_v_bins[destination_gid]
for (source, (this_destination_gid,
rest)) in zip_longest(sources, prj_gen_tuple):
this_source_u_bins = source_u_bins[source]
this_source_v_bins = source_v_bins[source]
(source_indexes, attr_dict) = rest
source_u_bin_dict = defaultdict(int)
source_v_bin_dict = defaultdict(int)
for source_gid in source_indexes:
source_u_bin = this_source_u_bins[source_gid]
source_v_bin = this_source_v_bins[source_gid]
source_u_bin_dict[source_u_bin] += 1
source_v_bin_dict[source_v_bin] += 1
local_u_bin_graph[dest_u_bin][source] = source_u_bin_dict
local_v_bin_graph[dest_v_bin][source] = source_v_bin_dict
local_u_bin_graphs = comm.gather(dict(local_u_bin_graph), root=0)
local_v_bin_graphs = comm.gather(dict(local_v_bin_graph), root=0)
u_bin_graph = None
v_bin_graph = None
nu = None
nv = None
if rank == 0:
u_bin_edges = {destination: dict(ChainMap(*local_u_bin_graphs))}
v_bin_edges = {destination: dict(ChainMap(*local_v_bin_graphs))}
nu = len(u_bins)
u_bin_graph = nx.Graph()
for pop in [destination] + list(sources):
for i in range(nu):
u_bin_graph.add_node((pop, i))
for i, ss in viewitems(u_bin_edges[destination]):
for source, ids in viewitems(ss):
u_bin_graph.add_weighted_edges_from([
((source, j), (destination, i), count)
for j, count in viewitems(ids)
])
nv = len(v_bins)
v_bin_graph = nx.Graph()
for pop in [destination] + list(sources):
for i in range(nv):
v_bin_graph.add_node((pop, i))
for i, ss in viewitems(v_bin_edges[destination]):
for source, ids in viewitems(ss):
v_bin_graph.add_weighted_edges_from([
((source, j), (destination, i), count)
for j, count in viewitems(ids)
])
label = '%s to %s' % (str(sources), destination)
return {
'label': label,
'bin size': bin_size,
'destination': destination,
'sources': sources,
'U graph': u_bin_graph,
'V graph': v_bin_graph
}
def filter_by_distance(self, destination_gid, source_population,
source_layer):
"""
Given the id of a target neuron, returns the distances along u and v
and the gids of source neurons whose axons potentially contact the target neuron.
:param destination_gid: int
:param source_population: string
:return: tuple of array of int
"""
destination_coords = self.soma_coords[
self.destination_population][destination_gid]
source_coords = self.soma_coords[source_population]
destination_distances = self.soma_distances[
self.destination_population][destination_gid]
source_distances = self.soma_distances[source_population]
destination_u, destination_v, destination_l = destination_coords
destination_distance_u, destination_distance_v = destination_distances
distance_u_lst = []
distance_v_lst = []
source_u_lst = []
source_v_lst = []
source_gid_lst = []
if source_layer in self.width[source_population]:
layer_key = source_layer
elif 'default' in self.width[source_population]:
layer_key = 'default'
else:
raise RuntimeError('connection_generator.filter_by_distance: missing configuration for layer %s' % \
str(source_layer))
source_width = self.width[source_population][layer_key]
source_offset = self.offset[source_population][layer_key]
max_distance_u = source_width['u'] + source_offset['u']
max_distance_v = source_width['v'] + source_offset['v']
for (source_gid, coords) in viewitems(source_coords):
source_u, source_v, source_l = coords
source_distance_u, source_distance_v = source_distances[source_gid]
distance_u = abs(destination_distance_u - source_distance_u)
distance_v = abs(destination_distance_v - source_distance_v)
if ((max_distance_u - distance_u) > 0.0) and (
(max_distance_v - distance_v) > 0.0):
source_u_lst.append(source_u)
source_v_lst.append(source_v)
distance_u_lst.append(distance_u)
distance_v_lst.append(distance_v)
source_gid_lst.append(source_gid)
return destination_u, destination_v, np.asarray(
source_u_lst), np.asarray(source_v_lst), np.asarray(
distance_u_lst), np.asarray(distance_v_lst), np.asarray(
source_gid_lst, dtype=np.uint32)
def generate_uv_distance_connections(comm,
population_dict,
connection_config,
connection_prob,
forest_path,
synapse_seed,
connectivity_seed,
cluster_seed,
synapse_namespace,
connectivity_namespace,
connectivity_path,
io_size,
chunk_size,
value_chunk_size,
cache_size,
write_size=1,
dry_run=False):
"""
Generates connectivity based on U, V distance-weighted probabilities.
:param comm: mpi4py MPI communicator
:param connection_config: connection configuration object (instance of env.ConnectionConfig)
:param connection_prob: ConnectionProb instance
:param forest_path: location of file with neuronal trees and synapse information
:param synapse_seed: random seed for synapse partitioning
:param connectivity_seed: random seed for connectivity generation
:param cluster_seed: random seed for determining connectivity clustering for repeated connections from the same source
:param synapse_namespace: namespace of synapse properties
:param connectivity_namespace: namespace of connectivity attributes
:param io_size: number of I/O ranks to use for parallel connectivity append
:param chunk_size: HDF5 chunk size for connectivity file (pointer and index datasets)
:param value_chunk_size: HDF5 chunk size for connectivity file (value datasets)
:param cache_size: how many cells to read ahead
:param write_size: how many cells to write out at the same time
"""
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
start_time = time.time()
ranstream_syn = np.random.RandomState()
ranstream_con = np.random.RandomState()
destination_population = connection_prob.destination_population
source_populations = sorted(
connection_config[destination_population].keys())
for source_population in source_populations:
if rank == 0:
logger.info('%s -> %s:' %
(source_population, destination_population))
logger.info(
str(connection_config[destination_population]
[source_population]))
projection_config = connection_config[destination_population]
projection_synapse_dict = {
source_population: (projection_config[source_population].type,
projection_config[source_population].layers,
projection_config[source_population].sections,
projection_config[source_population].proportions,
projection_config[source_population].contacts)
for source_population in source_populations
}
comm.barrier()
total_count = 0
gid_count = 0
connection_dict = defaultdict(lambda: {})
projection_dict = {}
for destination_gid, synapse_dict in NeuroH5CellAttrGen(forest_path, \
destination_population, \
namespace=synapse_namespace, \
comm=comm, io_size=io_size, \
cache_size=cache_size):
last_time = time.time()
if destination_gid is None:
logger.info('Rank %i destination gid is None' % rank)
else:
logger.info(
'Rank %i received attributes for destination: %s, gid: %i' %
(rank, destination_population, destination_gid))
ranstream_con.seed(destination_gid + connectivity_seed)
ranstream_syn.seed(destination_gid + synapse_seed)
projection_prob_dict = {}
for source_population in source_populations:
source_layers = projection_config[source_population].layers
projection_prob_dict[source_population] = \
connection_prob.get_prob(destination_gid, source_population, source_layers)
for layer, (probs, source_gids, distances_u, distances_v) in \
viewitems(projection_prob_dict[source_population]):
if len(distances_u) > 0:
max_u_distance = np.max(distances_u)
min_u_distance = np.min(distances_u)
if rank == 0:
logger.info(
'Rank %i has %d possible sources from population %s for destination: %s, layer %s, gid: %i; max U distance: %f min U distance: %f'
% (rank, len(source_gids),
source_population, destination_population,
str(layer), destination_gid, max_u_distance,
min_u_distance))
else:
logger.warning(
'Rank %i has %d possible sources from population %s for destination: %s, layer %s, gid: %i'
% (rank, len(source_gids),
source_population, destination_population,
str(layer), destination_gid))
count = generate_synaptic_connections(
rank, destination_gid, ranstream_syn, ranstream_con,
cluster_seed + destination_gid, destination_gid, synapse_dict,
population_dict, projection_synapse_dict, projection_prob_dict,
connection_dict)
total_count += count
logger.info(
'Rank %i took %i s to compute %d edges for destination: %s, gid: %i'
% (rank, time.time() - last_time, count,
destination_population, destination_gid))
if gid_count % write_size == 0:
last_time = time.time()
if len(connection_dict) > 0:
projection_dict = {destination_population: connection_dict}
else:
projection_dict = {}
if not dry_run:
append_graph(connectivity_path,
projection_dict,
io_size=io_size,
comm=comm)
if rank == 0:
if connection_dict:
for (prj, prj_dict) in viewitems(connection_dict):
logger.info("%s: %s" %
(prj, str(list(prj_dict.keys()))))
logger.info(
'Appending connectivity for %i projections took %i s' %
(len(connection_dict), time.time() - last_time))
projection_dict.clear()
connection_dict.clear()
gc.collect()
gid_count += 1
last_time = time.time()
if len(connection_dict) > 0:
projection_dict = {destination_population: connection_dict}
else:
projection_dict = {}
if not dry_run:
append_graph(connectivity_path,
projection_dict,
io_size=io_size,
comm=comm)
if rank == 0:
if connection_dict:
for (prj, prj_dict) in viewitems(connection_dict):
logger.info("%s: %s" % (prj, str(list(prj_dict.keys()))))
logger.info(
'Appending connectivity for %i projections took %i s' %
(len(connection_dict), time.time() - last_time))
global_count = comm.gather(total_count, root=0)
if rank == 0:
logger.info(
'%i ranks took %i s to generate %i edges' %
(comm.size, time.time() - start_time, np.sum(global_count)))
def __init__(self,
comm=None,
config_file=None,
template_paths="templates",
hoc_lib_path=None,
configure_nrn=True,
dataset_prefix=None,
config_prefix=None,
results_path=None,
results_file_id=None,
results_namespace_id=None,
node_rank_file=None,
io_size=0,
recording_profile=None,
recording_fraction=1.0,
coredat=False,
tstop=0.,
v_init=-65,
stimulus_onset=0.0,
n_trials=1,
max_walltime_hours=0.5,
checkpoint_interval=500.0,
checkpoint_clear_data=True,
results_write_time=0,
dt=0.025,
ldbal=False,
lptbal=False,
transfer_debug=False,
cell_selection_path=None,
spike_input_path=None,
spike_input_namespace=None,
spike_input_attr=None,
cleanup=True,
cache_queries=False,
profile_memory=False,
verbose=False,
**kwargs):
"""
:param comm: :class:'MPI.COMM_WORLD'
:param config_file: str; model configuration file name
:param template_paths: str; colon-separated list of paths to directories containing hoc cell templates
:param hoc_lib_path: str; path to directory containing required hoc libraries
:param dataset_prefix: str; path to directory containing required neuroh5 data files
:param config_prefix: str; path to directory containing network and cell mechanism config files
:param results_path: str; path to directory to export output files
:param results_file_id: str; label for neuroh5 files to write spike and voltage trace data
:param results_namespace_id: str; label for neuroh5 namespaces to write spike and voltage trace data
:param node_rank_file: str; name of file specifying assignment of node gids to MPI ranks
:param io_size: int; the number of MPI ranks to be used for I/O operations
:param recording_profile: str; intracellular recording configuration to use
:param coredat: bool; Save CoreNEURON data
:param tstop: int; physical time to simulate (ms)
:param v_init: float; initialization membrane potential (mV)
:param stimulus_onset: float; starting time of stimulus (ms)
:param max_walltime_hours: float; maximum wall time (hours)
:param results_write_time: float; time to write out results at end of simulation
:param dt: float; simulation time step
:param ldbal: bool; estimate load balance based on cell complexity
:param lptbal: bool; calculate load balance with LPT algorithm
:param cleanup: bool; clean up auxiliary cell and synapse structures after network init
:param profile: bool; profile memory usage
:param cache_queries: bool; whether to use a cache to speed up queries to filter_synapses
:param verbose: bool; print verbose diagnostic messages while constructing the network
"""
self.kwargs = kwargs
self.SWC_Types = {}
self.SWC_Type_index = {}
self.Synapse_Types = {}
self.layers = {}
self.globals = {}
self.gidset = set([])
self.gjlist = []
self.cells = defaultdict(list)
self.artificial_cells = defaultdict(dict)
self.biophys_cells = defaultdict(dict)
self.spike_onset_delay = {}
self.recording_sets = {}
self.pc = None
if comm is None:
self.comm = MPI.COMM_WORLD
else:
self.comm = comm
rank = self.comm.Get_rank()
if configure_nrn:
from dentate.neuron_utils import h, find_template
# If true, the biophysical cells and synapses dictionary will be freed
# as synapses and connections are instantiated.
self.cleanup = cleanup
# If true, compute and print memory usage at various points
# during simulation initialization
self.profile_memory = profile_memory
# print verbose diagnostic messages
self.verbose = verbose
config_logging(verbose)
self.logger = get_root_logger()
# Directories for cell templates
if template_paths is not None:
self.template_paths = template_paths.split(':')
else:
self.template_paths = []
self.template_dict = {}
# The location of required hoc libraries
self.hoc_lib_path = hoc_lib_path
# Checkpoint interval in ms of simulation time
self.checkpoint_interval = max(float(checkpoint_interval), 1.0)
self.checkpoint_clear_data = checkpoint_clear_data
self.last_checkpoint = 0.
# The location of all datasets
self.dataset_prefix = dataset_prefix
# The path where results files should be written
self.results_path = results_path
# Identifier used to construct results data namespaces
self.results_namespace_id = results_namespace_id
# Identifier used to construct results data files
self.results_file_id = results_file_id
# Number of MPI ranks to be used for I/O operations
self.io_size = int(io_size)
# Initialization voltage
self.v_init = float(v_init)
# simulation time [ms]
self.tstop = float(tstop)
# stimulus onset time [ms]
self.stimulus_onset = float(stimulus_onset)
# number of trials
self.n_trials = int(n_trials)
# maximum wall time in hours
self.max_walltime_hours = float(max_walltime_hours)
# time to write out results at end of simulation
self.results_write_time = float(results_write_time)
# time step
self.dt = float(dt)
# used to estimate cell complexity
self.cxvec = None
# measure/perform load balancing
self.optldbal = ldbal
self.optlptbal = lptbal
self.transfer_debug = transfer_debug
# Save CoreNEURON data
self.coredat = coredat
# cache queries to filter_synapses
self.cache_queries = cache_queries
self.config_prefix = config_prefix
if config_file is not None:
if config_prefix is not None:
config_file_path = self.config_prefix + '/' + config_file
else:
config_file_path = config_file
if not os.path.isfile(config_file_path):
raise RuntimeError("configuration file %s was not found" %
config_file_path)
with open(config_file_path) as fp:
self.model_config = yaml.load(fp, IncludeLoader)
else:
raise RuntimeError("missing configuration file")
if 'Definitions' in self.model_config:
self.parse_definitions()
self.SWC_Type_index = dict([(item[1], item[0])
for item in viewitems(self.SWC_Types)])
if 'Global Parameters' in self.model_config:
self.parse_globals()
self.geometry = None
if 'Geometry' in self.model_config:
self.geometry = self.model_config['Geometry']
if 'Origin' in self.geometry['Parametric Surface']:
self.parse_origin_coords()
self.celltypes = self.model_config['Cell Types']
self.cell_attribute_info = {}
# The name of this model
if 'Model Name' in self.model_config:
self.modelName = self.model_config['Model Name']
# The dataset to use for constructing the network
if 'Dataset Name' in self.model_config:
self.datasetName = self.model_config['Dataset Name']
if rank == 0:
self.logger.info('env.dataset_prefix = %s' %
str(self.dataset_prefix))
# Cell selection for simulations of subsets of the network
self.cell_selection = None
self.cell_selection_path = cell_selection_path
if rank == 0:
self.logger.info('env.cell_selection_path = %s' %
str(self.cell_selection_path))
if cell_selection_path is not None:
with open(cell_selection_path) as fp:
self.cell_selection = yaml.load(fp, IncludeLoader)
# Spike input path
self.spike_input_path = spike_input_path
self.spike_input_ns = spike_input_namespace
self.spike_input_attr = spike_input_attr
self.spike_input_attribute_info = None
if self.spike_input_path is not None:
if rank == 0:
self.logger.info('env.spike_input_path = %s' %
str(self.spike_input_path))
self.spike_input_attribute_info = \
read_cell_attribute_info(self.spike_input_path, sorted(self.Populations.keys()), comm=self.comm)
if rank == 0:
self.logger.info('env.spike_input_attribute_info = %s' %
str(self.spike_input_attribute_info))
if results_path:
if self.results_file_id is None:
self.results_file_path = "%s/%s_results.h5" % (
self.results_path, self.modelName)
else:
self.results_file_path = "%s/%s_results_%s.h5" % (
self.results_path, self.modelName, self.results_file_id)
else:
if self.results_file_id is None:
self.results_file_path = "%s_results.h5" % (self.modelName)
else:
self.results_file_path = "%s_results_%s.h5" % (
self.modelName, self.results_file_id)
if 'Connection Generator' in self.model_config:
self.parse_connection_config()
self.parse_gapjunction_config()
if self.dataset_prefix is not None:
self.dataset_path = os.path.join(self.dataset_prefix,
self.datasetName)
if 'Cell Data' in self.model_config:
self.data_file_path = os.path.join(
self.dataset_path, self.model_config['Cell Data'])
self.forest_file_path = os.path.join(
self.dataset_path, self.model_config['Cell Data'])
self.load_celltypes()
else:
self.data_file_path = None
self.forest_file_path = None
if rank == 0:
self.logger.info('env.data_file_path = %s' %
self.data_file_path)
if 'Connection Data' in self.model_config:
self.connectivity_file_path = os.path.join(
self.dataset_path, self.model_config['Connection Data'])
else:
self.connectivity_file_path = None
if 'Gap Junction Data' in self.model_config:
self.gapjunctions_file_path = os.path.join(
self.dataset_path, self.model_config['Gap Junction Data'])
else:
self.gapjunctions_file_path = None
else:
self.dataset_path = None
self.data_file_path = None
self.connectivity_file_path = None
self.forest_file_path = None
self.gapjunctions_file_path = None
self.node_ranks = None
if node_rank_file:
self.load_node_ranks(node_rank_file)
self.netclamp_config = None
if 'Network Clamp' in self.model_config:
self.parse_netclamp_config()
self.stimulus_config = None
self.arena_id = None
self.trajectory_id = None
if 'Stimulus' in self.model_config:
self.parse_stimulus_config()
self.init_stimulus_config(**kwargs)
self.analysis_config = None
if 'Analysis' in self.model_config:
self.analysis_config = self.model_config['Analysis']
self.projection_dict = defaultdict(list)
if self.dataset_prefix is not None:
if rank == 0:
self.logger.info('env.connectivity_file_path = %s' %
str(self.connectivity_file_path))
if self.connectivity_file_path is not None:
for (src,
dst) in read_projection_names(self.connectivity_file_path,
comm=self.comm):
self.projection_dict[dst].append(src)
if rank == 0:
self.logger.info('projection_dict = %s' %
str(self.projection_dict))
# Configuration profile for recording intracellular quantities
assert ((recording_fraction >= 0.0) and (recording_fraction <= 1.0))
self.recording_fraction = recording_fraction
self.recording_profile = None
if ('Recording' in self.model_config) and (recording_profile
is not None):
self.recording_profile = self.model_config['Recording'][
'Intracellular'][recording_profile]
self.recording_profile['label'] = recording_profile
for recvar, recdict in viewitems(
self.recording_profile.get('synaptic quantity', {})):
filters = {}
if 'syn types' in recdict:
filters['syn_types'] = recdict['syn types']
if 'swc types' in recdict:
filters['swc_types'] = recdict['swc types']
if 'layers' in recdict:
filters['layers'] = recdict['layers']
if 'sources' in recdict:
filters['sources'] = recdict['sources']
syn_filters = get_syn_filter_dict(self, filters, convert=True)
recdict['syn_filters'] = syn_filters
# Configuration profile for recording local field potentials
self.LFP_config = {}
if 'Recording' in self.model_config:
for label, config in viewitems(
self.model_config['Recording']['LFP']):
self.LFP_config[label] = {
'position': tuple(config['position']),
'maxEDist': config['maxEDist'],
'fraction': config['fraction'],
'rho': config['rho'],
'dt': config['dt']
}
self.t_vec = None
self.id_vec = None
self.t_rec = None
self.recs_dict = {} # Intracellular samples on this host
for pop_name, _ in viewitems(self.Populations):
self.recs_dict[pop_name] = defaultdict(list)
# used to calculate model construction times and run time
self.mkcellstime = 0
self.mkstimtime = 0
self.connectcellstime = 0
self.connectgjstime = 0
self.simtime = None
self.lfp = {}
self.edge_count = defaultdict(dict)
self.syns_set = defaultdict(set)
def export_to_file(self,
file_path,
model_label=None,
category=None,
append=True):
"""
Exports simulated data and metadata to an HDF5 file. Arrays are saved as datasets and metadata is saved as
attributes. Repeated simulations are stored in enumerated groups.
:param file_path: str (path)
:param model_label: int or str
:param category: str
:param append: bool
"""
if append:
io_type = 'a'
else:
io_type = 'w'
with h5py.File(file_path, io_type) as f:
target = get_h5py_group(f, [model_label, 'sim_output', category],
create=True)
target.attrs['enumerated'] = True
simiter = len(target)
if str(simiter) not in target:
target.create_group(str(simiter))
target[str(simiter)].create_dataset('time',
compression='gzip',
data=self.tvec)
target[str(simiter)]['time'].attrs['dt'] = self.dt
for parameter in self.parameters:
set_h5py_attr(target[str(simiter)].attrs, parameter,
self.parameters[parameter])
if len(self.stims) > 0:
target[str(simiter)].create_group('stims')
for name, stim_dict in viewitems(self.stims):
stim = target[str(simiter)]['stims'].create_dataset(
name, compression='gzip', data=stim_dict['vec'])
cell = stim_dict['cell']
stim.attrs['cell'] = cell.gid
node = stim_dict['node']
stim.attrs['index'] = node.index
set_h5py_attr(stim.attrs, 'type', node.type)
loc = stim_dict['stim'].get_segment().x
stim.attrs['loc'] = loc
distance = get_distance_to_node(cell, cell.tree.root, node,
loc)
stim.attrs['soma_distance'] = distance
distance = get_distance_to_node(
cell, get_dendrite_origin(cell, node), node, loc)
stim.attrs['branch_distance'] = distance
stim.attrs['amp'] = stim_dict['stim'].amp
stim.attrs['delay'] = stim_dict['stim'].delay
stim.attrs['dur'] = stim_dict['stim'].dur
set_h5py_attr(stim.attrs, 'description',
stim_dict['description'])
target[str(simiter)].create_group('recs')
for name, rec_dict in viewitems(self.recs):
rec = target[str(simiter)]['recs'].create_dataset(
name, compression='gzip', data=rec_dict['vec'])
cell = rec_dict['cell']
rec.attrs['cell'] = cell.gid
node = rec_dict['node']
rec.attrs['index'] = node.index
set_h5py_attr(rec.attrs, 'type', node.type)
loc = rec_dict['loc']
rec.attrs['loc'] = loc
distance = get_distance_to_node(cell, cell.tree.root, node,
loc)
rec.attrs['soma_distance'] = distance
distance = get_distance_to_node(
cell, get_dendrite_origin(cell, node), node, loc)
node_is_terminal = is_terminal(node)
branch_order = get_branch_order(cell, node)
rec.attrs['branch_distance'] = distance
rec.attrs['is_terminal'] = node_is_terminal
rec.attrs['branch_order'] = branch_order
set_h5py_attr(rec.attrs, 'ylabel', rec_dict['ylabel'])
set_h5py_attr(rec.attrs, 'units', rec_dict['units'])
set_h5py_attr(rec.attrs, 'description',
rec_dict['description'])
def spikeout(env, output_path, t_start=None, clear_data=False):
"""
Writes spike times to specified NeuroH5 output file.
:param env:
:param output_path:
:param clear_data:
:return:
"""
equilibration_duration = float(
env.stimulus_config['Equilibration Duration'])
n_trials = env.n_trials
t_vec = np.array(env.t_vec, dtype=np.float32)
id_vec = np.array(env.id_vec, dtype=np.uint32)
trial_time_ranges = get_trial_time_ranges(env.t_rec.to_python(),
env.n_trials)
trial_time_bins = [
t_trial_start for t_trial_start, t_trial_end in trial_time_ranges
]
trial_dur = np.asarray([env.tstop + equilibration_duration] * n_trials,
dtype=np.float32)
binlst = []
typelst = sorted(env.celltypes.keys())
binvect = np.asarray([env.celltypes[k]['start'] for k in typelst])
sort_idx = np.argsort(binvect, axis=0)
pop_names = [typelst[i] for i in sort_idx]
bins = binvect[sort_idx][1:]
inds = np.digitize(id_vec, bins)
if env.results_namespace_id is None:
namespace_id = "Spike Events"
else:
namespace_id = "Spike Events %s" % str(env.results_namespace_id)
for i, pop_name in enumerate(pop_names):
spkdict = {}
sinds = np.where(inds == i)
if len(sinds) > 0:
ids = id_vec[sinds]
ts = t_vec[sinds]
for j in range(0, len(ids)):
gid = ids[j]
t = ts[j]
if (t_start is None) or (t >= t_start):
if gid in spkdict:
spkdict[gid]['t'].append(t)
else:
spkdict[gid] = {'t': [t]}
for gid in spkdict:
spiketrain = np.array(spkdict[gid]['t'], dtype=np.float32)
if gid in env.spike_onset_delay:
spiketrain -= env.spike_onset_delay[gid]
trial_bins = np.digitize(spiketrain, trial_time_bins) - 1
trial_spikes = [
np.copy(spiketrain[np.where(trial_bins == trial_i)[0]])
for trial_i in range(n_trials)
]
for trial_i, trial_spiketrain in enumerate(trial_spikes):
trial_spiketrain = trial_spikes[trial_i]
trial_spiketrain -= np.sum(
trial_dur[:(trial_i)]) + equilibration_duration
spkdict[gid]['t'] = np.concatenate(trial_spikes)
spkdict[gid]['Trial Duration'] = trial_dur
spkdict[gid]['Trial Index'] = np.asarray(trial_bins,
dtype=np.uint8)
append_cell_attributes(output_path,
pop_name,
spkdict,
namespace=namespace_id,
comm=env.comm,
io_size=env.io_size)
del (spkdict)
if clear_data:
env.t_vec.resize(0)
env.id_vec.resize(0)
env.comm.barrier()
if env.comm.Get_rank() == 0:
logger.info("*** Output spike results to file %s" % output_path)
def interp_soma_distances(comm,
ip_dist_u,
ip_dist_v,
soma_coords,
layer_extents,
population_layers,
interp_chunk_size=1000,
populations=None,
allgather=False):
"""Interpolates path lengths of cell coordinates along the dimensions of an `RBFVolume` instance.
Parameters
----------
comm : MPIComm
mpi4py MPI communicator
ip_dist_u : RBFInterpolant
Interpolation function for computing arc distances along the first dimension of the volume.
ip_dist_v : RBFInterpolant
Interpolation function for computing arc distances along the second dimension of the volume.
soma_coords : { population_name : coords_dict }
A dictionary that maps each cell population name to a dictionary of coordinates. The dictionary of coordinates must have the following type:
coords_dict : { gid : (u, v, l) }
where:
- gid: cell identifier
- u, v, l: floating point coordinates
population_layers: { population_name : layers }
A dictionary of population count per layer
Argument layers has the following type:
{ layer_name: count }
allgather: boolean (default: False)
if True, the results are gathered from all ranks and combined
Returns
-------
A dictionary of the form:
{ population: { gid: (distance_U, distance_V } }
"""
rank = comm.rank
size = comm.size
if populations is None:
populations = sorted(soma_coords.keys())
soma_distances = {}
for pop in populations:
coords_dict = soma_coords[pop]
if rank == 0:
logger.info(
'Computing soma distances for %d cells from population %s...' %
(len(coords_dict), pop))
count = 0
local_dist_dict = {}
pop_layer = population_layers[pop]
u_obs = []
v_obs = []
gids = []
for gid, coords in viewitems(coords_dict):
if gid % size == rank:
soma_u, soma_v, soma_l = coords
try:
assert (uvl_in_bounds(coords, layer_extents, pop_layer))
except Exception as e:
logger.error("gid %i: out of limits error for coordinates: %f %f %f)" % \
(gid, soma_u, soma_v, soma_l))
raise e
u_obs.append(np.array([soma_u, soma_v, soma_l]).ravel())
v_obs.append(np.array([soma_u, soma_v, soma_l]).ravel())
gids.append(gid)
if len(u_obs) > 0:
u_obs_array = np.vstack(u_obs)
v_obs_array = np.vstack(v_obs)
distance_u_obs = ip_dist_u(u_obs_array).reshape(-1, 1)
distance_v_obs = ip_dist_v(v_obs_array).reshape(-1, 1)
distance_u = np.mean(distance_u_obs, axis=1)
distance_v = np.mean(distance_v_obs, axis=1)
try:
assert (np.all(np.isfinite(distance_u)))
assert (np.all(np.isfinite(distance_v)))
except Exception as e:
u_nan_idxs = np.where(np.isnan(distance_u))[0]
v_nan_idxs = np.where(np.isnan(distance_v))[0]
logger.error('Invalid distances: u: %s; v: %s',
str(u_obs_array[u_nan_idxs]),
str(v_obs_array[v_nan_idxs]))
raise e
for (i, gid) in enumerate(gids):
local_dist_dict[gid] = (distance_u[i], distance_v[i])
if rank == 0:
logger.info('gid %i: distances: %f %f' %
(gid, distance_u[i], distance_v[i]))
if allgather:
dist_dicts = comm.allgather(local_dist_dict)
combined_dist_dict = {}
for dist_dict in dist_dicts:
for k, v in viewitems(dist_dict):
combined_dist_dict[k] = v
soma_distances[pop] = combined_dist_dict
else:
soma_distances[pop] = local_dist_dict
return soma_distances
def __init__(self, destination_population, soma_coords, soma_distances,
extents):
"""
Warning: This method does not produce an absolute probability. It must be normalized so that the total area
(volume) under the distribution is 1 before sampling.
:param destination_population: post-synaptic population name
:param soma_distances: a dictionary that contains per-population dicts of u, v distances of cell somas
:param extent: dict: {source: 'width': (tuple of float), 'offset': (tuple of float)}
"""
self.destination_population = destination_population
self.soma_coords = soma_coords
self.soma_distances = soma_distances
self.p_dist = defaultdict(dict)
self.width = defaultdict(dict)
self.offset = defaultdict(dict)
self.scale_factor = defaultdict(dict)
for source_population, layer_extents in viewitems(extents):
for layer, extents in viewitems(layer_extents):
extent_width = extents['width']
if 'offset' in extents:
extent_offset = extents['offset']
else:
extent_offset = (0., 0.)
u_extent = (float(extent_width[0]) / 2.0) - float(
extent_offset[0])
v_extent = (float(extent_width[1]) / 2.0) - float(
extent_offset[1])
self.width[source_population][layer] = {
'u': u_extent,
'v': v_extent
}
self.scale_factor[source_population][layer] = \
{axis: self.width[source_population][layer][axis] / 3. \
for axis in self.width[source_population][layer]}
if extent_offset is None:
self.offset[source_population][layer] = {'u': 0., 'v': 0.}
else:
self.offset[source_population][layer] = {'u': float(extent_offset[0]), \
'v': float(extent_offset[1])}
self.p_dist[source_population][layer] = \
(lambda source_population, layer: \
np.vectorize(lambda distance_u, distance_v: \
(norm.pdf(np.abs(distance_u) - self.offset[source_population][layer]['u'], \
scale=self.scale_factor[source_population][layer]['u']) * \
norm.pdf(np.abs(distance_v) - self.offset[source_population][layer]['v'], \
scale=self.scale_factor[source_population][layer]['v'])), \
otypes=[float]))(source_population, layer)
logger.info('population %s: layer: %s: u ' \
'width: %f v width: %f u scale_factor: %f v scale_factor: %f' % \
(source_population, str(layer), \
self.width[source_population][layer]['u'], \
self.width[source_population][layer]['v'], \
self.scale_factor[source_population][layer]['u'], \
self.scale_factor[source_population][layer]['v']))
