def main(template_path, forest_path, synapses_path, connections_path, config_path):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
env = Env(comm=comm, config_file=config_path, template_paths=template_path)
h('objref nil, pc, tlog, Vlog, spikelog')
h.load_file("nrngui.hoc")
h.xopen ("./tests/rn.hoc")
h.xopen(template_path+'/HICAPCell.hoc')
pop_name = "HCC"
gid = 1043250
(trees_dict,_) = read_tree_selection (forest_path, pop_name, [gid], comm=env.comm)
(_, tree) = next(trees_dict)
v_init = -67
template_class = getattr(h, "HICAPCell")
passive_test(template_class, tree, v_init)
ap_test(template_class, tree, v_init)
ap_rate_test(template_class, tree, v_init)
fi_test(template_class, tree, v_init)
if synapses_path and connections_path:
synapses_iter = read_cell_attribute_selection (synapses_path, pop_name, [gid],
"Synapse Attributes", comm=env.comm)
(_, synapses_dict) = next(synapses_iter)
connections = read_graph_selection(file_name=connections_path, selection=[gid],
namespaces=['Synapses', 'Connections'], comm=env.comm)
synapse_test(template_class, gid, tree, synapses_dict, connections, v_init, env)
802644, 395919, 540509, 746873, 278368, 540518, 409447, 147304, 802666, 16235, 802668,
933748, 876521, 16249, 16258, 417771, 671626, 540557, 540559, 671632, 409491, 985753,
409496, 147371, 671661, 409521, 540594, 933813, 671676, 409542, 802759, 898380,
540620, 933838, 278482, 147411, 671705, 671706, 147421, 802784, 16353, 278498, 16358,
802793, 671722, 802795, 540652, 671725, 16366, 933871, 671729, 278514, 16375, 898388,
409594, 147455, 147457, 540674, 16390, 933896, 671753, 898392, 278546, 540693, 278550,
278551, 671770, 147488, 16417, 16425, 147500, 409645, 147504, 671796, 308575, 16444,
592565, 278592, 147524, 540743, 409675, 671820, 16462, 409679, 802903, 409693, 540768,
540769, 278630, 671847, 147565, 671854, 802935, 671869, 147583, 147587, 671881,
934026, 671890, 540820, 278685, 540832, 802982, 671911, 934056, 540841, 147629,
278702, 278704, 854728, 671923, 278712, 960746, 540858, 409788, 409793, 540878,
540880, 803026, 409814, 147686, 16618, 147691, 934126, 409847, 409853, 803072, 16642,
409861, 147718, 16649, 395991, 934163, 803096, 278809, 409882, 147739, 540957, 934175,
147749, 934182, 540967, 409897, 278826, 934188, 16691, 16692, 707608, 540984, 177546]
gid_range = sorted([ gid for gid in gid_set if gid % size == rank ])
print("rank %i: gid_range = %s" % (rank, str(gid_range)))
connectivity_file_path='/oasis/scratch/comet/iraikov/temp_project/dentate/Full_Scale_Control/DG_Connections_Full_Scale_20190824.h5'
(graph, a) = read_graph_selection(connectivity_file_path, selection=gid_range, projections=[('MPP', 'GC'), ('MC', 'GC'), ('BC', 'GC')],
comm=comm, namespaces=['Synapses', 'Connections'])
print(a)
for dst, edges in graph['GC']['MPP']:
presyn_gids, _ = edges
print("rank %d: %d %s edges for node %d" % (rank, len(presyn_gids), "MPP", dst))
for dst, edges in graph['GC']['MC']:
presyn_gids, _ = edges
print("rank %d: %d %s edges for node %d" % (rank, len(presyn_gids), "MC", dst))
def main(config, coordinates, gid, field_width, peak_rate, input_features_path,
input_features_namespaces, output_features_namespace,
output_weights_path, output_features_path, initial_weights_path,
reference_weights_path, h5types_path, synapse_name,
initial_weights_namespace, reference_weights_namespace,
output_weights_namespace, reference_weights_are_delta,
connections_path, optimize_method, destination, sources, arena_id,
max_delta_weight, field_width_scale, max_iter, verbose, dry_run,
plot):
"""
:param config: str (path to .yaml file)
:param coordinates: tuple of float
:param gid: int
:param field_width: float
:param peak_rate: float
:param input_features_path: str (path to .h5 file)
:param input_features_namespaces: str
:param output_features_namespace: str
:param output_weights_path: str (path to .h5 file)
:param output_features_path: str (path to .h5 file)
:param initial_weights_path: str (path to .h5 file)
:param reference_weights_path: str (path to .h5 file)
:param h5types_path: str (path to .h5 file)
:param synapse_name: str
:param initial_weights_namespace: str
:param output_weights_namespace: str
:param reference_weights_are_delta: bool
:param connections_path: str (path to .h5 file)
:param destination: str (population name)
:param sources: list of str (population name)
:param arena_id: str
:param max_delta_weight: float
:param field_width_scale: float
:param max_iter: int
:param verbose: bool
:param dry_run: bool
:param interactive: bool
:param plot: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
env = Env(config_file=config)
if not dry_run:
if output_weights_path is None:
raise RuntimeError(
'Missing required argument: output_weights_path.')
if not os.path.isfile(output_weights_path):
if initial_weights_path is not None and os.path.isfile(
initial_weights_path):
input_file_path = initial_weights_path
elif h5types_path is not None and os.path.isfile(h5types_path):
input_file_path = h5types_path
else:
raise RuntimeError(
'Missing required source for h5types: either an initial_weights_path or an '
'h5types_path must be provided.')
with h5py.File(output_weights_path, 'a') as output_file:
with h5py.File(input_file_path, 'r') as input_file:
input_file.copy('/H5Types', output_file)
this_input_features_namespaces = [
'%s %s' % (input_features_namespace, arena_id)
for input_features_namespace in input_features_namespaces
]
features_attr_names = ['Arena Rate Map']
spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm
arena = env.stimulus_config['Arena'][arena_id]
default_run_vel = arena.properties['default run velocity'] # cm/s
arena_x, arena_y = stimulus.get_2D_arena_spatial_mesh(
arena, spatial_resolution)
dim_x = len(arena_x)
dim_y = len(arena_y)
if gid is None:
target_gids = []
else:
target_gids = [gid]
dst_input_features = defaultdict(dict)
num_fields = len(coordinates)
this_field_width = np.array([field_width] * num_fields, dtype=np.float32)
this_scaled_field_width = np.array([field_width * field_width_scale] *
num_fields,
dtype=np.float32)
this_peak_rate = np.array([peak_rate] * num_fields, dtype=np.float32)
this_x0 = np.array([x for x, y in coordinates], dtype=np.float32)
this_y0 = np.array([y for x, y in coordinates], dtype=np.float32)
this_rate_map = np.asarray(get_rate_map(this_x0, this_y0, this_field_width,
this_peak_rate, arena_x, arena_y),
dtype=np.float32)
target_map = np.asarray(get_rate_map(this_x0, this_y0,
this_scaled_field_width,
this_peak_rate, arena_x, arena_y),
dtype=np.float32)
selectivity_type = env.selectivity_types['place']
dst_input_features[destination][target_gid] = {
'Selectivity Type': np.array([selectivity_type], dtype=np.uint8),
'Num Fields': np.array([num_fields], dtype=np.uint8),
'Field Width': this_field_width,
'Peak Rate': this_peak_rate,
'X Offset': this_x0,
'Y Offset': this_y0,
'Arena Rate Map': this_rate_map.ravel()
}
initial_weights_by_syn_id_dict = dict()
selection = [target_gid]
if initial_weights_path is not None:
initial_weights_iter = \
read_cell_attribute_selection(initial_weights_path, destination, namespace=initial_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(initial_weights_iter)
syn_ids = syn_weight_attr_dict[target_gid]['syn_id']
weights = syn_weight_attr_dict[target_gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
initial_weights_by_syn_id_dict[int(syn_id)] = float(weight)
logger.info(
'destination: %s; gid %i; read initial synaptic weights for %i synapses'
% (destination, target_gid, len(initial_weights_by_syn_id_dict)))
reference_weights_by_syn_id_dict = None
if reference_weights_path is not None:
reference_weights_by_syn_id_dict = dict()
reference_weights_iter = \
read_cell_attribute_selection(reference_weights_path, destination, namespace=reference_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(reference_weights_iter)
syn_ids = syn_weight_attr_dict[target_gid]['syn_id']
weights = syn_weight_attr_dict[target_gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
reference_weights_by_syn_id_dict[int(syn_id)] = float(weight)
logger.info(
'destination: %s; gid %i; read reference synaptic weights for %i synapses'
% (destination, target_gid, len(reference_weights_by_syn_id_dict)))
source_gid_set_dict = defaultdict(set)
syn_ids_by_source_gid_dict = defaultdict(list)
initial_weights_by_source_gid_dict = dict()
if reference_weights_by_syn_id_dict is None:
reference_weights_by_source_gid_dict = None
else:
reference_weights_by_source_gid_dict = dict()
(graph, edge_attr_info) = read_graph_selection(file_name=connections_path,
selection=[target_gid],
namespaces=['Synapses'])
syn_id_attr_index = None
for source, edge_iter in viewitems(graph[destination]):
if source not in sources:
continue
this_edge_attr_info = edge_attr_info[destination][source]
if 'Synapses' in this_edge_attr_info and \
'syn_id' in this_edge_attr_info['Synapses']:
syn_id_attr_index = this_edge_attr_info['Synapses']['syn_id']
for (destination_gid, edges) in edge_iter:
assert destination_gid == target_gid
source_gids, edge_attrs = edges
syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
count = 0
for i in range(len(source_gids)):
this_source_gid = int(source_gids[i])
source_gid_set_dict[source].add(this_source_gid)
this_syn_id = int(syn_ids[i])
if this_syn_id not in initial_weights_by_syn_id_dict:
this_weight = \
env.connection_config[destination][source].mechanisms['default'][synapse_name]['weight']
initial_weights_by_syn_id_dict[this_syn_id] = this_weight
syn_ids_by_source_gid_dict[this_source_gid].append(this_syn_id)
if this_source_gid not in initial_weights_by_source_gid_dict:
initial_weights_by_source_gid_dict[this_source_gid] = \
initial_weights_by_syn_id_dict[this_syn_id]
if reference_weights_by_source_gid_dict is not None:
reference_weights_by_source_gid_dict[this_source_gid] = \
reference_weights_by_syn_id_dict[this_syn_id]
count += 1
logger.info(
'destination: %s; gid %i; set initial synaptic weights for %d inputs from source population '
'%s' % (destination, destination_gid, count, source))
syn_count_by_source_gid_dict = dict()
for source_gid in syn_ids_by_source_gid_dict:
syn_count_by_source_gid_dict[source_gid] = len(
syn_ids_by_source_gid_dict[source_gid])
input_rate_maps_by_source_gid_dict = dict()
for source in sources:
source_gids = list(source_gid_set_dict[source])
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(
input_features_path,
source,
namespace=input_features_namespace,
mask=set(features_attr_names),
selection=source_gids)
count = 0
for gid, attr_dict in input_features_iter:
input_rate_maps_by_source_gid_dict[gid] = attr_dict[
'Arena Rate Map'].reshape((dim_x, dim_y))
count += 1
logger.info('Read %s feature data for %i cells in population %s' %
(input_features_namespace, count, source))
if is_interactive:
context.update(locals())
normalized_delta_weights_dict, arena_LS_map = \
synapses.generate_structured_weights(target_map=target_map,
initial_weight_dict=initial_weights_by_source_gid_dict,
input_rate_map_dict=input_rate_maps_by_source_gid_dict,
syn_count_dict=syn_count_by_source_gid_dict,
max_delta_weight=max_delta_weight, arena_x=arena_x, arena_y=arena_y,
reference_weight_dict=reference_weights_by_source_gid_dict,
reference_weights_are_delta=reference_weights_are_delta,
reference_weights_namespace=reference_weights_namespace,
optimize_method=optimize_method, verbose=verbose, plot=plot)
output_syn_ids = np.empty(len(initial_weights_by_syn_id_dict),
dtype='uint32')
output_weights = np.empty(len(initial_weights_by_syn_id_dict),
dtype='float32')
i = 0
for source_gid, this_weight in viewitems(normalized_delta_weights_dict):
for syn_id in syn_ids_by_source_gid_dict[source_gid]:
output_syn_ids[i] = syn_id
output_weights[i] = this_weight
i += 1
output_weights_dict = {
target_gid: {
'syn_id': output_syn_ids,
synapse_name: output_weights
}
}
logger.info('destination: %s; gid %i; generated %s for %i synapses' %
(destination, target_gid, output_weights_namespace,
len(output_weights)))
if not dry_run:
this_output_weights_namespace = '%s %s' % (output_weights_namespace,
arena_id)
logger.info('Destination: %s; appending %s ...' %
(destination, this_output_weights_namespace))
append_cell_attributes(output_weights_path,
destination,
output_weights_dict,
namespace=this_output_weights_namespace)
logger.info('Destination: %s; appended %s' %
(destination, this_output_weights_namespace))
output_weights_dict.clear()
if output_features_path is not None:
this_output_features_namespace = '%s %s' % (
output_features_namespace, arena_id)
cell_attr_dict = dst_input_features[destination]
cell_attr_dict[target_gid]['Arena State Map'] = np.asarray(
arena_LS_map.ravel(), dtype=np.float32)
logger.info('Destination: %s; appending %s ...' %
(destination, this_output_features_namespace))
append_cell_attributes(output_features_path,
destination,
cell_attr_dict,
namespace=this_output_features_namespace)
if is_interactive:
context.update(locals())
def main(config_file, config_prefix, input_path, population, template_paths,
dataset_prefix, results_path, results_file_id, results_namespace_id,
v_init, io_size, chunk_size, value_chunk_size, write_size, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
if io_size == -1:
io_size = comm.size
if results_file_id is None:
if rank == 0:
result_file_id = uuid.uuid4()
results_file_id = comm.bcast(results_file_id, root=0)
if results_namespace_id is None:
results_namespace_id = 'Cell Clamp Results'
comm = MPI.COMM_WORLD
np.seterr(all='raise')
verbose = True
params = dict(locals())
env = Env(**params)
configure_hoc_env(env)
if rank == 0:
io_utils.mkout(env, env.results_file_path)
env.comm.barrier()
env.cell_selection = {}
template_class = load_cell_template(env, population)
if input_path is not None:
env.data_file_path = input_path
env.load_celltypes()
synapse_config = env.celltypes[population]['synapses']
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
start_time = time.time()
count = 0
gid_count = 0
attr_dict = {}
if input_path is None:
cell_path = env.data_file_path
connectivity_path = env.connectivity_file_path
else:
cell_path = input_path
connectivity_path = input_path
for gid, morph_dict in NeuroH5TreeGen(cell_path,
population,
io_size=io_size,
comm=env.comm,
topology=True):
local_time = time.time()
if gid is not None:
color = 0
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid: %i' % (rank, gid))
cell_dict = {'morph': morph_dict}
synapses_iter = read_cell_attribute_selection(cell_path,
population, [gid],
'Synapse Attributes',
comm=comm0)
_, synapse_dict = next(synapses_iter)
cell_dict['synapse'] = synapse_dict
if has_weights:
cell_weights_iters = [
read_cell_attribute_selection(cell_path,
population, [gid],
weights_namespace,
comm=comm0)
for weights_namespace in weights_namespaces
]
weight_dict = dict(
zip_longest(weights_namespaces, cell_weights_iters))
cell_dict['weight'] = weight_dict
(graph,
a) = read_graph_selection(file_name=connectivity_path,
selection=[gid],
namespaces=['Synapses', 'Connections'],
comm=comm0)
cell_dict['connectivity'] = (graph, a)
gid_count += 1
attr_dict[gid] = {}
attr_dict[gid].update(
cell_clamp.measure_passive(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap_rate(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_fi(gid,
population,
v_init,
env,
cell_dict=cell_dict))
else:
color = 1
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid is None' % (rank))
comm0.Free()
count += 1
if (results_path is not None) and (count % write_size == 0):
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
attr_dict = {}
env.comm.barrier()
if results_path is not None:
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
global_count = env.comm.gather(gid_count, root=0)
MPI.Finalize()
def write_connection_selection(env,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Loads NeuroH5 connectivity file, and writes the corresponding
synapse and network connection mechanisms for the selected postsynaptic cells.
:param env: an instance of the `dentate.Env` class
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
connectivity_file_path = env.connectivity_file_path
forest_file_path = env.forest_file_path
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
syn_attrs = env.synapse_attributes
if populations is None:
pop_names = sorted(env.cell_selection.keys())
else:
pop_names = populations
input_sources = {pop_name: set([]) for pop_name in env.celltypes}
for (postsyn_name, presyn_names) in sorted(viewitems(env.projection_dict)):
if rank == 0:
logger.info('*** Writing connection selection of population %s' %
(postsyn_name))
if postsyn_name not in pop_names:
continue
gid_range = [
gid for gid in env.cell_selection[postsyn_name]
if gid % nhosts == rank
]
synapse_config = env.celltypes[postsyn_name]['synapses']
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
else:
has_weights = False
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(
synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(
synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
if rank == 0:
logger.info('*** Reading synaptic attributes of population %s' %
(postsyn_name))
syn_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace='Synapse Attributes',
comm=env.comm)
syn_attributes_output_dict = dict(list(syn_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
syn_attributes_output_dict,
namespace='Synapse Attributes',
**write_kwds)
del syn_attributes_output_dict
del syn_attributes_iter
if has_weights:
for weights_namespace in sorted(weights_namespaces):
weight_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace=weights_namespace,
comm=env.comm)
weight_attributes_output_dict = dict(
list(weight_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
weight_attributes_output_dict,
namespace=weights_namespace,
**write_kwds)
del weight_attributes_output_dict
del weight_attributes_iter
logger.info(
'*** Rank %i: reading connectivity selection from file %s for postsynaptic population: %s: selection: %s'
% (rank, connectivity_file_path, postsyn_name, str(gid_range)))
(graph, attr_info) = read_graph_selection(connectivity_file_path, selection=gid_range, \
projections=[ (presyn_name, postsyn_name) for presyn_name in sorted(presyn_names) ], \
comm=env.comm, namespaces=['Synapses', 'Connections'])
for presyn_name in sorted(presyn_names):
gid_dict = {}
edge_count = 0
node_count = 0
if postsyn_name in graph:
if postsyn_name in attr_info and presyn_name in attr_info[
postsyn_name]:
edge_attr_info = attr_info[postsyn_name][presyn_name]
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
if 'Synapses' in edge_attr_info and \
'syn_id' in edge_attr_info['Synapses'] and \
'Connections' in edge_attr_info and \
'distance' in edge_attr_info['Connections']:
syn_id_attr_index = edge_attr_info['Synapses']['syn_id']
distance_attr_index = edge_attr_info['Connections'][
'distance']
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
edge_iter = compose_iter(lambda edgeset: input_sources[presyn_name].update(edgeset[1][0]), \
graph[postsyn_name][presyn_name])
for (postsyn_gid, edges) in edge_iter:
presyn_gids, edge_attrs = edges
edge_syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
edge_dists = edge_attrs['Connections'][distance_attr_index]
gid_dict[postsyn_gid] = (presyn_gids, {
'Synapses': {
'syn_id': edge_syn_ids
},
'Connections': {
'distance': edge_dists
}
})
edge_count += len(presyn_gids)
node_count += 1
logger.info(
'*** Rank %d: Writing projection %s -> %s selection: %d nodes, %d edges'
% (rank, presyn_name, postsyn_name, node_count, edge_count))
write_graph(write_selection_file_path, \
src_pop_name=presyn_name, dst_pop_name=postsyn_name, \
edges=gid_dict, comm=env.comm, io_size=env.io_size)
env.comm.barrier()
return input_sources
def init_circuit_context(env,
pop_name,
gid,
load_edges=False,
connection_graph=None,
load_weights=False,
weight_dict=None,
load_synapses=False,
synapses_dict=None,
set_edge_delays=True,
**kwargs):
syn_attrs = env.synapse_attributes
synapse_config = env.celltypes[pop_name]['synapses']
has_weights = False
weight_config = []
if 'weights' in synapse_config:
has_weights = True
weight_config = synapse_config['weights']
init_synapses = False
init_weights = False
init_edges = False
if load_edges or (connection_graph is not None):
init_synapses = True
init_edges = True
if has_weights and (load_weights or (weight_dict is not None)):
init_synapses = True
init_weights = True
if load_synapses or (synapses_dict is not None):
init_synapses = True
if init_synapses:
if synapses_dict is not None:
syn_attrs.init_syn_id_attrs(gid, **synapses_dict)
elif load_synapses or load_edges:
if (pop_name in env.cell_attribute_info) and (
'Synapse Attributes' in env.cell_attribute_info[pop_name]):
synapses_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name, [gid],
'Synapse Attributes',
mask=set([
'syn_ids', 'syn_locs', 'syn_secs', 'syn_layers',
'syn_types', 'swc_types'
]),
comm=env.comm)
syn_attrs.init_syn_id_attrs_from_iter(synapses_iter)
else:
raise RuntimeError(
'init_circuit_context: synapse attributes not found for %s: gid: %i'
% (pop_name, gid))
else:
raise RuntimeError(
"init_circuit_context: invalid synapses parameters")
if init_weights and has_weights:
for weight_config_dict in weight_config:
expr_closure = weight_config_dict.get('closure', None)
weights_namespaces = weight_config_dict['namespace']
cell_weights_dicts = {}
if weight_dict is not None:
for weights_namespace in weights_namespaces:
if weights_namespace in weight_dict:
cell_weights_dicts[weights_namespace] = weight_dict[
weights_namespace]
elif load_weights:
if (env.data_file_path is None):
raise RuntimeError(
'init_circuit_context: load_weights=True but data file path is not specified '
)
for weights_namespace in weights_namespaces:
cell_weights_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name,
selection=[gid],
namespace=weights_namespace,
comm=env.comm)
for cell_weights_gid, cell_weights_dict in cell_weights_iter:
assert (cell_weights_gid == gid)
cell_weights_dicts[
weights_namespace] = cell_weights_dict
else:
raise RuntimeError(
"init_circuit_context: invalid weights parameters")
if len(weights_namespaces) != len(cell_weights_dicts):
logger.warning(
"init_circuit_context: Unable to load all weights namespaces: %s"
% str(weights_namespaces))
multiple_weights = 'error'
append_weights = False
for weights_namespace in weights_namespaces:
if weights_namespace in cell_weights_dicts:
cell_weights_dict = cell_weights_dicts[weights_namespace]
weights_syn_ids = cell_weights_dict['syn_id']
for syn_name in (syn_name for syn_name in cell_weights_dict
if syn_name != 'syn_id'):
weights_values = cell_weights_dict[syn_name]
syn_attrs.add_mech_attrs_from_iter(
gid,
syn_name,
zip_longest(
weights_syn_ids,
[{
'weight': Promise(expr_closure, [x])
} for x in weights_values]
if expr_closure else [{
'weight': x
} for x in weights_values]),
multiple=multiple_weights,
append=append_weights)
logger.info(
'init_circuit_context: gid: %i; found %i %s synaptic weights in namespace %s'
% (gid, len(cell_weights_dict[syn_name]), syn_name,
weights_namespace))
logger.info(
'weight_values min/max/mean: %.02f / %.02f / %.02f'
% (np.min(weights_values), np.max(weights_values),
np.mean(weights_values)))
expr_closure = None
append_weights = True
multiple_weights = 'overwrite'
if init_edges:
if connection_graph is not None:
(graph, a) = connection_graph
elif load_edges:
if env.connectivity_file_path is None:
raise RuntimeError(
'init_circuit_context: load_edges=True but connectivity file path is not specified '
)
elif os.path.isfile(env.connectivity_file_path):
(graph, a) = read_graph_selection(
file_name=env.connectivity_file_path,
selection=[gid],
namespaces=['Synapses', 'Connections'],
comm=env.comm)
else:
raise RuntimeError(
'init_circuit_context: connection file %s not found' %
env.connectivity_file_path)
else:
(graph, a) = None, None
if graph is not None:
if pop_name in graph:
for presyn_name in graph[pop_name].keys():
edge_iter = graph[pop_name][presyn_name]
syn_attrs.init_edge_attrs_from_iter(pop_name, presyn_name, a,
edge_iter, set_edge_delays)
else:
logger.error(
'init_circuit_context: connection attributes not found for %s: gid: %i'
% (pop_name, gid))
raise Exception
def load_biophys_cell_dicts(env,
pop_name,
gid_set,
data_file_path=None,
load_connections=True,
validate_tree=True):
"""
Loads the data necessary to instantiate BiophysCell into the given dictionary.
:param env: an instance of env.Env
:param pop_name: population name
:param gid: gid
:param data_file_path: str or None
:param load_connections: bool
:param validate_tree: bool
Environment can be instantiated as:
env = Env(config_file, template_paths, dataset_prefix, config_prefix)
:param template_paths: str; colon-separated list of paths to directories containing hoc cell templates
:param dataset_prefix: str; path to directory containing required neuroh5 data files
:param config_prefix: str; path to directory containing network and cell mechanism config files
"""
synapse_config = env.celltypes[pop_name]['synapses']
has_weights = False
weights_config = None
if 'weights' in synapse_config:
has_weights = True
weights_config = synapse_config['weights']
## Loads cell morphological data, synaptic attributes and connection data
tree_dicts = {}
synapses_dicts = {}
weight_dicts = {}
connection_graphs = {gid: {pop_name: {}} for gid in gid_set}
graph_attr_info = None
gid_list = list(gid_set)
tree_attr_iter, _ = read_tree_selection(env.data_file_path,
pop_name,
gid_list,
comm=env.comm,
topology=True,
validate=validate_tree)
for gid, tree_dict in tree_attr_iter:
tree_dicts[gid] = tree_dict
if load_connections:
synapses_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name,
gid_list,
'Synapse Attributes',
mask=set([
'syn_ids', 'syn_locs', 'syn_secs', 'syn_layers', 'syn_types',
'swc_types'
]),
comm=env.comm)
for gid, attr_dict in synapses_iter:
synapses_dicts[gid] = attr_dict
if has_weights:
for config in weights_config:
weights_namespaces = config['namespace']
cell_weights_iters = [
read_cell_attribute_selection(env.data_file_path,
pop_name,
gid_list,
weights_namespace,
comm=env.comm)
for weights_namespace in weights_namespaces
]
for weights_namespace, cell_weights_iter in zip_longest(
weights_namespaces, cell_weights_iters):
for gid, cell_weights_dict in cell_weights_iter:
this_weights_dict = weight_dicts.get(gid, {})
this_weights_dict[
weights_namespace] = cell_weights_dict
weight_dicts[gid] = this_weights_dict
graph, graph_attr_info = read_graph_selection(
file_name=env.connectivity_file_path,
selection=gid_list,
namespaces=['Synapses', 'Connections'],
comm=env.comm)
if pop_name in graph:
for presyn_name in graph[pop_name].keys():
edge_iter = graph[pop_name][presyn_name]
for (postsyn_gid, edges) in edge_iter:
connection_graphs[postsyn_gid][pop_name][presyn_name] = [
(postsyn_gid, edges)
]
cell_dicts = {}
for gid in gid_set:
this_cell_dict = {}
tree_dict = tree_dicts[gid]
this_cell_dict['morph'] = tree_dict
if load_connections:
synapses_dict = synapses_dicts[gid]
weight_dict = weight_dicts.get(gid, None)
connection_graph = connection_graphs[gid]
this_cell_dict['synapse'] = synapses_dict
this_cell_dict['connectivity'] = connection_graph, graph_attr_info
this_cell_dict['weight'] = weight_dict
cell_dicts[gid] = this_cell_dict
return cell_dicts
def main(config, coordinates, gid, field_width, peak_rate, input_features_path, input_features_namespaces,
output_weights_path, output_features_path, weights_path, h5types_path, synapse_name, initial_weights_namespace,
structured_weights_namespace, connections_path, destination, sources, arena_id, baseline_weight,
field_width_scale, max_iter, verbose, dry_run, interactive):
"""
:param config: str (path to .yaml file)
:param weights_path: str (path to .h5 file)
:param initial_weights_namespace: str
:param structured_weights_namespace: str
:param connections_path: str (path to .h5 file)
:param destination: str
:param sources: list of str
:param verbose:
:param dry_run:
:return:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
env = Env(config_file=config)
if output_weights_path is None:
if weights_path is None:
raise RuntimeError('Output weights path must be specified when weights path is not specified.')
output_weights_path = weights_path
if (not dry_run):
if not os.path.isfile(output_weights_path):
if weights_path is not None:
input_file = h5py.File(weights_path,'r')
elif h5types_path is not None:
input_file = h5py.File(h5types_path,'r')
else:
raise RuntimeError('h5types input path must be specified when weights path is not specified.')
output_file = h5py.File(output_weights_path,'w')
input_file.copy('/H5Types',output_file)
input_file.close()
output_file.close()
this_input_features_namespaces = ['%s %s' % (input_features_namespace, arena_id) for input_features_namespace in input_features_namespaces]
initial_weights_dict = None
if weights_path is not None:
logger.info('Reading initial weights data from %s...' % weights_path)
cell_attributes_dict = read_cell_attribute_selection(weights_path, destination,
namespaces=[initial_weights_namespace],
selection=[gid])
if initial_weights_namespace in cell_attributes_dict:
initial_weights_iter = cell_attributes_dict[initial_weights_namespace]
initial_weights_dict = { gid: attr_dict for gid, attr_dict in initial_weights_iter }
else:
raise RuntimeError('Initial weights namespace %s was not found in file %s' % (initial_weights_namespace, weights_path))
logger.info('Rank %i; destination: %s; read synaptic weights for %i cells' %
(env.comm.rank, destination, len(initial_weights_dict)))
features_attr_names = ['Num Fields', 'Field Width', 'Peak Rate', 'X Offset', 'Y Offset', 'Arena Rate Map']
local_random = np.random.RandomState()
seed_offset = int(env.model_config['Random Seeds']['GC Structured Weights'])
local_random.seed(int(gid + seed_offset))
spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm
arena = env.stimulus_config['Arena'][arena_id]
default_run_vel = arena.properties['default run velocity'] # cm/s
x, y = stimulus.get_2D_arena_spatial_mesh(arena, spatial_resolution)
plasticity_kernel = lambda x, y, x_loc, y_loc, sx, sy: gauss2d(x-x_loc, y-y_loc, sx=sx, sy=sy)
plasticity_kernel = np.vectorize(plasticity_kernel, excluded=[2,3,4,5])
dst_input_features = defaultdict(dict)
num_fields = len(coordinates)
this_field_width = np.array([field_width]*num_fields, dtype=np.float32)
this_peak_rate = np.array([peak_rate]*num_fields, dtype=np.float32)
this_x0 = np.array([x for x, y in coordinates], dtype=np.float32)
this_y0 = np.array([y for x, y in coordinates], dtype=np.float32)
this_rate_map = np.asarray(get_rate_map(this_x0, this_y0, this_field_width, this_peak_rate, x, y),
dtype=np.float32)
selectivity_type = env.selectivity_types['place']
dst_input_features[destination][gid] = {
'Selectivity Type': np.array([selectivity_type], dtype=np.uint8),
'Num Fields': np.array([num_fields], dtype=np.uint8),
'Field Width': this_field_width,
'Peak Rate': this_peak_rate,
'X Offset': this_x0,
'Y Offset': this_y0,
'Arena Rate Map': this_rate_map.ravel() }
selection=[gid]
structured_weights_dict = {}
source_syn_dict = defaultdict(lambda: defaultdict(list))
syn_weight_dict = {}
if weights_path is not None:
initial_weights_iter = read_cell_attribute_selection(weights_path, destination,
namespace=initial_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(initial_weights_iter)
syn_ids = syn_weight_attr_dict[gid]['syn_id']
weights = syn_weight_attr_dict[gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
syn_weight_dict[int(syn_id)] = float(weight)
logger.info('destination: %s; gid %i; received synaptic weights for %i synapses' %
(destination, gid, len(syn_weight_dict)))
(graph, edge_attr_info) = read_graph_selection(file_name=connections_path,
selection=[gid],
namespaces=['Synapses'])
syn_id_attr_index = None
for source, edge_iter in viewitems(graph[destination]):
this_edge_attr_info = edge_attr_info[destination][source]
if 'Synapses' in this_edge_attr_info and \
'syn_id' in this_edge_attr_info['Synapses']:
syn_id_attr_index = this_edge_attr_info['Synapses']['syn_id']
for (destination_gid, edges) in edge_iter:
assert destination_gid == gid
source_gids, edge_attrs = edges
syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
this_source_syn_dict = source_syn_dict[source]
count = 0
for i in range(len(source_gids)):
this_source_gid = source_gids[i]
this_syn_id = syn_ids[i]
this_syn_wgt = syn_weight_dict.get(this_syn_id, 0.0)
this_source_syn_dict[this_source_gid].append((this_syn_id, this_syn_wgt))
count += 1
logger.info('destination: %s; gid %i; %d synaptic weights from source population %s' %
(destination, gid, count, source))
src_input_features = defaultdict(dict)
for source in sources:
source_gids = list(source_syn_dict[source].keys())
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(input_features_path, source,
namespace=input_features_namespace,
mask=set(features_attr_names),
selection=source_gids)
this_src_input_features = src_input_features[source]
count = 0
for gid, attr_dict in input_features_iter:
this_src_input_features[gid] = attr_dict
count += 1
logger.info('Read %s feature data for %i cells in population %s' % (input_features_namespace, count, source))
this_syn_weights = \
synapses.generate_structured_weights(destination_gid,
destination,
synapse_name,
sources,
dst_input_features,
src_input_features,
source_syn_dict,
spatial_mesh=(x,y),
plasticity_kernel=plasticity_kernel,
field_width_scale=field_width_scale,
baseline_weight=baseline_weight,
local_random=local_random,
interactive=interactive)
assert this_syn_weights is not None
structured_weights_dict[destination_gid] = this_syn_weights
logger.info('destination: %s; gid %i; generated structured weights for %i inputs'
% (destination, destination_gid, len(this_syn_weights['syn_id'])))
gc.collect()
if not dry_run:
logger.info('Destination: %s; appending structured weights...' % (destination))
this_structured_weights_namespace = '%s %s' % (structured_weights_namespace, arena_id)
append_cell_attributes(output_weights_path, destination, structured_weights_dict,
namespace=this_structured_weights_namespace)
logger.info('Destination: %s; appended structured weights' % (destination))
structured_weights_dict.clear()
if output_features_path is not None:
output_features_namespace = 'Place Selectivity %s' % arena_id
cell_attr_dict = dst_input_features[destination]
logger.info('Destination: %s; appending features...' % (destination))
append_cell_attributes(output_features_path, destination,
cell_attr_dict, namespace=output_features_namespace)
gc.collect()
del(syn_weight_dict)
del(src_input_features)
del(dst_input_features)