def main(population, forest_path, forest_measurement_namespace, attr_name,
selection_path):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
population_ranges = read_population_ranges(forest_path)[0]
selection = []
f = open(selection_path, 'r')
for line in f.readlines():
selection.append(int(line))
f.close()
columns = [attr_name]
df_dict = {}
it = read_cell_attribute_selection(forest_path,
population,
namespace=forest_measurement_namespace,
selection=selection)
for cell_gid, meas_dict in it:
cell_attr = meas_dict[attr_name]
df_dict[cell_gid] = [np.sum(cell_attr)]
df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns)
df = df.reindex(selection)
df.to_csv('tree.%s.%s.csv' % (attr_name, population))
def main(template_path, forest_path, synapses_path, connections_path, config_path):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
env = Env(comm=comm, config_file=config_path, template_paths=template_path)
h('objref nil, pc, tlog, Vlog, spikelog')
h.load_file("nrngui.hoc")
h.xopen ("./tests/rn.hoc")
h.xopen(template_path+'/HICAPCell.hoc')
pop_name = "HCC"
gid = 1043250
(trees_dict,_) = read_tree_selection (forest_path, pop_name, [gid], comm=env.comm)
(_, tree) = next(trees_dict)
v_init = -67
template_class = getattr(h, "HICAPCell")
passive_test(template_class, tree, v_init)
ap_test(template_class, tree, v_init)
ap_rate_test(template_class, tree, v_init)
fi_test(template_class, tree, v_init)
if synapses_path and connections_path:
synapses_iter = read_cell_attribute_selection (synapses_path, pop_name, [gid],
"Synapse Attributes", comm=env.comm)
(_, synapses_dict) = next(synapses_iter)
connections = read_graph_selection(file_name=connections_path, selection=[gid],
namespaces=['Synapses', 'Connections'], comm=env.comm)
synapse_test(template_class, gid, tree, synapses_dict, connections, v_init, env)
def main(population, features_path, features_namespace, selection_path):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
population_ranges = read_population_ranges(features_path)[0]
soma_coords = {}
selection = []
f = open(selection_path, 'r')
for line in f.readlines():
selection.append(int(line))
f.close()
columns = ['Field Width', 'X Offset', 'Y Offset']
df_dict = {}
it = read_cell_attribute_selection(features_path, population,
namespace=features_namespace,
selection=selection)
for cell_gid, features_dict in it:
cell_field_width = features_dict['Field Width'][0]
cell_xoffset = features_dict['X Offset'][0]
cell_yoffset = features_dict['Y Offset'][0]
df_dict[cell_gid] = [cell_field_width, cell_xoffset, cell_yoffset]
df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns)
df = df.reindex(selection)
df.to_csv('features.%s.csv' % population)
def read_target_rate_vector(context, eps=1e-2):
"""
"""
target_rate_map_arena = context.init_params['target_rate_map_arena']
target_rate_map_trajectory = context.init_params['target_rate_map_trajectory']
target_rate_map_path = context.init_params['target_rate_map_path']
target_rate_map_namespace = context.init_params.get('target_rate_map_namespace', 'Input Spikes')
trj_x, trj_y, trj_d, trj_t = stimulus.read_trajectory(target_rate_map_path, target_rate_map_arena, target_rate_map_trajectory)
time_range = (0., min(np.max(trj_t), context.init_params['tstop']))
time_step = context.env.stimulus_config['Temporal Resolution']
context.time_bins = np.arange(time_range[0], time_range[1], time_step)
context.state_time_bins = np.arange(time_range[0], time_range[1], time_step)[:-1]
input_namespace = '%s %s %s' % (target_rate_map_namespace, target_rate_map_arena, target_rate_map_trajectory)
it = read_cell_attribute_selection(target_rate_map_path, context.population, namespace=input_namespace,
selection=[context.gid], mask=set(['Trajectory Rate Map']),
comm=context.comm)
_, attr_dict = next(it)
trj_rate_map = attr_dict['Trajectory Rate Map']
target_rate_vector = np.interp(context.state_time_bins, trj_t, trj_rate_map)
target_rate_vector[np.abs(target_rate_vector) < eps] = 0.
return target_rate_vector
def main(template_path, forest_path, synapses_path, config_path):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
env = Env(comm=comm, config_file=config_path, template_paths=template_path)
h('objref nil, pc, tlog, Vlog, spikelog')
h.load_file("nrngui.hoc")
h.xopen("./tests/rn.hoc")
h.xopen(template_path + '/BasketCell.hoc')
pop_name = "BC"
gid = 1039000
(trees_dict, _) = read_tree_selection(forest_path,
pop_name, [gid],
comm=env.comm)
synapses_dict = read_cell_attribute_selection(synapses_path,
pop_name, [gid],
"Synapse Attributes",
comm=env.comm)
(_, tree) = next(trees_dict)
(_, synapses) = next(synapses_dict)
v_init = -60
template_class = getattr(h, "BasketCell")
ap_test(template_class, tree, v_init)
passive_test(template_class, tree, v_init)
ap_rate_test(template_class, tree, v_init)
fi_test(template_class, tree, v_init)
gap_junction_test(env, template_class, tree, v_init)
synapse_test(template_class, gid, tree, synapses, v_init, env)
def main(population, coords_path, coords_namespace, distances_namespace,
selection_path):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
population_ranges = read_population_ranges(coords_path)[0]
soma_coords = {}
selection = []
f = open(selection_path, 'r')
for line in f.readlines():
selection.append(int(line))
f.close()
columns = ['U', 'V', 'L']
df_dict = {}
it = read_cell_attribute_selection(coords_path,
population,
namespace=coords_namespace,
selection=selection)
for cell_gid, coords_dict in it:
cell_u = coords_dict['U Coordinate'][0]
cell_v = coords_dict['V Coordinate'][0]
cell_l = coords_dict['L Coordinate'][0]
df_dict[cell_gid] = [cell_u, cell_v, cell_l]
if distances_namespace is not None:
columns.extend(['U Distance', 'V Distance'])
it = read_cell_attribute_selection(coords_path,
population,
namespace=distances_namespace,
selection=selection)
for cell_gid, distances_dict in it:
cell_ud = distances_dict['U Distance'][0]
cell_vd = distances_dict['V Distance'][0]
df_dict[cell_gid].extend([cell_ud, cell_vd])
df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns)
df = df.reindex(selection)
df.to_csv('coords.%s.csv' % population)
def main(config_path, template_paths, forest_path, synapses_path):
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
env = Env(comm=comm,
config_file=config_path,
template_paths=template_paths)
neuron_utils.configure_hoc_env(env)
h.pc = h.ParallelContext()
v_init = -65.0
popName = "MC"
gid = 1000000
env.load_cell_template(popName)
(trees, _) = read_tree_selection(forest_path, popName, [gid], comm=comm)
if synapses_path is not None:
synapses_iter = read_cell_attribute_selection (synapses_path, popName, [gid], \
"Synapse Attributes", comm=comm)
else:
synapses_iter = None
gid, tree = next(trees)
if synapses_iter is not None:
(_, synapses) = next(synapses_iter)
else:
synapses = None
if 'mech_file' in env.celltypes[popName]:
mech_file_path = env.config_prefix + '/' + env.celltypes[popName][
'mech_file']
else:
mech_file_path = None
template_class = getattr(h, "MossyCell")
if (synapses is not None):
synapse_test(template_class, mech_file_path, gid, tree, synapses,
v_init, env)
def main(arena_id, config, config_prefix, dataset_prefix, distances_namespace, spike_input_path, spike_input_namespace, spike_input_attr, input_features_namespaces, input_features_path, selection_path, output_path, io_size, trajectory_id, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
env = Env(comm=comm, config_file=config,
config_prefix=config_prefix, dataset_prefix=dataset_prefix,
results_path=output_path, spike_input_path=spike_input_path,
spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr,
arena_id=arena_id, trajectory_id=trajectory_id, io_size=io_size)
selection = []
f = open(selection_path, 'r')
for line in f.readlines():
selection.append(int(line))
f.close()
selection = set(selection)
pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm)
distance_U_dict = {}
distance_V_dict = {}
range_U_dict = {}
range_V_dict = {}
selection_dict = defaultdict(set)
comm0 = env.comm.Split(2 if rank == 0 else 0, 0)
if rank == 0:
for population in pop_ranges:
distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0)
soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k,v) in distances }
del distances
numitems = len(list(soma_distances.keys()))
if numitems == 0:
continue
distance_U_array = np.asarray([soma_distances[gid][0] for gid in soma_distances])
distance_V_array = np.asarray([soma_distances[gid][1] for gid in soma_distances])
U_min = np.min(distance_U_array)
U_max = np.max(distance_U_array)
V_min = np.min(distance_V_array)
V_max = np.max(distance_V_array)
range_U_dict[population] = (U_min, U_max)
range_V_dict[population] = (V_min, V_max)
distance_U = { gid: soma_distances[gid][0] for gid in soma_distances }
distance_V = { gid: soma_distances[gid][1] for gid in soma_distances }
distance_U_dict[population] = distance_U
distance_V_dict[population] = distance_V
min_dist = U_min
max_dist = U_max
selection_dict[population] = set([ k for k in distance_U if k in selection ])
env.comm.barrier()
write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName)
if rank == 0:
io_utils.mkout(env, write_selection_file_path)
env.comm.barrier()
selection_dict = env.comm.bcast(dict(selection_dict), root=0)
env.cell_selection = selection_dict
io_utils.write_cell_selection(env, write_selection_file_path)
input_selection = io_utils.write_connection_selection(env, write_selection_file_path)
if spike_input_path:
io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path)
if input_features_path:
for this_input_features_namespace in sorted(input_features_namespaces):
for population in sorted(input_selection):
logger.info(f"Extracting input features {this_input_features_namespace} for population {population}...")
it = read_cell_attribute_selection(input_features_path, population,
namespace=f"{this_input_features_namespace} {arena_id}",
selection=input_selection[population], comm=env.comm)
output_features_dict = { cell_gid : cell_features_dict for cell_gid, cell_features_dict in it }
append_cell_attributes(write_selection_file_path, population, output_features_dict,
namespace=f"{this_input_features_namespace} {arena_id}",
io_size=io_size, comm=env.comm)
env.comm.barrier()
def main(config, coordinates, gid, field_width, peak_rate, input_features_path,
input_features_namespaces, output_features_namespace,
output_weights_path, output_features_path, initial_weights_path,
reference_weights_path, h5types_path, synapse_name,
initial_weights_namespace, reference_weights_namespace,
output_weights_namespace, reference_weights_are_delta,
connections_path, optimize_method, destination, sources, arena_id,
max_delta_weight, field_width_scale, max_iter, verbose, dry_run,
plot):
"""
:param config: str (path to .yaml file)
:param coordinates: tuple of float
:param gid: int
:param field_width: float
:param peak_rate: float
:param input_features_path: str (path to .h5 file)
:param input_features_namespaces: str
:param output_features_namespace: str
:param output_weights_path: str (path to .h5 file)
:param output_features_path: str (path to .h5 file)
:param initial_weights_path: str (path to .h5 file)
:param reference_weights_path: str (path to .h5 file)
:param h5types_path: str (path to .h5 file)
:param synapse_name: str
:param initial_weights_namespace: str
:param output_weights_namespace: str
:param reference_weights_are_delta: bool
:param connections_path: str (path to .h5 file)
:param destination: str (population name)
:param sources: list of str (population name)
:param arena_id: str
:param max_delta_weight: float
:param field_width_scale: float
:param max_iter: int
:param verbose: bool
:param dry_run: bool
:param interactive: bool
:param plot: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
env = Env(config_file=config)
if not dry_run:
if output_weights_path is None:
raise RuntimeError(
'Missing required argument: output_weights_path.')
if not os.path.isfile(output_weights_path):
if initial_weights_path is not None and os.path.isfile(
initial_weights_path):
input_file_path = initial_weights_path
elif h5types_path is not None and os.path.isfile(h5types_path):
input_file_path = h5types_path
else:
raise RuntimeError(
'Missing required source for h5types: either an initial_weights_path or an '
'h5types_path must be provided.')
with h5py.File(output_weights_path, 'a') as output_file:
with h5py.File(input_file_path, 'r') as input_file:
input_file.copy('/H5Types', output_file)
this_input_features_namespaces = [
'%s %s' % (input_features_namespace, arena_id)
for input_features_namespace in input_features_namespaces
]
features_attr_names = ['Arena Rate Map']
spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm
arena = env.stimulus_config['Arena'][arena_id]
default_run_vel = arena.properties['default run velocity'] # cm/s
arena_x, arena_y = stimulus.get_2D_arena_spatial_mesh(
arena, spatial_resolution)
dim_x = len(arena_x)
dim_y = len(arena_y)
if gid is None:
target_gids = []
else:
target_gids = [gid]
dst_input_features = defaultdict(dict)
num_fields = len(coordinates)
this_field_width = np.array([field_width] * num_fields, dtype=np.float32)
this_scaled_field_width = np.array([field_width * field_width_scale] *
num_fields,
dtype=np.float32)
this_peak_rate = np.array([peak_rate] * num_fields, dtype=np.float32)
this_x0 = np.array([x for x, y in coordinates], dtype=np.float32)
this_y0 = np.array([y for x, y in coordinates], dtype=np.float32)
this_rate_map = np.asarray(get_rate_map(this_x0, this_y0, this_field_width,
this_peak_rate, arena_x, arena_y),
dtype=np.float32)
target_map = np.asarray(get_rate_map(this_x0, this_y0,
this_scaled_field_width,
this_peak_rate, arena_x, arena_y),
dtype=np.float32)
selectivity_type = env.selectivity_types['place']
dst_input_features[destination][target_gid] = {
'Selectivity Type': np.array([selectivity_type], dtype=np.uint8),
'Num Fields': np.array([num_fields], dtype=np.uint8),
'Field Width': this_field_width,
'Peak Rate': this_peak_rate,
'X Offset': this_x0,
'Y Offset': this_y0,
'Arena Rate Map': this_rate_map.ravel()
}
initial_weights_by_syn_id_dict = dict()
selection = [target_gid]
if initial_weights_path is not None:
initial_weights_iter = \
read_cell_attribute_selection(initial_weights_path, destination, namespace=initial_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(initial_weights_iter)
syn_ids = syn_weight_attr_dict[target_gid]['syn_id']
weights = syn_weight_attr_dict[target_gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
initial_weights_by_syn_id_dict[int(syn_id)] = float(weight)
logger.info(
'destination: %s; gid %i; read initial synaptic weights for %i synapses'
% (destination, target_gid, len(initial_weights_by_syn_id_dict)))
reference_weights_by_syn_id_dict = None
if reference_weights_path is not None:
reference_weights_by_syn_id_dict = dict()
reference_weights_iter = \
read_cell_attribute_selection(reference_weights_path, destination, namespace=reference_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(reference_weights_iter)
syn_ids = syn_weight_attr_dict[target_gid]['syn_id']
weights = syn_weight_attr_dict[target_gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
reference_weights_by_syn_id_dict[int(syn_id)] = float(weight)
logger.info(
'destination: %s; gid %i; read reference synaptic weights for %i synapses'
% (destination, target_gid, len(reference_weights_by_syn_id_dict)))
source_gid_set_dict = defaultdict(set)
syn_ids_by_source_gid_dict = defaultdict(list)
initial_weights_by_source_gid_dict = dict()
if reference_weights_by_syn_id_dict is None:
reference_weights_by_source_gid_dict = None
else:
reference_weights_by_source_gid_dict = dict()
(graph, edge_attr_info) = read_graph_selection(file_name=connections_path,
selection=[target_gid],
namespaces=['Synapses'])
syn_id_attr_index = None
for source, edge_iter in viewitems(graph[destination]):
if source not in sources:
continue
this_edge_attr_info = edge_attr_info[destination][source]
if 'Synapses' in this_edge_attr_info and \
'syn_id' in this_edge_attr_info['Synapses']:
syn_id_attr_index = this_edge_attr_info['Synapses']['syn_id']
for (destination_gid, edges) in edge_iter:
assert destination_gid == target_gid
source_gids, edge_attrs = edges
syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
count = 0
for i in range(len(source_gids)):
this_source_gid = int(source_gids[i])
source_gid_set_dict[source].add(this_source_gid)
this_syn_id = int(syn_ids[i])
if this_syn_id not in initial_weights_by_syn_id_dict:
this_weight = \
env.connection_config[destination][source].mechanisms['default'][synapse_name]['weight']
initial_weights_by_syn_id_dict[this_syn_id] = this_weight
syn_ids_by_source_gid_dict[this_source_gid].append(this_syn_id)
if this_source_gid not in initial_weights_by_source_gid_dict:
initial_weights_by_source_gid_dict[this_source_gid] = \
initial_weights_by_syn_id_dict[this_syn_id]
if reference_weights_by_source_gid_dict is not None:
reference_weights_by_source_gid_dict[this_source_gid] = \
reference_weights_by_syn_id_dict[this_syn_id]
count += 1
logger.info(
'destination: %s; gid %i; set initial synaptic weights for %d inputs from source population '
'%s' % (destination, destination_gid, count, source))
syn_count_by_source_gid_dict = dict()
for source_gid in syn_ids_by_source_gid_dict:
syn_count_by_source_gid_dict[source_gid] = len(
syn_ids_by_source_gid_dict[source_gid])
input_rate_maps_by_source_gid_dict = dict()
for source in sources:
source_gids = list(source_gid_set_dict[source])
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(
input_features_path,
source,
namespace=input_features_namespace,
mask=set(features_attr_names),
selection=source_gids)
count = 0
for gid, attr_dict in input_features_iter:
input_rate_maps_by_source_gid_dict[gid] = attr_dict[
'Arena Rate Map'].reshape((dim_x, dim_y))
count += 1
logger.info('Read %s feature data for %i cells in population %s' %
(input_features_namespace, count, source))
if is_interactive:
context.update(locals())
normalized_delta_weights_dict, arena_LS_map = \
synapses.generate_structured_weights(target_map=target_map,
initial_weight_dict=initial_weights_by_source_gid_dict,
input_rate_map_dict=input_rate_maps_by_source_gid_dict,
syn_count_dict=syn_count_by_source_gid_dict,
max_delta_weight=max_delta_weight, arena_x=arena_x, arena_y=arena_y,
reference_weight_dict=reference_weights_by_source_gid_dict,
reference_weights_are_delta=reference_weights_are_delta,
reference_weights_namespace=reference_weights_namespace,
optimize_method=optimize_method, verbose=verbose, plot=plot)
output_syn_ids = np.empty(len(initial_weights_by_syn_id_dict),
dtype='uint32')
output_weights = np.empty(len(initial_weights_by_syn_id_dict),
dtype='float32')
i = 0
for source_gid, this_weight in viewitems(normalized_delta_weights_dict):
for syn_id in syn_ids_by_source_gid_dict[source_gid]:
output_syn_ids[i] = syn_id
output_weights[i] = this_weight
i += 1
output_weights_dict = {
target_gid: {
'syn_id': output_syn_ids,
synapse_name: output_weights
}
}
logger.info('destination: %s; gid %i; generated %s for %i synapses' %
(destination, target_gid, output_weights_namespace,
len(output_weights)))
if not dry_run:
this_output_weights_namespace = '%s %s' % (output_weights_namespace,
arena_id)
logger.info('Destination: %s; appending %s ...' %
(destination, this_output_weights_namespace))
append_cell_attributes(output_weights_path,
destination,
output_weights_dict,
namespace=this_output_weights_namespace)
logger.info('Destination: %s; appended %s' %
(destination, this_output_weights_namespace))
output_weights_dict.clear()
if output_features_path is not None:
this_output_features_namespace = '%s %s' % (
output_features_namespace, arena_id)
cell_attr_dict = dst_input_features[destination]
cell_attr_dict[target_gid]['Arena State Map'] = np.asarray(
arena_LS_map.ravel(), dtype=np.float32)
logger.info('Destination: %s; appending %s ...' %
(destination, this_output_features_namespace))
append_cell_attributes(output_features_path,
destination,
cell_attr_dict,
namespace=this_output_features_namespace)
if is_interactive:
context.update(locals())
def read_state(input_file,
population_names,
namespace_id,
time_variable='t',
state_variable='v',
time_range=None,
max_units=None,
gid=None,
comm=None,
n_trials=-1):
if comm is None:
comm = MPI.COMM_WORLD
pop_state_dict = {}
logger.info(
'Reading state data from populations %s, namespace %s gid = %s...' %
(str(population_names), namespace_id, str(gid)))
attr_info_dict = read_cell_attribute_info(input_file,
populations=population_names,
read_cell_index=True)
for pop_name in population_names:
cell_index = None
pop_state_dict[pop_name] = {}
for attr_name, attr_cell_index in attr_info_dict[pop_name][
namespace_id]:
if state_variable == attr_name:
cell_index = attr_cell_index
if cell_index is None:
raise RuntimeError(
'read_state: Unable to find recordings for state variable %s in population %s namespace %s'
% (state_variable, pop_name, str(namespace_id)))
cell_set = set(cell_index)
# Limit to max_units
if gid is None:
if (max_units is not None) and (len(cell_set) > max_units):
logger.info(
' Reading only randomly sampled %i out of %i units for population %s'
% (max_units, len(cell_set), pop_name))
sample_inds = np.random.randint(0,
len(cell_set) - 1,
size=int(max_units))
cell_set_lst = list(cell_set)
gid_set = set([cell_set_lst[i] for i in sample_inds])
else:
gid_set = cell_set
else:
gid_set = set(gid)
state_dict = {}
if gid is None:
valiter = read_cell_attributes(input_file,
pop_name,
namespace=namespace_id,
comm=comm)
else:
valiter = read_cell_attribute_selection(input_file,
pop_name,
namespace=namespace_id,
selection=list(gid_set),
comm=comm)
if time_range is None:
for cellind, vals in valiter:
if cellind is not None:
trial_dur = vals.get('trial duration', None)
distance = vals.get('distance', [None])[0]
section = vals.get('section', [None])[0]
loc = vals.get('loc', [None])[0]
tvals = np.asarray(vals[time_variable], dtype=np.float32)
svals = np.asarray(vals[state_variable], dtype=np.float32)
trial_bounds = list(
np.where(np.isclose(tvals, tvals[0], atol=1e-4))[0])
n_trial_bounds = len(trial_bounds)
trial_bounds.append(len(tvals))
if n_trials == -1:
this_n_trials = n_trial_bounds
else:
this_n_trials = min(n_trial_bounds, n_trials)
if this_n_trials > 1:
state_dict[cellind] = (np.split(
tvals, trial_bounds[1:n_trials]),
np.split(
svals,
trial_bounds[1:n_trials]),
distance, section, loc)
else:
state_dict[cellind] = ([tvals[:trial_bounds[1]]
], [svals[:trial_bounds[1]]],
distance, section, loc)
else:
for cellind, vals in valiter:
if cellind is not None:
distance = vals.get('distance', [None])[0]
section = vals.get('section', [None])[0]
loc = vals.get('loc', [None])[0]
tinds = np.argwhere((vals[time_variable] <= time_range[1])
&
(vals[time_variable] >= time_range[0]))
tvals = np.asarray(vals[time_variable][tinds],
dtype=np.float32).reshape((-1, ))
svals = np.asarray(vals[state_variable][tinds],
dtype=np.float32).reshape((-1, ))
trial_bounds = list(
np.where(np.isclose(tvals, tvals[0], atol=1e-4))[0])
n_trial_bounds = len(trial_bounds)
trial_bounds.append(len(tvals))
if n_trials == -1:
this_n_trials = n_trial_bounds
else:
this_n_trials = min(n_trial_bounds, n_trials)
if this_n_trials > 1:
state_dict[cellind] = (np.split(
tvals, trial_bounds[1:n_trials]),
np.split(
svals,
trial_bounds[1:n_trials]),
distance, section, loc)
else:
state_dict[cellind] = ([tvals[:trial_bounds[1]]
], [svals[:trial_bounds[1]]],
distance, section, loc)
pop_state_dict[pop_name] = state_dict
return {
'states': pop_state_dict,
'time_variable': time_variable,
'state_variable': state_variable
}
def main(config_file, config_prefix, input_path, population, template_paths,
dataset_prefix, results_path, results_file_id, results_namespace_id,
v_init, io_size, chunk_size, value_chunk_size, write_size, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
if io_size == -1:
io_size = comm.size
if results_file_id is None:
if rank == 0:
result_file_id = uuid.uuid4()
results_file_id = comm.bcast(results_file_id, root=0)
if results_namespace_id is None:
results_namespace_id = 'Cell Clamp Results'
comm = MPI.COMM_WORLD
np.seterr(all='raise')
verbose = True
params = dict(locals())
env = Env(**params)
configure_hoc_env(env)
if rank == 0:
io_utils.mkout(env, env.results_file_path)
env.comm.barrier()
env.cell_selection = {}
template_class = load_cell_template(env, population)
if input_path is not None:
env.data_file_path = input_path
env.load_celltypes()
synapse_config = env.celltypes[population]['synapses']
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
start_time = time.time()
count = 0
gid_count = 0
attr_dict = {}
if input_path is None:
cell_path = env.data_file_path
connectivity_path = env.connectivity_file_path
else:
cell_path = input_path
connectivity_path = input_path
for gid, morph_dict in NeuroH5TreeGen(cell_path,
population,
io_size=io_size,
comm=env.comm,
topology=True):
local_time = time.time()
if gid is not None:
color = 0
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid: %i' % (rank, gid))
cell_dict = {'morph': morph_dict}
synapses_iter = read_cell_attribute_selection(cell_path,
population, [gid],
'Synapse Attributes',
comm=comm0)
_, synapse_dict = next(synapses_iter)
cell_dict['synapse'] = synapse_dict
if has_weights:
cell_weights_iters = [
read_cell_attribute_selection(cell_path,
population, [gid],
weights_namespace,
comm=comm0)
for weights_namespace in weights_namespaces
]
weight_dict = dict(
zip_longest(weights_namespaces, cell_weights_iters))
cell_dict['weight'] = weight_dict
(graph,
a) = read_graph_selection(file_name=connectivity_path,
selection=[gid],
namespaces=['Synapses', 'Connections'],
comm=comm0)
cell_dict['connectivity'] = (graph, a)
gid_count += 1
attr_dict[gid] = {}
attr_dict[gid].update(
cell_clamp.measure_passive(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap_rate(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_fi(gid,
population,
v_init,
env,
cell_dict=cell_dict))
else:
color = 1
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid is None' % (rank))
comm0.Free()
count += 1
if (results_path is not None) and (count % write_size == 0):
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
attr_dict = {}
env.comm.barrier()
if results_path is not None:
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
global_count = env.comm.gather(gid_count, root=0)
MPI.Finalize()
def write_input_cell_selection(env,
input_sources,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Writes out predefined spike trains when only a subset of the network is instantiated.
:param env: an instance of the `dentate.Env` class
:param input_sources: a dictionary of the form { pop_name, gid_sources }
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
dataset_path = env.dataset_path
input_file_path = env.data_file_path
if populations is None:
pop_names = sorted(env.celltypes.keys())
else:
pop_names = populations
for pop_name, gid_range in sorted(viewitems(input_sources)):
if pop_name not in pop_names:
continue
spikes_output_dict = {}
if (env.cell_selection is not None) and (pop_name
in env.cell_selection):
local_gid_range = gid_range.difference(
set(env.cell_selection[pop_name]))
else:
local_gid_range = gid_range
gid_ranges = env.comm.allgather(local_gid_range)
this_gid_range = set([])
for gid_range in gid_ranges:
for gid in gid_range:
if gid % nhosts == rank:
this_gid_range.add(gid)
has_spike_train = False
spike_input_source_loc = []
if (env.spike_input_attribute_info is not None) and (env.spike_input_ns
is not None):
if (pop_name in env.spike_input_attribute_info) and \
(env.spike_input_ns in env.spike_input_attribute_info[pop_name]):
has_spike_train = True
spike_input_source_loc.append(
(env.spike_input_path, env.spike_input_ns))
if (env.cell_attribute_info is not None) and (env.spike_input_ns
is not None):
if (pop_name in env.cell_attribute_info) and \
(env.spike_input_ns in env.cell_attribute_info[pop_name]):
has_spike_train = True
spike_input_source_loc.append(
(input_file_path, env.spike_input_ns))
if rank == 0:
logger.info(
'*** Reading spike trains for population %s: %d cells: has_spike_train = %s'
% (pop_name, len(this_gid_range), str(has_spike_train)))
if has_spike_train:
vecstim_attr_set = set(['t'])
if env.spike_input_attr is not None:
vecstim_attr_set.add(env.spike_input_attr)
if 'spike train' in env.celltypes[pop_name]:
vecstim_attr_set.add(
env.celltypes[pop_name]['spike train']['attribute'])
cell_spikes_iters = [ read_cell_attribute_selection(input_path, pop_name, \
list(this_gid_range), \
namespace=input_ns, \
mask=vecstim_attr_set, \
comm=env.comm) for (input_path, input_ns) in spike_input_source_loc ]
for cell_spikes_iter in cell_spikes_iters:
spikes_output_dict.update(dict(list(cell_spikes_iter)))
if rank == 0:
logger.info('*** Writing spike trains for population %s: %s' %
(pop_name, str(spikes_output_dict)))
write_cell_attributes(write_selection_file_path, pop_name, spikes_output_dict, \
namespace=env.spike_input_ns, **write_kwds)
def write_connection_selection(env,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Loads NeuroH5 connectivity file, and writes the corresponding
synapse and network connection mechanisms for the selected postsynaptic cells.
:param env: an instance of the `dentate.Env` class
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
connectivity_file_path = env.connectivity_file_path
forest_file_path = env.forest_file_path
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
syn_attrs = env.synapse_attributes
if populations is None:
pop_names = sorted(env.cell_selection.keys())
else:
pop_names = populations
input_sources = {pop_name: set([]) for pop_name in env.celltypes}
for (postsyn_name, presyn_names) in sorted(viewitems(env.projection_dict)):
if rank == 0:
logger.info('*** Writing connection selection of population %s' %
(postsyn_name))
if postsyn_name not in pop_names:
continue
gid_range = [
gid for gid in env.cell_selection[postsyn_name]
if gid % nhosts == rank
]
synapse_config = env.celltypes[postsyn_name]['synapses']
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
else:
has_weights = False
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(
synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(
synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
if rank == 0:
logger.info('*** Reading synaptic attributes of population %s' %
(postsyn_name))
syn_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace='Synapse Attributes',
comm=env.comm)
syn_attributes_output_dict = dict(list(syn_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
syn_attributes_output_dict,
namespace='Synapse Attributes',
**write_kwds)
del syn_attributes_output_dict
del syn_attributes_iter
if has_weights:
for weights_namespace in sorted(weights_namespaces):
weight_attributes_iter = read_cell_attribute_selection(
forest_file_path,
postsyn_name,
selection=gid_range,
namespace=weights_namespace,
comm=env.comm)
weight_attributes_output_dict = dict(
list(weight_attributes_iter))
write_cell_attributes(write_selection_file_path,
postsyn_name,
weight_attributes_output_dict,
namespace=weights_namespace,
**write_kwds)
del weight_attributes_output_dict
del weight_attributes_iter
logger.info(
'*** Rank %i: reading connectivity selection from file %s for postsynaptic population: %s: selection: %s'
% (rank, connectivity_file_path, postsyn_name, str(gid_range)))
(graph, attr_info) = read_graph_selection(connectivity_file_path, selection=gid_range, \
projections=[ (presyn_name, postsyn_name) for presyn_name in sorted(presyn_names) ], \
comm=env.comm, namespaces=['Synapses', 'Connections'])
for presyn_name in sorted(presyn_names):
gid_dict = {}
edge_count = 0
node_count = 0
if postsyn_name in graph:
if postsyn_name in attr_info and presyn_name in attr_info[
postsyn_name]:
edge_attr_info = attr_info[postsyn_name][presyn_name]
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
if 'Synapses' in edge_attr_info and \
'syn_id' in edge_attr_info['Synapses'] and \
'Connections' in edge_attr_info and \
'distance' in edge_attr_info['Connections']:
syn_id_attr_index = edge_attr_info['Synapses']['syn_id']
distance_attr_index = edge_attr_info['Connections'][
'distance']
else:
raise RuntimeError('write_connection_selection: missing edge attributes for projection %s -> %s' % \
(presyn_name, postsyn_name))
edge_iter = compose_iter(lambda edgeset: input_sources[presyn_name].update(edgeset[1][0]), \
graph[postsyn_name][presyn_name])
for (postsyn_gid, edges) in edge_iter:
presyn_gids, edge_attrs = edges
edge_syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
edge_dists = edge_attrs['Connections'][distance_attr_index]
gid_dict[postsyn_gid] = (presyn_gids, {
'Synapses': {
'syn_id': edge_syn_ids
},
'Connections': {
'distance': edge_dists
}
})
edge_count += len(presyn_gids)
node_count += 1
logger.info(
'*** Rank %d: Writing projection %s -> %s selection: %d nodes, %d edges'
% (rank, presyn_name, postsyn_name, node_count, edge_count))
write_graph(write_selection_file_path, \
src_pop_name=presyn_name, dst_pop_name=postsyn_name, \
edges=gid_dict, comm=env.comm, io_size=env.io_size)
env.comm.barrier()
return input_sources
def write_cell_selection(env,
write_selection_file_path,
populations=None,
write_kwds={}):
"""
Writes out the data necessary to instantiate the selected cells.
:param env: an instance of the `dentate.Env` class
"""
if 'comm' not in write_kwds:
write_kwds['comm'] = env.comm
if 'io_size' not in write_kwds:
write_kwds['io_size'] = env.io_size
rank = int(env.comm.Get_rank())
nhosts = int(env.comm.Get_size())
dataset_path = env.dataset_path
data_file_path = env.data_file_path
if populations is None:
pop_names = sorted(env.cell_selection.keys())
else:
pop_names = populations
for pop_name in pop_names:
gid_range = [
gid for gid in env.cell_selection[pop_name] if gid % nhosts == rank
]
trees_output_dict = {}
coords_output_dict = {}
num_cells = 0
if (pop_name in env.cell_attribute_info) and (
'Trees' in env.cell_attribute_info[pop_name]):
if rank == 0:
logger.info("*** Reading trees for population %s" % pop_name)
cell_tree_iter, _ = read_tree_selection(data_file_path, pop_name, selection=gid_range, \
topology=False, comm=env.comm)
if rank == 0:
logger.info("*** Done reading trees for population %s" %
pop_name)
for i, (gid, tree) in enumerate(cell_tree_iter):
trees_output_dict[gid] = tree
num_cells += 1
assert (len(trees_output_dict) == len(gid_range))
elif (pop_name in env.cell_attribute_info) and (
'Coordinates' in env.cell_attribute_info[pop_name]):
if rank == 0:
logger.info("*** Reading coordinates for population %s" %
pop_name)
cell_attributes_iter = read_cell_attribute_selection(data_file_path, pop_name, selection=gid_range, \
namespace='Coordinates', comm=env.comm)
if rank == 0:
logger.info("*** Done reading coordinates for population %s" %
pop_name)
for i, (gid, coords) in enumerate(cell_attributes_iter):
coords_output_dict[gid] = coords
num_cells += 1
if rank == 0:
logger.info(
"*** Writing cell selection for population %s to file %s" %
(pop_name, write_selection_file_path))
append_cell_trees(write_selection_file_path,
pop_name,
trees_output_dict,
create_index=True,
**write_kwds)
write_cell_attributes(write_selection_file_path,
pop_name,
coords_output_dict,
namespace='Coordinates',
**write_kwds)
def main(config, coordinates, gid, field_width, peak_rate, input_features_path, input_features_namespaces,
output_weights_path, output_features_path, weights_path, h5types_path, synapse_name, initial_weights_namespace,
structured_weights_namespace, connections_path, destination, sources, arena_id, baseline_weight,
field_width_scale, max_iter, verbose, dry_run, interactive):
"""
:param config: str (path to .yaml file)
:param weights_path: str (path to .h5 file)
:param initial_weights_namespace: str
:param structured_weights_namespace: str
:param connections_path: str (path to .h5 file)
:param destination: str
:param sources: list of str
:param verbose:
:param dry_run:
:return:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
env = Env(config_file=config)
if output_weights_path is None:
if weights_path is None:
raise RuntimeError('Output weights path must be specified when weights path is not specified.')
output_weights_path = weights_path
if (not dry_run):
if not os.path.isfile(output_weights_path):
if weights_path is not None:
input_file = h5py.File(weights_path,'r')
elif h5types_path is not None:
input_file = h5py.File(h5types_path,'r')
else:
raise RuntimeError('h5types input path must be specified when weights path is not specified.')
output_file = h5py.File(output_weights_path,'w')
input_file.copy('/H5Types',output_file)
input_file.close()
output_file.close()
this_input_features_namespaces = ['%s %s' % (input_features_namespace, arena_id) for input_features_namespace in input_features_namespaces]
initial_weights_dict = None
if weights_path is not None:
logger.info('Reading initial weights data from %s...' % weights_path)
cell_attributes_dict = read_cell_attribute_selection(weights_path, destination,
namespaces=[initial_weights_namespace],
selection=[gid])
if initial_weights_namespace in cell_attributes_dict:
initial_weights_iter = cell_attributes_dict[initial_weights_namespace]
initial_weights_dict = { gid: attr_dict for gid, attr_dict in initial_weights_iter }
else:
raise RuntimeError('Initial weights namespace %s was not found in file %s' % (initial_weights_namespace, weights_path))
logger.info('Rank %i; destination: %s; read synaptic weights for %i cells' %
(env.comm.rank, destination, len(initial_weights_dict)))
features_attr_names = ['Num Fields', 'Field Width', 'Peak Rate', 'X Offset', 'Y Offset', 'Arena Rate Map']
local_random = np.random.RandomState()
seed_offset = int(env.model_config['Random Seeds']['GC Structured Weights'])
local_random.seed(int(gid + seed_offset))
spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm
arena = env.stimulus_config['Arena'][arena_id]
default_run_vel = arena.properties['default run velocity'] # cm/s
x, y = stimulus.get_2D_arena_spatial_mesh(arena, spatial_resolution)
plasticity_kernel = lambda x, y, x_loc, y_loc, sx, sy: gauss2d(x-x_loc, y-y_loc, sx=sx, sy=sy)
plasticity_kernel = np.vectorize(plasticity_kernel, excluded=[2,3,4,5])
dst_input_features = defaultdict(dict)
num_fields = len(coordinates)
this_field_width = np.array([field_width]*num_fields, dtype=np.float32)
this_peak_rate = np.array([peak_rate]*num_fields, dtype=np.float32)
this_x0 = np.array([x for x, y in coordinates], dtype=np.float32)
this_y0 = np.array([y for x, y in coordinates], dtype=np.float32)
this_rate_map = np.asarray(get_rate_map(this_x0, this_y0, this_field_width, this_peak_rate, x, y),
dtype=np.float32)
selectivity_type = env.selectivity_types['place']
dst_input_features[destination][gid] = {
'Selectivity Type': np.array([selectivity_type], dtype=np.uint8),
'Num Fields': np.array([num_fields], dtype=np.uint8),
'Field Width': this_field_width,
'Peak Rate': this_peak_rate,
'X Offset': this_x0,
'Y Offset': this_y0,
'Arena Rate Map': this_rate_map.ravel() }
selection=[gid]
structured_weights_dict = {}
source_syn_dict = defaultdict(lambda: defaultdict(list))
syn_weight_dict = {}
if weights_path is not None:
initial_weights_iter = read_cell_attribute_selection(weights_path, destination,
namespace=initial_weights_namespace,
selection=selection)
syn_weight_attr_dict = dict(initial_weights_iter)
syn_ids = syn_weight_attr_dict[gid]['syn_id']
weights = syn_weight_attr_dict[gid][synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
syn_weight_dict[int(syn_id)] = float(weight)
logger.info('destination: %s; gid %i; received synaptic weights for %i synapses' %
(destination, gid, len(syn_weight_dict)))
(graph, edge_attr_info) = read_graph_selection(file_name=connections_path,
selection=[gid],
namespaces=['Synapses'])
syn_id_attr_index = None
for source, edge_iter in viewitems(graph[destination]):
this_edge_attr_info = edge_attr_info[destination][source]
if 'Synapses' in this_edge_attr_info and \
'syn_id' in this_edge_attr_info['Synapses']:
syn_id_attr_index = this_edge_attr_info['Synapses']['syn_id']
for (destination_gid, edges) in edge_iter:
assert destination_gid == gid
source_gids, edge_attrs = edges
syn_ids = edge_attrs['Synapses'][syn_id_attr_index]
this_source_syn_dict = source_syn_dict[source]
count = 0
for i in range(len(source_gids)):
this_source_gid = source_gids[i]
this_syn_id = syn_ids[i]
this_syn_wgt = syn_weight_dict.get(this_syn_id, 0.0)
this_source_syn_dict[this_source_gid].append((this_syn_id, this_syn_wgt))
count += 1
logger.info('destination: %s; gid %i; %d synaptic weights from source population %s' %
(destination, gid, count, source))
src_input_features = defaultdict(dict)
for source in sources:
source_gids = list(source_syn_dict[source].keys())
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(input_features_path, source,
namespace=input_features_namespace,
mask=set(features_attr_names),
selection=source_gids)
this_src_input_features = src_input_features[source]
count = 0
for gid, attr_dict in input_features_iter:
this_src_input_features[gid] = attr_dict
count += 1
logger.info('Read %s feature data for %i cells in population %s' % (input_features_namespace, count, source))
this_syn_weights = \
synapses.generate_structured_weights(destination_gid,
destination,
synapse_name,
sources,
dst_input_features,
src_input_features,
source_syn_dict,
spatial_mesh=(x,y),
plasticity_kernel=plasticity_kernel,
field_width_scale=field_width_scale,
baseline_weight=baseline_weight,
local_random=local_random,
interactive=interactive)
assert this_syn_weights is not None
structured_weights_dict[destination_gid] = this_syn_weights
logger.info('destination: %s; gid %i; generated structured weights for %i inputs'
% (destination, destination_gid, len(this_syn_weights['syn_id'])))
gc.collect()
if not dry_run:
logger.info('Destination: %s; appending structured weights...' % (destination))
this_structured_weights_namespace = '%s %s' % (structured_weights_namespace, arena_id)
append_cell_attributes(output_weights_path, destination, structured_weights_dict,
namespace=this_structured_weights_namespace)
logger.info('Destination: %s; appended structured weights' % (destination))
structured_weights_dict.clear()
if output_features_path is not None:
output_features_namespace = 'Place Selectivity %s' % arena_id
cell_attr_dict = dst_input_features[destination]
logger.info('Destination: %s; appending features...' % (destination))
append_cell_attributes(output_features_path, destination,
cell_attr_dict, namespace=output_features_namespace)
gc.collect()
del(syn_weight_dict)
del(src_input_features)
del(dst_input_features)
def main(config, coordinates, field_width, gid, input_features_path,
input_features_namespaces, initial_weights_path,
output_features_namespace, output_features_path, output_weights_path,
reference_weights_path, h5types_path, synapse_name,
initial_weights_namespace, output_weights_namespace,
reference_weights_namespace, connections_path, destination, sources,
non_structured_sources, non_structured_weights_namespace,
non_structured_weights_path, arena_id, field_width_scale,
max_delta_weight, max_opt_iter, max_weight_decay_fraction,
optimize_method, optimize_tol, optimize_grad, peak_rate,
reference_weights_are_delta, arena_margin, target_amplitude, io_size,
chunk_size, value_chunk_size, cache_size, write_size, verbose,
dry_run, plot, show_fig, save_fig):
"""
:param config: str (path to .yaml file)
:param input_features_path: str (path to .h5 file)
:param initial_weights_path: str (path to .h5 file)
:param initial_weights_namespace: str
:param output_weights_namespace: str
:param connections_path: str (path to .h5 file)
:param destination: str
:param sources: list of str
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
:param write_size:
:param verbose:
:param dry_run:
:return:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
comm = MPI.COMM_WORLD
rank = comm.rank
nranks = comm.size
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%s: %i ranks have been allocated' % (__file__, comm.size))
env = Env(comm=comm, config_file=config, io_size=io_size)
if plot and (not save_fig) and (not show_fig):
show_fig = True
if (not dry_run) and (rank == 0):
if not os.path.isfile(output_weights_path):
if initial_weights_path is not None:
input_file = h5py.File(initial_weights_path, 'r')
elif h5types_path is not None:
input_file = h5py.File(h5types_path, 'r')
else:
raise RuntimeError(
'h5types input path must be specified when weights path is not specified.'
)
output_file = h5py.File(output_weights_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
env.comm.barrier()
LTD_output_weights_namespace = 'LTD %s %s' % (output_weights_namespace,
arena_id)
LTP_output_weights_namespace = 'LTP %s %s' % (output_weights_namespace,
arena_id)
this_input_features_namespaces = [
'%s %s' % (input_features_namespace, arena_id)
for input_features_namespace in input_features_namespaces
]
selectivity_type_index = {
i: n
for n, i in viewitems(env.selectivity_types)
}
target_selectivity_type_name = 'place'
target_selectivity_type = env.selectivity_types[
target_selectivity_type_name]
features_attrs = defaultdict(dict)
source_features_attr_names = [
'Selectivity Type', 'Num Fields', 'Field Width', 'Peak Rate',
'Module ID', 'Grid Spacing', 'Grid Orientation',
'Field Width Concentration Factor', 'X Offset', 'Y Offset'
]
target_features_attr_names = [
'Selectivity Type', 'Num Fields', 'Field Width', 'Peak Rate',
'X Offset', 'Y Offset'
]
local_random = np.random.RandomState()
seed_offset = int(
env.model_config['Random Seeds']['GC Structured Weights'])
spatial_resolution = env.stimulus_config['Spatial Resolution'] # cm
arena = env.stimulus_config['Arena'][arena_id]
default_run_vel = arena.properties['default run velocity'] # cm/s
gid_count = 0
start_time = time.time()
target_gid_set = None
if len(gid) > 0:
target_gid_set = set(gid)
all_sources = sources + non_structured_sources
connection_gen_list = [ NeuroH5ProjectionGen(connections_path, source, destination, namespaces=['Synapses'], comm=comm) \
for source in all_sources ]
output_features_dict = {}
LTP_output_weights_dict = {}
LTD_output_weights_dict = {}
for iter_count, attr_gen_package in enumerate(
zip_longest(*connection_gen_list)):
local_time = time.time()
this_gid = attr_gen_package[0][0]
if not all([
attr_gen_items[0] == this_gid
for attr_gen_items in attr_gen_package
]):
raise Exception(
'Rank: %i; destination: %s; this_gid not matched across multiple attribute '
'generators: %s' %
(rank, destination,
[attr_gen_items[0] for attr_gen_items in attr_gen_package]))
if (target_gid_set is not None) and (this_gid not in target_gid_set):
continue
if this_gid is None:
selection = []
logger.info('Rank: %i received None' % rank)
else:
selection = [this_gid]
local_random.seed(int(this_gid + seed_offset))
has_structured_weights = False
dst_input_features_attr_dict = {}
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(
input_features_path,
destination,
namespace=input_features_namespace,
mask=set(target_features_attr_names),
comm=env.comm,
selection=selection)
count = 0
for gid, attr_dict in input_features_iter:
dst_input_features_attr_dict[gid] = attr_dict
count += 1
if rank == 0:
logger.info(
'Read %s feature data for %i cells in population %s' %
(input_features_namespace, count, destination))
arena_margin_size = 0.
arena_margin = max(arena_margin, 0.)
target_selectivity_features_dict = {}
target_selectivity_config_dict = {}
target_field_width_dict = {}
for gid in selection:
target_selectivity_features_dict[
gid] = dst_input_features_attr_dict.get(gid, {})
target_selectivity_features_dict[gid][
'Selectivity Type'] = np.asarray([target_selectivity_type],
dtype=np.uint8)
num_fields = target_selectivity_features_dict[gid]['Num Fields'][0]
if coordinates[0] is not None:
num_fields = 1
target_selectivity_features_dict[gid]['X Offset'] = np.asarray(
[coordinates[0]], dtype=np.float32)
target_selectivity_features_dict[gid]['Y Offset'] = np.asarray(
[coordinates[1]], dtype=np.float32)
target_selectivity_features_dict[gid][
'Num Fields'] = np.asarray([num_fields], dtype=np.uint8)
if field_width is not None:
target_selectivity_features_dict[gid][
'Field Width'] = np.asarray([field_width] * num_fields,
dtype=np.float32)
else:
this_field_width = target_selectivity_features_dict[gid][
'Field Width']
target_selectivity_features_dict[gid][
'Field Width'] = this_field_width[:num_fields]
if peak_rate is not None:
target_selectivity_features_dict[gid][
'Peak Rate'] = np.asarray([peak_rate] * num_fields,
dtype=np.float32)
input_cell_config = stimulus.get_input_cell_config(
target_selectivity_type,
selectivity_type_index,
selectivity_attr_dict=target_selectivity_features_dict[gid])
if input_cell_config.num_fields > 0:
arena_margin_size = max(
arena_margin_size,
np.max(input_cell_config.field_width) * arena_margin)
target_field_width_dict[gid] = input_cell_config.field_width
target_selectivity_config_dict[gid] = input_cell_config
has_structured_weights = True
arena_x, arena_y = stimulus.get_2D_arena_spatial_mesh(
arena, spatial_resolution, margin=arena_margin_size)
for gid, input_cell_config in viewitems(
target_selectivity_config_dict):
target_map = np.asarray(input_cell_config.get_rate_map(
arena_x, arena_y, scale=field_width_scale),
dtype=np.float32)
target_selectivity_features_dict[gid][
'Arena Rate Map'] = target_map
if not has_structured_weights:
selection = []
initial_weights_by_syn_id_dict = defaultdict(lambda: dict())
initial_weights_by_source_gid_dict = defaultdict(lambda: dict())
if initial_weights_path is not None:
initial_weights_iter = \
read_cell_attribute_selection(initial_weights_path, destination,
namespace=initial_weights_namespace,
selection=selection)
initial_weights_gid_count = 0
initial_weights_syn_count = 0
for this_gid, syn_weight_attr_dict in initial_weights_iter:
syn_ids = syn_weight_attr_dict['syn_id']
weights = syn_weight_attr_dict[synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
initial_weights_by_syn_id_dict[this_gid][int(
syn_id)] = float(weight)
initial_weights_gid_count += 1
initial_weights_syn_count += len(syn_ids)
logger.info(
'destination: %s; read initial synaptic weights for %i gids and %i syns'
% (destination, initial_weights_gid_count,
initial_weights_syn_count))
if len(non_structured_sources) > 0:
non_structured_weights_by_syn_id_dict = defaultdict(lambda: dict())
non_structured_weights_by_source_gid_dict = defaultdict(
lambda: dict())
else:
non_structured_weights_by_syn_id_dict = None
if non_structured_weights_path is not None:
non_structured_weights_iter = \
read_cell_attribute_selection(initial_weights_path, destination,
namespace=non_structured_weights_namespace,
selection=selection)
non_structured_weights_gid_count = 0
non_structured_weights_syn_count = 0
for this_gid, syn_weight_attr_dict in non_structured_weights_iter:
syn_ids = syn_weight_attr_dict['syn_id']
weights = syn_weight_attr_dict[synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
non_structured_weights_by_syn_id_dict[this_gid][int(
syn_id)] = float(weight)
non_structured_weights_gid_count += 1
non_structured_weights_syn_count += len(syn_ids)
logger.info(
'destination: %s; read non-structured synaptic weights for %i gids and %i syns'
% (
destination,
non_structured_weights_gid_count,
non_structured_weights_syn_count,
))
reference_weights_by_syn_id_dict = None
reference_weights_by_source_gid_dict = defaultdict(lambda: dict())
if reference_weights_path is not None:
reference_weights_by_syn_id_dict = defaultdict(lambda: dict())
reference_weights_iter = \
read_cell_attribute_selection(reference_weights_path, destination, namespace=reference_weights_namespace,
selection=selection)
reference_weights_gid_count = 0
for this_gid, syn_weight_attr_dict in reference_weights_iter:
syn_ids = syn_weight_attr_dict['syn_id']
weights = syn_weight_attr_dict[synapse_name]
for (syn_id, weight) in zip(syn_ids, weights):
reference_weights_by_syn_id_dict[this_gid][int(
syn_id)] = float(weight)
logger.info(
'destination: %s; read reference synaptic weights for %i gids'
% (destination, reference_weights_gid_count))
syn_count_by_source_gid_dict = defaultdict(int)
source_gid_set_dict = defaultdict(set)
syn_ids_by_source_gid_dict = defaultdict(list)
structured_syn_id_count = 0
if has_structured_weights:
for source, (destination_gid, (source_gid_array,
conn_attr_dict)) in zip_longest(
all_sources, attr_gen_package):
syn_ids = conn_attr_dict['Synapses']['syn_id']
count = 0
this_initial_weights_by_syn_id_dict = None
this_initial_weights_by_source_gid_dict = None
this_reference_weights_by_syn_id_dict = None
this_reference_weights_by_source_gid_dict = None
this_non_structured_weights_by_syn_id_dict = None
this_non_structured_weights_by_source_gid_dict = None
if destination_gid is not None:
this_initial_weights_by_syn_id_dict = initial_weights_by_syn_id_dict[
destination_gid]
this_initial_weights_by_source_gid_dict = initial_weights_by_source_gid_dict[
destination_gid]
if reference_weights_by_syn_id_dict is not None:
this_reference_weights_by_syn_id_dict = reference_weights_by_syn_id_dict[
destination_gid]
this_reference_weights_by_source_gid_dict = reference_weights_by_source_gid_dict[
destination_gid]
this_non_structured_weights_by_syn_id_dict = non_structured_weights_by_syn_id_dict[
destination_gid]
this_non_structured_weights_by_source_gid_dict = non_structured_weights_by_source_gid_dict[
destination_gid]
for i in range(len(source_gid_array)):
this_source_gid = source_gid_array[i]
this_syn_id = syn_ids[i]
if this_syn_id in this_initial_weights_by_syn_id_dict:
this_syn_wgt = this_initial_weights_by_syn_id_dict[
this_syn_id]
if this_source_gid not in this_initial_weights_by_source_gid_dict:
this_initial_weights_by_source_gid_dict[
this_source_gid] = this_syn_wgt
if this_reference_weights_by_syn_id_dict is not None:
this_reference_weights_by_source_gid_dict[this_source_gid] = \
this_reference_weights_by_syn_id_dict[this_syn_id]
elif this_syn_id in this_non_structured_weights_by_syn_id_dict:
this_syn_wgt = this_non_structured_weights_by_syn_id_dict[
this_syn_id]
if this_source_gid not in this_non_structured_weights_by_source_gid_dict:
this_non_structured_weights_by_source_gid_dict[
this_source_gid] = this_syn_wgt
source_gid_set_dict[source].add(this_source_gid)
syn_ids_by_source_gid_dict[this_source_gid].append(
this_syn_id)
syn_count_by_source_gid_dict[this_source_gid] += 1
count += 1
if source not in non_structured_sources:
structured_syn_id_count += len(syn_ids)
logger.info(
'Rank %i; destination: %s; gid %i; %d edges from source population %s'
% (rank, destination, this_gid, count, source))
input_rate_maps_by_source_gid_dict = {}
if len(non_structured_sources) > 0:
non_structured_input_rate_maps_by_source_gid_dict = {}
else:
non_structured_input_rate_maps_by_source_gid_dict = None
for source in all_sources:
if has_structured_weights:
source_gids = list(source_gid_set_dict[source])
else:
source_gids = []
if rank == 0:
logger.info(
'Reading %s feature data for %i cells in population %s...'
% (input_features_namespace, len(source_gids), source))
for input_features_namespace in this_input_features_namespaces:
input_features_iter = read_cell_attribute_selection(
input_features_path,
source,
namespace=input_features_namespace,
mask=set(source_features_attr_names),
comm=env.comm,
selection=source_gids)
count = 0
for gid, attr_dict in input_features_iter:
this_selectivity_type = attr_dict['Selectivity Type'][0]
this_selectivity_type_name = selectivity_type_index[
this_selectivity_type]
input_cell_config = stimulus.get_input_cell_config(
this_selectivity_type,
selectivity_type_index,
selectivity_attr_dict=attr_dict)
this_arena_rate_map = np.asarray(
input_cell_config.get_rate_map(arena_x, arena_y),
dtype=np.float32)
if source in non_structured_sources:
non_structured_input_rate_maps_by_source_gid_dict[
gid] = this_arena_rate_map
else:
input_rate_maps_by_source_gid_dict[
gid] = this_arena_rate_map
count += 1
if rank == 0:
logger.info(
'Read %s feature data for %i cells in population %s' %
(input_features_namespace, count, source))
if has_structured_weights:
if is_interactive:
context.update(locals())
save_fig_path = None
if save_fig is not None:
save_fig_path = '%s/Structured Weights %s %d.png' % (
save_fig, destination, this_gid)
normalized_LTP_delta_weights_dict, LTD_delta_weights_dict, arena_LS_map = \
synapses.generate_structured_weights(target_map=target_selectivity_features_dict[this_gid]['Arena Rate Map'],
initial_weight_dict=this_initial_weights_by_source_gid_dict,
reference_weight_dict=this_reference_weights_by_source_gid_dict,
reference_weights_are_delta=reference_weights_are_delta,
reference_weights_namespace=reference_weights_namespace,
input_rate_map_dict=input_rate_maps_by_source_gid_dict,
non_structured_input_rate_map_dict=non_structured_input_rate_maps_by_source_gid_dict,
non_structured_weights_dict=this_non_structured_weights_by_source_gid_dict,
syn_count_dict=syn_count_by_source_gid_dict,
max_delta_weight=max_delta_weight,
max_opt_iter=max_opt_iter,
max_weight_decay_fraction=max_weight_decay_fraction,
target_amplitude=target_amplitude,
arena_x=arena_x, arena_y=arena_y,
optimize_method=optimize_method,
optimize_tol=optimize_tol,
optimize_grad=optimize_grad,
verbose=verbose, plot=plot, show_fig=show_fig,
save_fig=save_fig_path,
fig_kwargs={'gid': this_gid,
'field_width': target_field_width_dict[this_gid]})
gc.collect()
this_selectivity_dict = target_selectivity_features_dict[this_gid]
output_features_dict[this_gid] = {
fld: this_selectivity_dict[fld]
for fld in [
'Selectivity Type', 'Num Fields', 'Field Width',
'Peak Rate', 'X Offset', 'Y Offset'
]
}
output_features_dict[this_gid]['Arena State Map'] = np.asarray(
arena_LS_map.ravel(), dtype=np.float32)
output_syn_ids = np.empty(structured_syn_id_count, dtype='uint32')
LTD_output_weights = np.empty(structured_syn_id_count,
dtype='float32')
LTP_output_weights = np.empty(structured_syn_id_count,
dtype='float32')
i = 0
for source_gid in normalized_LTP_delta_weights_dict:
for syn_id in syn_ids_by_source_gid_dict[source_gid]:
output_syn_ids[i] = syn_id
LTP_output_weights[i] = normalized_LTP_delta_weights_dict[
source_gid]
LTD_output_weights[i] = LTD_delta_weights_dict[source_gid]
i += 1
LTP_output_weights_dict[this_gid] = {
'syn_id': output_syn_ids,
synapse_name: LTP_output_weights
}
LTD_output_weights_dict[this_gid] = {
'syn_id': output_syn_ids,
synapse_name: LTD_output_weights
}
logger.info(
'Rank %i; destination: %s; gid %i; generated structured weights for %i inputs in %.2f '
's' % (rank, destination, this_gid, len(output_syn_ids),
time.time() - local_time))
gid_count += 1
if iter_count % write_size == 0:
if not dry_run:
append_cell_attributes(output_weights_path,
destination,
LTD_output_weights_dict,
namespace=LTD_output_weights_namespace,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
append_cell_attributes(output_weights_path,
destination,
LTP_output_weights_dict,
namespace=LTP_output_weights_namespace,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
count = comm.reduce(len(LTP_output_weights_dict),
op=MPI.SUM,
root=0)
if rank == 0:
logger.info(
'Destination: %s; appended weights for %i cells' %
(destination, count))
if output_features_path is not None:
if output_features_namespace is None:
output_features_namespace = '%s Selectivity' % target_selectivity_type_name.title(
)
this_output_features_namespace = '%s %s' % (
output_features_namespace, arena_id)
logger.info(str(output_features_dict))
append_cell_attributes(
output_features_path,
destination,
output_features_dict,
namespace=this_output_features_namespace)
count = comm.reduce(len(output_features_dict),
op=MPI.SUM,
root=0)
if rank == 0:
logger.info(
'Destination: %s; appended selectivity features for %i cells'
% (destination, count))
LTP_output_weights_dict.clear()
LTD_output_weights_dict.clear()
output_features_dict.clear()
gc.collect()
env.comm.barrier()
if not dry_run:
append_cell_attributes(output_weights_path,
destination,
LTD_output_weights_dict,
namespace=LTD_output_weights_namespace,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
append_cell_attributes(output_weights_path,
destination,
LTP_output_weights_dict,
namespace=LTP_output_weights_namespace,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
count = comm.reduce(len(LTP_output_weights_dict), op=MPI.SUM, root=0)
if rank == 0:
logger.info('Destination: %s; appended weights for %i cells' %
(destination, count))
if output_features_path is not None:
if output_features_namespace is None:
output_features_namespace = 'Selectivity Features'
this_output_features_namespace = '%s %s' % (
output_features_namespace, arena_id)
append_cell_attributes(output_features_path,
destination,
output_features_dict,
namespace=this_output_features_namespace)
count = comm.reduce(len(output_features_dict), op=MPI.SUM, root=0)
if rank == 0:
logger.info(
'Destination: %s; appended selectivity features for %i cells'
% (destination, count))
env.comm.barrier()
global_count = comm.gather(gid_count, root=0)
if rank == 0:
logger.info(
'destination: %s; %i ranks assigned structured weights to %i cells in %.2f s'
% (destination, comm.size, np.sum(global_count),
time.time() - start_time))
def init_circuit_context(env,
pop_name,
gid,
load_edges=False,
connection_graph=None,
load_weights=False,
weight_dict=None,
load_synapses=False,
synapses_dict=None,
set_edge_delays=True,
**kwargs):
syn_attrs = env.synapse_attributes
synapse_config = env.celltypes[pop_name]['synapses']
has_weights = False
weight_config = []
if 'weights' in synapse_config:
has_weights = True
weight_config = synapse_config['weights']
init_synapses = False
init_weights = False
init_edges = False
if load_edges or (connection_graph is not None):
init_synapses = True
init_edges = True
if has_weights and (load_weights or (weight_dict is not None)):
init_synapses = True
init_weights = True
if load_synapses or (synapses_dict is not None):
init_synapses = True
if init_synapses:
if synapses_dict is not None:
syn_attrs.init_syn_id_attrs(gid, **synapses_dict)
elif load_synapses or load_edges:
if (pop_name in env.cell_attribute_info) and (
'Synapse Attributes' in env.cell_attribute_info[pop_name]):
synapses_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name, [gid],
'Synapse Attributes',
mask=set([
'syn_ids', 'syn_locs', 'syn_secs', 'syn_layers',
'syn_types', 'swc_types'
]),
comm=env.comm)
syn_attrs.init_syn_id_attrs_from_iter(synapses_iter)
else:
raise RuntimeError(
'init_circuit_context: synapse attributes not found for %s: gid: %i'
% (pop_name, gid))
else:
raise RuntimeError(
"init_circuit_context: invalid synapses parameters")
if init_weights and has_weights:
for weight_config_dict in weight_config:
expr_closure = weight_config_dict.get('closure', None)
weights_namespaces = weight_config_dict['namespace']
cell_weights_dicts = {}
if weight_dict is not None:
for weights_namespace in weights_namespaces:
if weights_namespace in weight_dict:
cell_weights_dicts[weights_namespace] = weight_dict[
weights_namespace]
elif load_weights:
if (env.data_file_path is None):
raise RuntimeError(
'init_circuit_context: load_weights=True but data file path is not specified '
)
for weights_namespace in weights_namespaces:
cell_weights_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name,
selection=[gid],
namespace=weights_namespace,
comm=env.comm)
for cell_weights_gid, cell_weights_dict in cell_weights_iter:
assert (cell_weights_gid == gid)
cell_weights_dicts[
weights_namespace] = cell_weights_dict
else:
raise RuntimeError(
"init_circuit_context: invalid weights parameters")
if len(weights_namespaces) != len(cell_weights_dicts):
logger.warning(
"init_circuit_context: Unable to load all weights namespaces: %s"
% str(weights_namespaces))
multiple_weights = 'error'
append_weights = False
for weights_namespace in weights_namespaces:
if weights_namespace in cell_weights_dicts:
cell_weights_dict = cell_weights_dicts[weights_namespace]
weights_syn_ids = cell_weights_dict['syn_id']
for syn_name in (syn_name for syn_name in cell_weights_dict
if syn_name != 'syn_id'):
weights_values = cell_weights_dict[syn_name]
syn_attrs.add_mech_attrs_from_iter(
gid,
syn_name,
zip_longest(
weights_syn_ids,
[{
'weight': Promise(expr_closure, [x])
} for x in weights_values]
if expr_closure else [{
'weight': x
} for x in weights_values]),
multiple=multiple_weights,
append=append_weights)
logger.info(
'init_circuit_context: gid: %i; found %i %s synaptic weights in namespace %s'
% (gid, len(cell_weights_dict[syn_name]), syn_name,
weights_namespace))
logger.info(
'weight_values min/max/mean: %.02f / %.02f / %.02f'
% (np.min(weights_values), np.max(weights_values),
np.mean(weights_values)))
expr_closure = None
append_weights = True
multiple_weights = 'overwrite'
if init_edges:
if connection_graph is not None:
(graph, a) = connection_graph
elif load_edges:
if env.connectivity_file_path is None:
raise RuntimeError(
'init_circuit_context: load_edges=True but connectivity file path is not specified '
)
elif os.path.isfile(env.connectivity_file_path):
(graph, a) = read_graph_selection(
file_name=env.connectivity_file_path,
selection=[gid],
namespaces=['Synapses', 'Connections'],
comm=env.comm)
else:
raise RuntimeError(
'init_circuit_context: connection file %s not found' %
env.connectivity_file_path)
else:
(graph, a) = None, None
if graph is not None:
if pop_name in graph:
for presyn_name in graph[pop_name].keys():
edge_iter = graph[pop_name][presyn_name]
syn_attrs.init_edge_attrs_from_iter(pop_name, presyn_name, a,
edge_iter, set_edge_delays)
else:
logger.error(
'init_circuit_context: connection attributes not found for %s: gid: %i'
% (pop_name, gid))
raise Exception
def load_biophys_cell_dicts(env,
pop_name,
gid_set,
data_file_path=None,
load_connections=True,
validate_tree=True):
"""
Loads the data necessary to instantiate BiophysCell into the given dictionary.
:param env: an instance of env.Env
:param pop_name: population name
:param gid: gid
:param data_file_path: str or None
:param load_connections: bool
:param validate_tree: bool
Environment can be instantiated as:
env = Env(config_file, template_paths, dataset_prefix, config_prefix)
:param template_paths: str; colon-separated list of paths to directories containing hoc cell templates
:param dataset_prefix: str; path to directory containing required neuroh5 data files
:param config_prefix: str; path to directory containing network and cell mechanism config files
"""
synapse_config = env.celltypes[pop_name]['synapses']
has_weights = False
weights_config = None
if 'weights' in synapse_config:
has_weights = True
weights_config = synapse_config['weights']
## Loads cell morphological data, synaptic attributes and connection data
tree_dicts = {}
synapses_dicts = {}
weight_dicts = {}
connection_graphs = {gid: {pop_name: {}} for gid in gid_set}
graph_attr_info = None
gid_list = list(gid_set)
tree_attr_iter, _ = read_tree_selection(env.data_file_path,
pop_name,
gid_list,
comm=env.comm,
topology=True,
validate=validate_tree)
for gid, tree_dict in tree_attr_iter:
tree_dicts[gid] = tree_dict
if load_connections:
synapses_iter = read_cell_attribute_selection(
env.data_file_path,
pop_name,
gid_list,
'Synapse Attributes',
mask=set([
'syn_ids', 'syn_locs', 'syn_secs', 'syn_layers', 'syn_types',
'swc_types'
]),
comm=env.comm)
for gid, attr_dict in synapses_iter:
synapses_dicts[gid] = attr_dict
if has_weights:
for config in weights_config:
weights_namespaces = config['namespace']
cell_weights_iters = [
read_cell_attribute_selection(env.data_file_path,
pop_name,
gid_list,
weights_namespace,
comm=env.comm)
for weights_namespace in weights_namespaces
]
for weights_namespace, cell_weights_iter in zip_longest(
weights_namespaces, cell_weights_iters):
for gid, cell_weights_dict in cell_weights_iter:
this_weights_dict = weight_dicts.get(gid, {})
this_weights_dict[
weights_namespace] = cell_weights_dict
weight_dicts[gid] = this_weights_dict
graph, graph_attr_info = read_graph_selection(
file_name=env.connectivity_file_path,
selection=gid_list,
namespaces=['Synapses', 'Connections'],
comm=env.comm)
if pop_name in graph:
for presyn_name in graph[pop_name].keys():
edge_iter = graph[pop_name][presyn_name]
for (postsyn_gid, edges) in edge_iter:
connection_graphs[postsyn_gid][pop_name][presyn_name] = [
(postsyn_gid, edges)
]
cell_dicts = {}
for gid in gid_set:
this_cell_dict = {}
tree_dict = tree_dicts[gid]
this_cell_dict['morph'] = tree_dict
if load_connections:
synapses_dict = synapses_dicts[gid]
weight_dict = weight_dicts.get(gid, None)
connection_graph = connection_graphs[gid]
this_cell_dict['synapse'] = synapses_dict
this_cell_dict['connectivity'] = connection_graph, graph_attr_info
this_cell_dict['weight'] = weight_dict
cell_dicts[gid] = this_cell_dict
return cell_dicts
