def main(population, trees_path, index_path, chunk_size, value_chunk_size):
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
print '%i ranks have been allocated' % comm.size
sys.stdout.flush()
f = open(index_path)
lines = f.readlines()
index = {}
while lines:
for l in lines:
a = filter(None, l.split(" "))
gid = int(a[0])
newgid = int(a[1])
index[gid] = newgid
lines = f.readlines()
f.close()
selection_dict = {}
for gid, morph_dict in NeuroH5TreeGen(comm, forest_path, population, io_size=io_size):
if index.has_key(gid):
newgid = index[gid]
selection_dict[newgid] = morph_dict
append_cell_trees (comm, output_path, population, selection_dict, io_size=io_size)
def main(config, template_path, prototype_gid, prototype_path, forest_path, population, io_size, verbose):
"""
:param config:
:param template_path:
:param prototype_gid:
:param prototype_path:
:param forest_path:
:param population:
:param io_size:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(script_name)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=MPI.COMM_WORLD, config_file=config, template_paths=template_path)
configure_hoc_env(env)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
layers = env.layers
layer_idx_dict = { layers[layer_name]: layer_name
for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus'] }
(tree_iter, _) = read_tree_selection(prototype_path, population, selection=[prototype_gid])
(_, prototype_morph_dict) = next(tree_iter)
prototype_x = prototype_morph_dict['x']
prototype_y = prototype_morph_dict['y']
prototype_z = prototype_morph_dict['z']
prototype_xyz = (prototype_x, prototype_y, prototype_z)
(pop_ranges, _) = read_population_ranges(forest_path, comm=comm)
start_time = time.time()
(population_start, _) = pop_ranges[population]
template_class = load_cell_template(env, population, bcast_template=True)
for gid, morph_dict in NeuroH5TreeGen(forest_path, population, io_size=io_size, cache_size=1, comm=comm, topology=True):
# trees, _ = scatter_read_trees(forest_path, population, io_size=io_size, comm=comm, topology=True)
# for gid, morph_dict in trees:
if gid is not None:
logger.info('Rank %i gid: %i' % (rank, gid))
secnodes_dict = morph_dict['section_topology']['nodes']
vx = morph_dict['x']
vy = morph_dict['y']
vz = morph_dict['z']
if compare_points((vx,vy,vz), prototype_xyz):
logger.info('Possible match: gid %i' % gid)
MPI.Finalize()
def main(config, template_path, output_path, forest_path, populations, io_size,
chunk_size, value_chunk_size, cache_size, verbose):
"""
:param config:
:param template_path:
:param forest_path:
:param populations:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(script_name)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=MPI.COMM_WORLD,
config_file=config,
template_paths=template_path)
h('objref nil, pc, templatePaths')
h.load_file("nrngui.hoc")
h.load_file("./templates/Value.hoc")
h.xopen("./lib.hoc")
h.pc = h.ParallelContext()
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
h.templatePaths = h.List()
for path in env.templatePaths:
h.templatePaths.append(h.Value(1, path))
if output_path is None:
output_path = forest_path
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(forest_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(pop_ranges, _) = read_population_ranges(forest_path, comm=comm)
start_time = time.time()
for population in populations:
logger.info('Rank %i population: %s' % (rank, population))
count = 0
(population_start, _) = pop_ranges[population]
template_name = env.celltypes[population]['template']
h.find_template(h.pc, h.templatePaths, template_name)
template_class = eval('h.%s' % template_name)
measures_dict = {}
for gid, morph_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=True):
if gid is not None:
logger.info('Rank %i gid: %i' % (rank, gid))
cell = cells.make_neurotree_cell(template_class,
neurotree_dict=morph_dict,
gid=gid)
secnodes_dict = morph_dict['section_topology']['nodes']
apicalidx = set(cell.apicalidx)
basalidx = set(cell.basalidx)
dendrite_area_dict = {k + 1: 0.0 for k in range(0, 4)}
dendrite_length_dict = {k + 1: 0.0 for k in range(0, 4)}
for (i, sec) in enumerate(cell.sections):
if (i in apicalidx) or (i in basalidx):
secnodes = secnodes_dict[i]
prev_layer = None
for seg in sec.allseg():
L = seg.sec.L
nseg = seg.sec.nseg
seg_l = old_div(L, nseg)
seg_area = h.area(seg.x)
layer = cells.get_node_attribute(
'layer', morph_dict, seg.sec, secnodes, seg.x)
layer = layer if layer > 0 else (
prev_layer if prev_layer is not None else 1)
prev_layer = layer
dendrite_length_dict[layer] += seg_l
dendrite_area_dict[layer] += seg_area
measures_dict[gid] = { 'dendrite_area': np.asarray([ dendrite_area_dict[k] for k in sorted(dendrite_area_dict.keys()) ], dtype=np.float32), \
'dendrite_length': np.asarray([ dendrite_length_dict[k] for k in sorted(dendrite_length_dict.keys()) ], dtype=np.float32) }
del cell
count += 1
else:
logger.info('Rank %i gid is None' % rank)
append_cell_attributes(output_path,
population,
measures_dict,
namespace='Tree Measurements',
comm=comm,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size,
cache_size=cache_size)
MPI.Finalize()
def main(config, template_path, output_path, forest_path, populations,
distance_bin_size, io_size, chunk_size, value_chunk_size, cache_size,
verbose):
"""
:param config:
:param template_path:
:param forest_path:
:param populations:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(script_name)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=MPI.COMM_WORLD,
config_file=config,
template_paths=template_path)
configure_hoc_env(env)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
if output_path is None:
output_path = forest_path
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(forest_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
layers = env.layers
layer_idx_dict = {
layers[layer_name]: layer_name
for layer_name in ['GCL', 'IML', 'MML', 'OML', 'Hilus']
}
(pop_ranges, _) = read_population_ranges(forest_path, comm=comm)
start_time = time.time()
for population in populations:
logger.info('Rank %i population: %s' % (rank, population))
count = 0
(population_start, _) = pop_ranges[population]
template_class = load_cell_template(env,
population,
bcast_template=True)
measures_dict = {}
for gid, morph_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=True):
if gid is not None:
logger.info('Rank %i gid: %i' % (rank, gid))
cell = cells.make_neurotree_cell(template_class,
neurotree_dict=morph_dict,
gid=gid)
secnodes_dict = morph_dict['section_topology']['nodes']
apicalidx = set(cell.apicalidx)
basalidx = set(cell.basalidx)
dendrite_area_dict = {k: 0.0 for k in layer_idx_dict}
dendrite_length_dict = {k: 0.0 for k in layer_idx_dict}
dendrite_distances = []
dendrite_diams = []
for (i, sec) in enumerate(cell.sections):
if (i in apicalidx) or (i in basalidx):
secnodes = secnodes_dict[i]
for seg in sec.allseg():
L = seg.sec.L
nseg = seg.sec.nseg
seg_l = L / nseg
seg_area = h.area(seg.x)
seg_diam = seg.diam
seg_distance = get_distance_to_node(
cell,
list(cell.soma)[0], seg.sec, seg.x)
dendrite_diams.append(seg_diam)
dendrite_distances.append(seg_distance)
layer = synapses.get_node_attribute(
'layer', morph_dict, seg.sec, secnodes, seg.x)
dendrite_length_dict[layer] += seg_l
dendrite_area_dict[layer] += seg_area
dendrite_distance_array = np.asarray(dendrite_distances)
dendrite_diam_array = np.asarray(dendrite_diams)
dendrite_distance_bin_range = int(
((np.max(dendrite_distance_array)) -
np.min(dendrite_distance_array)) / distance_bin_size) + 1
dendrite_distance_counts, dendrite_distance_edges = np.histogram(
dendrite_distance_array,
bins=dendrite_distance_bin_range,
density=False)
dendrite_diam_sums, _ = np.histogram(
dendrite_distance_array,
weights=dendrite_diam_array,
bins=dendrite_distance_bin_range,
density=False)
dendrite_mean_diam_hist = np.zeros_like(dendrite_diam_sums)
np.divide(dendrite_diam_sums,
dendrite_distance_counts,
where=dendrite_distance_counts > 0,
out=dendrite_mean_diam_hist)
dendrite_area_per_layer = np.asarray([
dendrite_area_dict[k]
for k in sorted(dendrite_area_dict.keys())
],
dtype=np.float32)
dendrite_length_per_layer = np.asarray([
dendrite_length_dict[k]
for k in sorted(dendrite_length_dict.keys())
],
dtype=np.float32)
measures_dict[gid] = {
'dendrite_distance_hist_edges':
np.asarray(dendrite_distance_edges, dtype=np.float32),
'dendrite_distance_counts':
np.asarray(dendrite_distance_counts, dtype=np.int32),
'dendrite_mean_diam_hist':
np.asarray(dendrite_mean_diam_hist, dtype=np.float32),
'dendrite_area_per_layer':
dendrite_area_per_layer,
'dendrite_length_per_layer':
dendrite_length_per_layer
}
del cell
count += 1
else:
logger.info('Rank %i gid is None' % rank)
append_cell_attributes(output_path,
population,
measures_dict,
namespace='Tree Measurements',
comm=comm,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size,
cache_size=cache_size)
MPI.Finalize()
def main(config, config_prefix, max_section_length, population, forest_path,
template_path, output_path, io_size, chunk_size, value_chunk_size,
dry_run, verbose):
"""
:param population: str
:param forest_path: str (path)
:param output_path: str (path)
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param verbose: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
env = Env(comm=comm,
config_file=config,
config_prefix=config_prefix,
template_paths=template_path)
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(forest_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(forest_pop_ranges, _) = read_population_ranges(forest_path)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
(pop_ranges, _) = read_population_ranges(output_path)
(population_start, population_count) = pop_ranges[population]
new_trees_dict = {}
for gid, tree_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=False):
if gid is not None:
logger.info("Rank %d received gid %d" % (rank, gid))
logger.info(pprint.pformat(tree_dict))
new_tree_dict = cells.resize_tree_sections(tree_dict,
max_section_length)
logger.info(pprint.pformat(new_tree_dict))
new_trees_dict[gid] = new_tree_dict
if not dry_run:
append_cell_trees(output_path,
population,
new_trees_dict,
io_size=io_size,
comm=comm)
comm.barrier()
if (not dry_run) and (rank == 0):
logger.info('Appended resized trees to %s' % output_path)
def main(config_file, config_prefix, input_path, population, template_paths,
dataset_prefix, results_path, results_file_id, results_namespace_id,
v_init, io_size, chunk_size, value_chunk_size, write_size, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
if io_size == -1:
io_size = comm.size
if results_file_id is None:
if rank == 0:
result_file_id = uuid.uuid4()
results_file_id = comm.bcast(results_file_id, root=0)
if results_namespace_id is None:
results_namespace_id = 'Cell Clamp Results'
comm = MPI.COMM_WORLD
np.seterr(all='raise')
verbose = True
params = dict(locals())
env = Env(**params)
configure_hoc_env(env)
if rank == 0:
io_utils.mkout(env, env.results_file_path)
env.comm.barrier()
env.cell_selection = {}
template_class = load_cell_template(env, population)
if input_path is not None:
env.data_file_path = input_path
env.load_celltypes()
synapse_config = env.celltypes[population]['synapses']
weights_namespaces = []
if 'weights' in synapse_config:
has_weights = synapse_config['weights']
if has_weights:
if 'weights namespace' in synapse_config:
weights_namespaces.append(synapse_config['weights namespace'])
elif 'weights namespaces' in synapse_config:
weights_namespaces.extend(synapse_config['weights namespaces'])
else:
weights_namespaces.append('Weights')
else:
has_weights = False
start_time = time.time()
count = 0
gid_count = 0
attr_dict = {}
if input_path is None:
cell_path = env.data_file_path
connectivity_path = env.connectivity_file_path
else:
cell_path = input_path
connectivity_path = input_path
for gid, morph_dict in NeuroH5TreeGen(cell_path,
population,
io_size=io_size,
comm=env.comm,
topology=True):
local_time = time.time()
if gid is not None:
color = 0
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid: %i' % (rank, gid))
cell_dict = {'morph': morph_dict}
synapses_iter = read_cell_attribute_selection(cell_path,
population, [gid],
'Synapse Attributes',
comm=comm0)
_, synapse_dict = next(synapses_iter)
cell_dict['synapse'] = synapse_dict
if has_weights:
cell_weights_iters = [
read_cell_attribute_selection(cell_path,
population, [gid],
weights_namespace,
comm=comm0)
for weights_namespace in weights_namespaces
]
weight_dict = dict(
zip_longest(weights_namespaces, cell_weights_iters))
cell_dict['weight'] = weight_dict
(graph,
a) = read_graph_selection(file_name=connectivity_path,
selection=[gid],
namespaces=['Synapses', 'Connections'],
comm=comm0)
cell_dict['connectivity'] = (graph, a)
gid_count += 1
attr_dict[gid] = {}
attr_dict[gid].update(
cell_clamp.measure_passive(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_ap_rate(gid,
population,
v_init,
env,
cell_dict=cell_dict))
attr_dict[gid].update(
cell_clamp.measure_fi(gid,
population,
v_init,
env,
cell_dict=cell_dict))
else:
color = 1
comm0 = comm.Split(color, 0)
logger.info('Rank %i gid is None' % (rank))
comm0.Free()
count += 1
if (results_path is not None) and (count % write_size == 0):
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
attr_dict = {}
env.comm.barrier()
if results_path is not None:
append_cell_attributes(env.results_file_path,
population,
attr_dict,
namespace=env.results_namespace_id,
comm=env.comm,
io_size=env.io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
global_count = env.comm.gather(gid_count, root=0)
MPI.Finalize()
def main(forest_path, cell_attr_path, connections_path, cell_attr_namespace,
cell_attr, destination, source, io_size, cache_size):
"""
:param forest_path: str (path)
:param cell_attr_path: str (path)
:param connections_path: str (path)
:param cell_attr_namespace: str
:param cell_attr: str
:param destination: str
:param source: str
:param io_size: int
:param cache_size: int
"""
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
print '%s: %i ranks have been allocated' % (os.path.basename(__file__),
comm.size)
sys.stdout.flush()
pop_ranges, pop_size = read_population_ranges(cell_attr_path, comm=comm)
destination_gid_offset = pop_ranges[destination][0]
source_gid_offset = pop_ranges[source][0]
maxiter = 10
cell_attr_matched = 0
cell_attr_processed = 0
edge_attr_matched = 0
edge_attr_processed = 0
tree_attr_matched = 0
tree_attr_processed = 0
cell_attr_gen = NeuroH5CellAttrGen(cell_attr_path,
destination,
comm=comm,
io_size=io_size,
cache_size=cache_size,
namespace=cell_attr_namespace)
index_map = get_cell_attributes_index_map(comm, cell_attr_path,
destination, cell_attr_namespace)
for itercount, (target_gid, attr_dict) in enumerate(cell_attr_gen):
print 'Rank: %i receieved target_gid: %s from the cell attribute generator.' % (
rank, str(target_gid))
attr_dict2 = select_cell_attributes(
target_gid,
comm,
cell_attr_path,
index_map,
destination,
cell_attr_namespace,
population_offset=destination_gid_offset)
if np.all(attr_dict[cell_attr][:] == attr_dict2[cell_attr][:]):
print 'Rank: %i; cell attributes match!' % rank
cell_attr_matched += 1
else:
print 'Rank: %i; cell attributes do not match.' % rank
comm.barrier()
cell_attr_processed += 1
if itercount > maxiter:
break
cell_attr_matched = comm.gather(cell_attr_matched, root=0)
cell_attr_processed = comm.gather(cell_attr_processed, root=0)
if connections_path is not None:
edge_attr_gen = NeuroH5ProjectionGen(connections_path,
source,
destination,
comm=comm,
cache_size=cache_size,
namespaces=['Synapses'])
index_map = get_edge_attributes_index_map(comm, connections_path,
source, destination)
processed = 0
for itercount, (target_gid, attr_package) in enumerate(edge_attr_gen):
print 'Rank: %i receieved target_gid: %s from the edge attribute generator.' % (
rank, str(target_gid))
source_indexes, attr_dict = attr_package
syn_ids = attr_dict['Synapses']['syn_id']
source_indexes2, attr_dict2 = select_edge_attributes(
target_gid,
comm,
connections_path,
index_map,
source,
destination,
namespaces=['Synapses'],
source_offset=source_gid_offset,
destination_offset=destination_gid_offset)
syn_ids2 = attr_dict2['Synapses']['syn_id']
if np.all(syn_ids == syn_ids2) and np.all(
source_indexes == source_indexes2):
print 'Rank: %i; edge attributes match!' % rank
edge_attr_matched += 1
else:
print 'Rank: %i; attributes do not match.' % rank
comm.barrier()
edge_attr_processed += 1
if itercount > maxiter:
break
edge_attr_matched = comm.gather(edge_attr_matched, root=0)
edge_attr_processed = comm.gather(edge_attr_processed, root=0)
if forest_path is not None:
tree_attr_gen = NeuroH5TreeGen(forest_path,
destination,
comm=comm,
io_size=io_size)
for itercount, (target_gid, attr_dict) in enumerate(tree_attr_gen):
print 'Rank: %i receieved target_gid: %s from the tree attribute generator.' % (
rank, str(target_gid))
attr_dict2 = select_tree_attributes(target_gid, comm, forest_path,
destination)
if (attr_dict.keys() == attr_dict2.keys()) and all(
attr_dict['layer'] == attr_dict2['layer']):
print 'Rank: %i; tree attributes match!' % rank
tree_attr_matched += 1
else:
print 'Rank: %i; tree attributes do not match.' % rank
comm.barrier()
tree_attr_processed += 1
if itercount > maxiter:
break
tree_attr_matched = comm.gather(tree_attr_matched, root=0)
tree_attr_processed = comm.gather(tree_attr_processed, root=0)
if comm.rank == 0:
print '%i / %i processed gids had matching cell attributes returned by both read methods' % \
(np.sum(cell_attr_matched), np.sum(cell_attr_processed))
print '%i / %i processed gids had matching edge attributes returned by both read methods' % \
(np.sum(edge_attr_matched), np.sum(edge_attr_processed))
print '%i / %i processed gids had matching tree attributes returned by both read methods' % \
(np.sum(tree_attr_matched), np.sum(tree_attr_processed))
def main(config, config_prefix, template_path, output_path, forest_path,
populations, distribution, io_size, chunk_size, value_chunk_size,
cache_size, write_size, verbose, dry_run):
"""
:param config:
:param config_prefix:
:param template_path:
:param forest_path:
:param populations:
:param distribution:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
env = Env(comm=MPI.COMM_WORLD,
config_file=config,
config_prefix=config_prefix,
template_paths=template_path)
configure_hoc_env(env)
if io_size == -1:
io_size = comm.size
if output_path is None:
output_path = forest_path
if not dry_run:
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(forest_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(pop_ranges, _) = read_population_ranges(forest_path, comm=comm)
start_time = time.time()
syn_stats = {}
for population in populations:
logger.info('Rank %i population: %s' % (rank, population))
(population_start, _) = pop_ranges[population]
template_class = load_cell_template(env, population)
density_dict = env.celltypes[population]['synapses']['density']
layer_set_dict = defaultdict(set)
swc_set_dict = defaultdict(set)
for sec_name, sec_dict in viewitems(density_dict):
for syn_type, syn_dict in viewitems(sec_dict):
swc_set_dict[syn_type].add(env.SWC_Types[sec_name])
for layer_name in syn_dict:
if layer_name != 'default':
layer = env.layers[layer_name]
layer_set_dict[syn_type].add(layer)
syn_stats_dict = { 'section': defaultdict(lambda: { 'excitatory': 0, 'inhibitory': 0 }), \
'layer': defaultdict(lambda: { 'excitatory': 0, 'inhibitory': 0 }), \
'swc_type': defaultdict(lambda: { 'excitatory': 0, 'inhibitory': 0 }), \
'total': { 'excitatory': 0, 'inhibitory': 0 } }
count = 0
gid_count = 0
synapse_dict = {}
for gid, morph_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=True):
local_time = time.time()
if gid is not None:
logger.info('Rank %i gid: %i' % (rank, gid))
cell = cells.make_neurotree_cell(template_class,
neurotree_dict=morph_dict,
gid=gid)
cell_sec_dict = {
'apical': (cell.apical, None),
'basal': (cell.basal, None),
'soma': (cell.soma, None),
'ais': (cell.ais, None),
'hillock': (cell.hillock, None)
}
cell_secidx_dict = {
'apical': cell.apicalidx,
'basal': cell.basalidx,
'soma': cell.somaidx,
'ais': cell.aisidx,
'hillock': cell.hilidx
}
random_seed = env.model_config['Random Seeds'][
'Synapse Locations'] + gid
if distribution == 'uniform':
syn_dict, seg_density_per_sec = synapses.distribute_uniform_synapses(
random_seed, env.Synapse_Types, env.SWC_Types,
env.layers, density_dict, morph_dict, cell_sec_dict,
cell_secidx_dict)
elif distribution == 'poisson':
syn_dict, seg_density_per_sec = synapses.distribute_poisson_synapses(
random_seed, env.Synapse_Types, env.SWC_Types,
env.layers, density_dict, morph_dict, cell_sec_dict,
cell_secidx_dict)
else:
raise Exception('Unknown distribution type: %s' %
distribution)
synapse_dict[gid] = syn_dict
this_syn_stats = update_syn_stats(env, syn_stats_dict,
syn_dict)
check_syns(gid, morph_dict, this_syn_stats,
seg_density_per_sec, layer_set_dict, swc_set_dict,
env, logger)
del cell
num_syns = len(synapse_dict[gid]['syn_ids'])
logger.info(
'Rank %i took %i s to compute %d synapse locations for %s gid: %i'
% (rank, time.time() - local_time, num_syns, population,
gid))
logger.info(
'%s gid %i synapses: %s' %
(population, gid, local_syn_summary(this_syn_stats)))
gid_count += 1
else:
logger.info('Rank %i gid is None' % rank)
if (not dry_run) and (gid_count % write_size == 0):
append_cell_attributes(output_path,
population,
synapse_dict,
namespace='Synapse Attributes',
comm=comm,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size,
cache_size=cache_size)
synapse_dict = {}
gc.collect()
syn_stats[population] = syn_stats_dict
count += 1
if not dry_run:
append_cell_attributes(output_path,
population,
synapse_dict,
namespace='Synapse Attributes',
comm=comm,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size,
cache_size=cache_size)
global_count = comm.gather(gid_count, root=0)
if gid_count > 0:
color = 1
else:
color = 0
comm0 = comm.Split(color, 0)
if color == 1:
summary = global_syn_summary(comm0,
syn_stats,
np.sum(global_count),
root=0)
if rank == 0:
logger.info(
'target: %s, %i ranks took %i s to compute synapse locations for %i cells'
% (population, comm.size, time.time() - start_time,
np.sum(global_count)))
logger.info(summary)
comm0.Free()
comm.barrier()
MPI.Finalize()
def main(population, forest_path, output_path, index_path, types_path,
index_namespace, coords_namespace, sample_count, io_size, chunk_size,
value_chunk_size, verbose):
"""
:param population: str
:param forest_path: str (path)
:param output_path: str (path)
:param index_path: str (path)
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param verbose: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
random.seed(13)
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(types_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(forest_pop_ranges, _) = read_population_ranges(forest_path)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
(pop_ranges, _) = read_population_ranges(output_path)
(population_start, population_count) = pop_ranges[population]
if rank == 0:
logger.info('reading new cell index map...')
reindex_map1 = {}
reindex_map_gen = bcast_cell_attributes(index_path,
population,
namespace=index_namespace,
root=0,
comm=comm)
for gid, attr_dict in reindex_map_gen:
reindex_map1[gid] = attr_dict['New Cell Index'][0]
if rank == 0:
logger.info('reading cell coordinates...')
old_coords_dict = {}
coords_map_gen = bcast_cell_attributes(index_path,
population,
namespace=coords_namespace,
root=0,
comm=comm)
for gid, attr_dict in coords_map_gen:
old_coords_dict[gid] = attr_dict
gc.collect()
if rank == 0:
logger.info('sampling cell population reindex...')
N = len(reindex_map1)
if sample_count is None:
sample_count = min(population_count, N)
else:
sample_count = min(sample_count, N)
reindex_map = None
if rank == 0:
reindex_map = {}
reindex_map = dict(
random_subset(utils.viewitems(reindex_map1), sample_count))
reindex_map = comm.bcast(reindex_map, root=0)
if rank == 0:
logger.info('computing new population index...')
gid_map = {
k: i + population_start
for i, k in enumerate(sorted(reindex_map.keys()))
}
new_coords_dict = {}
new_trees_dict = {}
for gid, old_trees_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=False):
if gid is not None and gid in reindex_map:
reindex_gid = reindex_map[gid]
new_gid = gid_map[gid]
new_trees_dict[new_gid] = old_trees_dict
new_coords_dict[new_gid] = old_coords_dict[gid]
logger.info('Rank: %i mapping old gid: %i to new gid: %i' %
(rank, gid, new_gid))
append_cell_trees(output_path,
population,
new_trees_dict,
io_size=io_size,
comm=comm)
append_cell_attributes(output_path, population, new_coords_dict, \
namespace=coords_namespace, io_size=io_size, comm=comm)
comm.barrier()
if rank == 0:
logger.info('Appended reindexed trees to %s' % output_path)