def main(coords_path, coords_namespace):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
print ('Allocated %i ranks' % size)
sys.stdout.flush()
population_ranges = read_population_ranges(coords_path)[0]
soma_coords = {}
for population in ['GC']:
attr_iter = bcast_cell_attributes(coords_path, population, namespace=coords_namespace,
root=0, mask=set(['U Coordinate', 'V Coordinate', 'L Coordinate']),
comm=comm)
for cell_gid, coords_dict in attr_iter:
cell_u = coords_dict['U Coordinate']
cell_v = coords_dict['V Coordinate']
print ('Rank %i: gid = %i u = %f v = %f' % (rank, cell_gid, cell_u, cell_v))
def main(config, coords_path, coords_namespace, distances_namespace,
populations, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config)
output_path = coords_path
soma_coords = {}
soma_distances = {}
if rank == 0:
logger.info('Reading population coordinates and distances...')
for population in populations:
coords = bcast_cell_attributes(coords_path,
population,
0,
namespace=coords_namespace,
comm=comm)
soma_coords[population] = {
k:
(v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0])
for (k, v) in coords
}
del coords
gc.collect()
distances = bcast_cell_attributes(coords_path,
population,
0,
namespace=distances_namespace,
comm=comm)
soma_distances = {
k: (v['U Distance'][0], v['V Distance'][0])
for (k, v) in distances
}
del distances
gc.collect()
def main(config, coords_path, coords_namespace, resample, resolution, populations, projection_depth, io_size, chunk_size, value_chunk_size, cache_size, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(script_name)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config)
soma_coords = {}
if rank == 0:
logger.info('Reading population coordinates...')
rotate = env.geometry['Parametric Surface']['Rotation']
min_l = float('inf')
max_l = 0.0
population_ranges = read_population_ranges(coords_path)[0]
population_extents = {}
for population in population_ranges:
min_extent = env.geometry['Cell Layers']['Minimum Extent'][population]
max_extent = env.geometry['Cell Layers']['Maximum Extent'][population]
min_l = min(min_extent[2], min_l)
max_l = max(max_extent[2], max_l)
population_extents[population] = (min_extent, max_extent)
for population in populations:
coords = bcast_cell_attributes(coords_path, population, 0, \
namespace=coords_namespace)
soma_coords[population] = { k: (v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0]) for (k,v) in coords }
del coords
gc.collect()
output_path = coords_path
soma_coords = icp_transform(comm, soma_coords, projection_depth, population_extents, \
populations=populations, rotate=rotate, verbose=verbose)
for population in populations:
if rank == 0:
logger.info('Writing transformed coordinates for population %s...' % population)
append_cell_attributes(output_path, population, soma_coords[population],
namespace='Soma Projections', comm=comm,
io_size=io_size, chunk_size=chunk_size,
value_chunk_size=value_chunk_size, cache_size=cache_size)
def main(coords_path, io_size, chunk_size, value_chunk_size):
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config)
output_path = coords_path
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
source_population_ranges = read_population_ranges(coords_path)
source_populations = list(source_population_ranges.keys())
for population in source_populations:
if rank == 0:
logger.info('population: ',population)
soma_coords = bcast_cell_attributes(0, coords_path, population,
namespace='Interpolated Coordinates', comm=comm)
#print soma_coords.keys()
u_coords = []
gids = []
for gid, attrs in viewitems(soma_coords):
u_coords.append(attrs['U Coordinate'])
gids.append(gid)
u_coordv = np.asarray(u_coords, dtype=np.float32)
gidv = np.asarray(gids, dtype=np.uint32)
sort_idx = np.argsort(u_coordv, axis=0)
offset = source_population_ranges[population][0]
sorted_coords_dict = {}
for i in range(0,sort_idx.size):
sorted_coords_dict[offset+i] = soma_coords[gidv[sort_idx[i][0]]]
append_cell_attributes(coords_path, population, sorted_coords_dict,
namespace='Sorted Coordinates', io_size=io_size, chunk_size=chunk_size,
value_chunk_size=value_chunk_size, comm=comm)
def main(config, config_prefix, coords_path, distances_namespace, bin_distance,
selectivity_path, selectivity_namespace, subset_seed, arena_id,
populations, io_size, cache_size, verbose, debug, show_fig, save_fig,
save_fig_dir, font_size, fig_size, colormap, fig_format):
"""
:param config: str (.yaml file name)
:param config_prefix: str (path to dir)
:param coords_path: str (path to file)
:param distances_namespace: str
:param bin_distance: float
:param selectivity_path: str
:param subset_seed: int; for reproducible choice of gids to plot individual rate maps
:param arena_id: str
:param populations: tuple of str
:param io_size: int
:param cache_size: int
:param verbose: bool
:param debug: bool
:param show_fig: bool
:param save_fig: str (base file name)
:param save_fig_dir: str (path to dir)
:param font_size: float
:param fig_format: str
"""
comm = MPI.COMM_WORLD
rank = comm.rank
config_logging(verbose)
env = Env(comm=comm,
config_file=config,
config_prefix=config_prefix,
template_paths=None)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
fig_options = copy.copy(default_fig_options)
fig_options.saveFigDir = save_fig_dir
fig_options.fontSize = font_size
fig_options.figFormat = fig_format
fig_options.showFig = show_fig
fig_options.figSize = fig_size
if save_fig is not None:
save_fig = '%s %s' % (save_fig, arena_id)
fig_options.saveFig = save_fig
population_ranges = read_population_ranges(selectivity_path, comm)[0]
coords_population_ranges = read_population_ranges(coords_path, comm)[0]
if len(populations) == 0:
populations = ('MC', 'ConMC', 'LPP', 'GC', 'MPP', 'CA3c')
valid_selectivity_namespaces = dict()
if rank == 0:
for population in populations:
if population not in population_ranges:
raise RuntimeError(
'plot_input_selectivity_features: specified population: %s not found in '
'provided selectivity_path: %s' %
(population, selectivity_path))
if population not in env.stimulus_config[
'Selectivity Type Probabilities']:
raise RuntimeError(
'plot_input_selectivity_features: selectivity type not specified for '
'population: %s' % population)
valid_selectivity_namespaces[population] = []
with h5py.File(selectivity_path, 'r') as selectivity_f:
for this_namespace in selectivity_f['Populations'][population]:
if f'{selectivity_namespace} {arena_id}' in this_namespace:
valid_selectivity_namespaces[population].append(
this_namespace)
if len(valid_selectivity_namespaces[population]) == 0:
raise RuntimeError(
'plot_input_selectivity_features: no selectivity data in arena: %s found '
'for specified population: %s in provided selectivity_path: %s'
% (arena_id, population, selectivity_path))
valid_selectivity_namespaces = comm.bcast(valid_selectivity_namespaces,
root=0)
selectivity_type_names = dict(
(val, key) for (key, val) in viewitems(env.selectivity_types))
reference_u_arc_distance_bounds = None
reference_v_arc_distance_bounds = None
if rank == 0:
for population in populations:
if population not in coords_population_ranges:
raise RuntimeError(
'plot_input_selectivity_features: specified population: %s not found in '
'provided coords_path: %s' % (population, coords_path))
with h5py.File(coords_path, 'r') as coords_f:
pop_size = population_ranges[population][1]
unique_gid_count = len(
set(coords_f['Populations'][population]
[distances_namespace]['U Distance']['Cell Index'][:]))
if pop_size != unique_gid_count:
raise RuntimeError(
'plot_input_selectivity_features: only %i/%i unique cell indexes found '
'for specified population: %s in provided coords_path: %s'
%
(unique_gid_count, pop_size, population, coords_path))
if reference_u_arc_distance_bounds is None:
try:
reference_u_arc_distance_bounds = \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Min'], \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Max']
except Exception:
raise RuntimeError(
'plot_input_selectivity_features: problem locating attributes '
'containing reference bounds in namespace: %s for population: %s from '
'coords_path: %s' %
(distances_namespace, population, coords_path))
if reference_v_arc_distance_bounds is None:
try:
reference_v_arc_distance_bounds = \
coords_f['Populations'][population][distances_namespace].attrs['Reference V Min'], \
coords_f['Populations'][population][distances_namespace].attrs['Reference V Max']
except Exception:
raise RuntimeError(
'plot_input_selectivity_features: problem locating attributes '
'containing reference bounds in namespace: %s for population: %s from '
'coords_path: %s' %
(distances_namespace, population, coords_path))
reference_u_arc_distance_bounds = comm.bcast(
reference_u_arc_distance_bounds, root=0)
reference_v_arc_distance_bounds = comm.bcast(
reference_v_arc_distance_bounds, root=0)
u_edges = np.arange(reference_u_arc_distance_bounds[0],
reference_u_arc_distance_bounds[1] + bin_distance / 2.,
bin_distance)
v_edges = np.arange(reference_v_arc_distance_bounds[0],
reference_v_arc_distance_bounds[1] + bin_distance / 2.,
bin_distance)
if arena_id not in env.stimulus_config['Arena']:
raise RuntimeError(
'Arena with ID: %s not specified by configuration at file path: %s'
% (arena_id, config_prefix + '/' + config))
arena = env.stimulus_config['Arena'][arena_id]
arena_x_mesh, arena_y_mesh = None, None
if rank == 0:
arena_x_mesh, arena_y_mesh = \
get_2D_arena_spatial_mesh(arena=arena, spatial_resolution=env.stimulus_config['Spatial Resolution'])
arena_x_mesh = comm.bcast(arena_x_mesh, root=0)
arena_y_mesh = comm.bcast(arena_y_mesh, root=0)
for population in populations:
start_time = time.time()
u_distances_by_gid = dict()
v_distances_by_gid = dict()
distances_attr_gen = \
bcast_cell_attributes(coords_path, population, root=0, namespace=distances_namespace, comm=comm)
for gid, distances_attr_dict in distances_attr_gen:
u_distances_by_gid[gid] = distances_attr_dict['U Distance'][0]
v_distances_by_gid[gid] = distances_attr_dict['V Distance'][0]
if rank == 0:
logger.info(
'Reading %i cell positions for population %s took %.2f s' %
(len(u_distances_by_gid), population,
time.time() - start_time))
for this_selectivity_namespace in valid_selectivity_namespaces[
population]:
start_time = time.time()
if rank == 0:
logger.info('Reading from %s namespace for population %s...' %
(this_selectivity_namespace, population))
gid_count = 0
gathered_cell_attributes = defaultdict(list)
gathered_component_attributes = defaultdict(list)
u_distances_by_cell = list()
v_distances_by_cell = list()
u_distances_by_component = list()
v_distances_by_component = list()
rate_map_sum_by_module = defaultdict(
lambda: np.zeros_like(arena_x_mesh))
start_time = time.time()
selectivity_attr_gen = NeuroH5CellAttrGen(
selectivity_path,
population,
namespace=this_selectivity_namespace,
comm=comm,
io_size=io_size,
cache_size=cache_size)
for iter_count, (
gid,
selectivity_attr_dict) in enumerate(selectivity_attr_gen):
if gid is not None:
gid_count += 1
this_selectivity_type = selectivity_attr_dict[
'Selectivity Type'][0]
this_selectivity_type_name = selectivity_type_names[
this_selectivity_type]
input_cell_config = \
get_input_cell_config(selectivity_type=this_selectivity_type,
selectivity_type_names=selectivity_type_names,
selectivity_attr_dict=selectivity_attr_dict)
rate_map = input_cell_config.get_rate_map(x=arena_x_mesh,
y=arena_y_mesh)
u_distances_by_cell.append(u_distances_by_gid[gid])
v_distances_by_cell.append(v_distances_by_gid[gid])
this_cell_attrs, component_count, this_component_attrs = input_cell_config.gather_attributes(
)
for attr_name, attr_val in viewitems(this_cell_attrs):
gathered_cell_attributes[attr_name].append(attr_val)
gathered_cell_attributes['Mean Rate'].append(
np.mean(rate_map))
if component_count > 0:
u_distances_by_component.extend(
[u_distances_by_gid[gid]] * component_count)
v_distances_by_component.extend(
[v_distances_by_gid[gid]] * component_count)
for attr_name, attr_val in viewitems(
this_component_attrs):
gathered_component_attributes[attr_name].extend(
attr_val)
this_module_id = this_cell_attrs['Module ID']
if debug and rank == 0:
fig_title = '%s %s cell %i' % (
population, this_selectivity_type_name, gid)
if save_fig is not None:
fig_options.saveFig = '%s %s' % (save_fig,
fig_title)
plot_2D_rate_map(
x=arena_x_mesh,
y=arena_y_mesh,
rate_map=rate_map,
peak_rate=env.stimulus_config['Peak Rate']
[population][this_selectivity_type],
title='%s\nModule: %i' %
(fig_title, this_module_id),
**fig_options())
rate_map_sum_by_module[this_module_id] = np.add(
rate_map, rate_map_sum_by_module[this_module_id])
if debug and iter_count >= 10:
break
cell_count_hist, _, _ = np.histogram2d(u_distances_by_cell,
v_distances_by_cell,
bins=[u_edges, v_edges])
component_count_hist, _, _ = np.histogram2d(
u_distances_by_component,
v_distances_by_component,
bins=[u_edges, v_edges])
if debug:
context.update(locals())
gathered_cell_attr_hist = dict()
gathered_component_attr_hist = dict()
for key in gathered_cell_attributes:
gathered_cell_attr_hist[key], _, _ = \
np.histogram2d(u_distances_by_cell, v_distances_by_cell, bins=[u_edges, v_edges],
weights=gathered_cell_attributes[key])
for key in gathered_component_attributes:
gathered_component_attr_hist[key], _, _ = \
np.histogram2d(u_distances_by_component, v_distances_by_component, bins=[u_edges, v_edges],
weights=gathered_component_attributes[key])
gid_count = comm.gather(gid_count, root=0)
cell_count_hist = comm.gather(cell_count_hist, root=0)
component_count_hist = comm.gather(component_count_hist, root=0)
gathered_cell_attr_hist = comm.gather(gathered_cell_attr_hist,
root=0)
gathered_component_attr_hist = comm.gather(
gathered_component_attr_hist, root=0)
rate_map_sum_by_module = dict(rate_map_sum_by_module)
rate_map_sum_by_module = comm.gather(rate_map_sum_by_module,
root=0)
if rank == 0:
gid_count = sum(gid_count)
cell_count_hist = np.sum(cell_count_hist, axis=0)
component_count_hist = np.sum(component_count_hist, axis=0)
merged_cell_attr_hist = defaultdict(
lambda: np.zeros_like(cell_count_hist))
merged_component_attr_hist = defaultdict(
lambda: np.zeros_like(component_count_hist))
for each_cell_attr_hist in gathered_cell_attr_hist:
for key in each_cell_attr_hist:
merged_cell_attr_hist[key] = np.add(
merged_cell_attr_hist[key],
each_cell_attr_hist[key])
for each_component_attr_hist in gathered_component_attr_hist:
for key in each_component_attr_hist:
merged_component_attr_hist[key] = np.add(
merged_component_attr_hist[key],
each_component_attr_hist[key])
merged_rate_map_sum_by_module = defaultdict(
lambda: np.zeros_like(arena_x_mesh))
for each_rate_map_sum_by_module in rate_map_sum_by_module:
for this_module_id in each_rate_map_sum_by_module:
merged_rate_map_sum_by_module[this_module_id] = \
np.add(merged_rate_map_sum_by_module[this_module_id],
each_rate_map_sum_by_module[this_module_id])
logger.info('Processing %i %s %s cells took %.2f s' %
(gid_count, population, this_selectivity_type_name,
time.time() - start_time))
if debug:
context.update(locals())
for key in merged_cell_attr_hist:
fig_title = '%s %s cells %s distribution' % (
population, this_selectivity_type_name, key)
if save_fig is not None:
fig_options.saveFig = '%s %s' % (save_fig, fig_title)
if colormap is not None:
fig_options.colormap = colormap
title = '%s %s cells\n%s distribution' % (
population, this_selectivity_type_name, key)
fig = plot_2D_histogram(
merged_cell_attr_hist[key],
x_edges=u_edges,
y_edges=v_edges,
norm=cell_count_hist,
ylabel='Transverse position (um)',
xlabel='Septo-temporal position (um)',
title=title,
cbar_label='Mean value per bin',
cbar=True,
**fig_options())
close_figure(fig)
for key in merged_component_attr_hist:
fig_title = '%s %s cells %s distribution' % (
population, this_selectivity_type_name, key)
if save_fig is not None:
fig_options.saveFig = '%s %s' % (save_fig, fig_title)
title = '%s %s cells\n%s distribution' % (
population, this_selectivity_type_name, key)
fig = plot_2D_histogram(
merged_component_attr_hist[key],
x_edges=u_edges,
y_edges=v_edges,
norm=component_count_hist,
ylabel='Transverse position (um)',
xlabel='Septo-temporal position (um)',
title=title,
cbar_label='Mean value per bin',
cbar=True,
**fig_options())
close_figure(fig)
for this_module_id in merged_rate_map_sum_by_module:
fig_title = '%s %s Module %i summed rate maps' % \
(population, this_selectivity_type_name, this_module_id)
if save_fig is not None:
fig_options.saveFig = '%s %s' % (save_fig, fig_title)
fig = plot_2D_rate_map(
x=arena_x_mesh,
y=arena_y_mesh,
rate_map=merged_rate_map_sum_by_module[this_module_id],
title='%s %s summed rate maps\nModule %i' %
(population, this_selectivity_type_name,
this_module_id),
**fig_options())
close_figure(fig)
if is_interactive and rank == 0:
context.update(locals())
def main(forest_path, connectivity_namespace, coords_path, coords_namespace, io_size, chunk_size, value_chunk_size,
cache_size):
"""
:param forest_path:
:param connectivity_namespace:
:param coords_path:
:param coords_namespace:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
"""
comm = MPI.COMM_WORLD
rank = comm.rank # The process ID (integer 0-3 for 4-process run)
if io_size == -1:
io_size = comm.size
if rank == 0:
print('%i ranks have been allocated' % comm.size)
sys.stdout.flush()
start_time = time.time()
soma_coords = {}
source_populations = list(read_population_ranges(MPI._addressof(comm), coords_path).keys())
for population in source_populations:
soma_coords[population] = bcast_cell_attributes(MPI._addressof(comm), 0, coords_path, population,
namespace=coords_namespace)
for population in soma_coords:
for cell in viewvalues(soma_coords[population]):
cell['u_index'] = get_array_index(u, cell['U Coordinate'][0])
cell['v_index'] = get_array_index(v, cell['V Coordinate'][0])
target = 'GC'
layer_set, swc_type_set, syn_type_set = set(), set(), set()
for source in layers[target]:
layer_set.update(layers[target][source])
swc_type_set.update(swc_types[target][source])
syn_type_set.update(syn_types[target][source])
count = 0
for target_gid, attributes_dict in NeuroH5CellAttrGen(MPI._addressof(comm), forest_path, target, io_size=io_size,
cache_size=cache_size, namespace='Synapse_Attributes'):
last_time = time.time()
connection_dict = {}
p_dict = {}
source_gid_dict = {}
if target_gid is None:
print('Rank %i target gid is None' % rank)
else:
print('Rank %i received attributes for target: %s, gid: %i' % (rank, target, target_gid))
synapse_dict = attributes_dict['Synapse_Attributes']
connection_dict[target_gid] = {}
local_np_random.seed(target_gid + connectivity_seed_offset)
connection_dict[target_gid]['source_gid'] = np.array([], dtype='uint32')
connection_dict[target_gid]['syn_id'] = np.array([], dtype='uint32')
for layer in layer_set:
for swc_type in swc_type_set:
for syn_type in syn_type_set:
sources, this_proportions = filter_sources(target, layer, swc_type, syn_type)
if sources:
if rank == 0 and count == 0:
source_list_str = '[' + ', '.join(['%s' % xi for xi in sources]) + ']'
print('Connections to target: %s in layer: %i ' \
'(swc_type: %i, syn_type: %i): %s' % \
(target, layer, swc_type, syn_type, source_list_str))
p, source_gid = np.array([]), np.array([])
for source, this_proportion in zip(sources, this_proportions):
if source not in source_gid_dict:
this_p, this_source_gid = p_connect.get_p(target, source, target_gid, soma_coords,
distance_U, distance_V)
source_gid_dict[source] = this_source_gid
p_dict[source] = this_p
else:
this_source_gid = source_gid_dict[source]
this_p = p_dict[source]
p = np.append(p, this_p * this_proportion)
source_gid = np.append(source_gid, this_source_gid)
syn_indexes = filter_synapses(synapse_dict, layer, swc_type, syn_type)
connection_dict[target_gid]['syn_id'] = \
np.append(connection_dict[target_gid]['syn_id'],
synapse_dict['syn_id'][syn_indexes]).astype('uint32', copy=False)
this_source_gid = local_np_random.choice(source_gid, len(syn_indexes), p=p)
connection_dict[target_gid]['source_gid'] = \
np.append(connection_dict[target_gid]['source_gid'],
this_source_gid).astype('uint32', copy=False)
count += 1
print('Rank %i took %i s to compute connectivity for target: %s, gid: %i' % (rank, time.time() - last_time,
target, target_gid))
sys.stdout.flush()
last_time = time.time()
append_cell_attributes(MPI._addressof(comm), forest_path, target, connection_dict,
namespace=connectivity_namespace, io_size=io_size, chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
if rank == 0:
print('Appending connectivity attributes for target: %s took %i s' % (target, time.time() - last_time))
sys.stdout.flush()
del connection_dict
del p_dict
del source_gid_dict
gc.collect()
global_count = comm.gather(count, root=0)
if rank == 0:
print('%i ranks took took %i s to compute connectivity for %i cells' % (comm.size, time.time() - start_time,
np.sum(global_count)))
def main(config, config_prefix, selectivity_path, selectivity_namespace, coords_path, distances_namespace,
arena_id, populations, n_trials, io_size, chunk_size,
value_chunk_size, cache_size, write_size, output_path, spikes_namespace, spike_train_attr_name, phase_mod,
gather, debug, plot, show_fig, save_fig, save_fig_dir, font_size, fig_format,
verbose, dry_run):
"""
:param config: str (.yaml file name)
:param config_prefix: str (path to dir)
:param selectivity_path: str (path to file)
:param selectivity_namespace: str
:param arena_id: str
:param populations: str
:param n_trials: int
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param cache_size: int
:param write_size: int
:param output_path: str (path to file)
:param spikes_namespace: str
:param spike_train_attr_name: str
:param gather: bool
:param debug: bool
:param plot: bool
:param show_fig: bool
:param save_fig: str (base file name)
:param save_fig_dir: str (path to dir)
:param font_size: float
:param fig_format: str
:param verbose: bool
:param dry_run: bool
"""
comm = MPI.COMM_WORLD
rank = comm.rank
config_logging(verbose)
if phase_mod and (coords_path is None):
raise RuntimeError("generate_input_spike_trains: when phase_mod is True, coords_path is required")
env = Env(comm=comm, config_file=config, config_prefix=config_prefix, template_paths=None)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
if save_fig is not None:
plot = True
if plot:
from dentate.plot import default_fig_options
fig_options = copy.copy(default_fig_options)
fig_options.saveFigDir = save_fig_dir
fig_options.fontSize = font_size
fig_options.figFormat = fig_format
fig_options.showFig = show_fig
population_ranges = read_population_ranges(selectivity_path, comm)[0]
if len(populations) == 0:
populations = sorted(population_ranges.keys())
soma_positions_dict = None
if coords_path is not None:
soma_positions_dict = {}
for population in populations:
reference_u_arc_distance_bounds = None
if rank == 0:
with h5py.File(coords_path, 'r') as coords_f:
reference_u_arc_distance_bounds = \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Min'], \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Max']
comm.barrier()
reference_u_arc_distance_bounds = comm.bcast(reference_u_arc_distance_bounds, root=0)
distances = bcast_cell_attributes(coords_path, population, namespace=distances_namespace, root=0)
abs_positions = { k: v['U Distance'][0] - reference_u_arc_distance_bounds[0] for (k,v) in distances }
soma_positions_dict[population] = abs_positions
del distances
if arena_id not in env.stimulus_config['Arena']:
raise RuntimeError('Arena with ID: %s not specified by configuration at file path: %s' %
(arena_id, config_prefix + '/' + config))
arena = env.stimulus_config['Arena'][arena_id]
valid_selectivity_namespaces = dict()
if rank == 0:
for population in populations:
if population not in population_ranges:
raise RuntimeError('generate_input_spike_trains: specified population: %s not found in '
'provided selectivity_path: %s' % (population, selectivity_path))
if population not in env.stimulus_config['Selectivity Type Probabilities']:
raise RuntimeError('generate_input_spike_trains: selectivity type not specified for '
'population: %s' % population)
valid_selectivity_namespaces[population] = []
with h5py.File(selectivity_path, 'r') as selectivity_f:
for this_namespace in selectivity_f['Populations'][population]:
if 'Selectivity %s' % arena_id in this_namespace:
valid_selectivity_namespaces[population].append(this_namespace)
if len(valid_selectivity_namespaces[population]) == 0:
raise RuntimeError('generate_input_spike_trains: no selectivity data in arena: %s found '
'for specified population: %s in provided selectivity_path: %s' %
(arena_id, population, selectivity_path))
comm.barrier()
valid_selectivity_namespaces = comm.bcast(valid_selectivity_namespaces, root=0)
selectivity_type_names = dict((val, key) for (key, val) in viewitems(env.selectivity_types))
equilibrate = get_equilibration(env)
for trajectory_id in sorted(arena.trajectories.keys()):
trajectory = arena.trajectories[trajectory_id]
t, x, y, d = None, None, None, None
if rank == 0:
t, x, y, d = generate_linear_trajectory(trajectory,
temporal_resolution=env.stimulus_config['Temporal Resolution'],
equilibration_duration=env.stimulus_config['Equilibration Duration'])
t = comm.bcast(t, root=0)
x = comm.bcast(x, root=0)
y = comm.bcast(y, root=0)
d = comm.bcast(d, root=0)
trajectory = t, x, y, d
trajectory_namespace = 'Trajectory %s %s' % (arena_id, trajectory_id)
output_namespace = '%s %s %s' % (spikes_namespace, arena_id, trajectory_id)
if not dry_run and rank == 0:
if output_path is None:
raise RuntimeError('generate_input_spike_trains: missing output_path')
if not os.path.isfile(output_path):
with h5py.File(output_path, 'w') as output_file:
input_file = h5py.File(selectivity_path, 'r')
input_file.copy('/H5Types', output_file)
input_file.close()
with h5py.File(output_path, 'a') as f:
if trajectory_namespace not in f:
logger.info('Appending %s datasets to file at path: %s' % (trajectory_namespace, output_path))
group = f.create_group(trajectory_namespace)
for key, value in zip(['t', 'x', 'y', 'd'], [t, x, y, d]):
dataset = group.create_dataset(key, data=value, dtype='float32')
else:
loaded_t = f[trajectory_namespace]['t'][:]
if len(t) != len(loaded_t):
raise RuntimeError('generate_input_spike_trains: file at path: %s already contains the '
'namespace: %s, but the dataset sizes are inconsistent with the provided input'
'configuration' % (output_path, trajectory_namespace))
comm.barrier()
if rank == 0:
context.update(locals())
spike_hist_sum_dict = {}
spike_hist_resolution = 1000
write_every = max(1, int(math.floor(write_size / comm.size)))
for population in populations:
req = comm.Ibarrier()
gc.collect()
req.wait()
pop_start = int(population_ranges[population][0])
num_cells = int(population_ranges[population][1])
phase_mod_config_dict = None
if phase_mod:
phase_mod_config_dict = oscillation_phase_mod_config(env, population, soma_positions_dict[population])
this_spike_hist_sum = defaultdict(lambda: np.zeros(spike_hist_resolution))
process_time = dict()
for this_selectivity_namespace in sorted(valid_selectivity_namespaces[population]):
if rank == 0:
logger.info('Generating input source spike trains for population %s [%s]...' % (population, this_selectivity_namespace))
start_time = time.time()
req = comm.Ibarrier()
selectivity_attr_gen = NeuroH5CellAttrGen(selectivity_path, population,
namespace=this_selectivity_namespace,
comm=comm, io_size=io_size,
cache_size=cache_size)
req.wait()
spikes_attr_dict = dict()
gid_count = 0
for iter_count, (gid, selectivity_attr_dict) in enumerate(selectivity_attr_gen):
if gid is not None:
if rank == 0:
logger.info(f'Rank {rank}: generating spike trains for gid {gid}...')
context.update(locals())
phase_mod_config = None
if phase_mod_config_dict is not None:
phase_mod_config = phase_mod_config_dict[gid]
spikes_attr_dict[gid] = \
generate_input_spike_trains(env, population, selectivity_type_names, trajectory,
gid, selectivity_attr_dict, n_trials=n_trials,
spike_train_attr_name=spike_train_attr_name,
spike_hist_resolution=spike_hist_resolution,
equilibrate=equilibrate,
phase_mod_config=phase_mod_config,
spike_hist_sum=this_spike_hist_sum,
return_selectivity_features=False,
debug= (debug_callback, context) if debug else False)
gid_count += 1
if (iter_count > 0 and iter_count % write_every == 0) or (debug and iter_count == 10):
req = comm.Ibarrier()
total_gid_count = comm.reduce(gid_count, root=0, op=MPI.SUM)
if rank == 0:
logger.info('generated spike trains for %i %s cells' %
(total_gid_count, population))
req.wait()
req = comm.Ibarrier()
if not dry_run:
append_cell_attributes(output_path, population, spikes_attr_dict,
namespace=output_namespace, comm=comm, io_size=io_size,
chunk_size=chunk_size, value_chunk_size=value_chunk_size)
req.wait()
req = comm.Ibarrier()
del spikes_attr_dict
spikes_attr_dict = dict()
gc.collect()
req.wait()
if debug and iter_count == 10:
break
if not dry_run:
req = comm.Ibarrier()
append_cell_attributes(output_path, population, spikes_attr_dict,
namespace=output_namespace, comm=comm, io_size=io_size,
chunk_size=chunk_size, value_chunk_size=value_chunk_size)
req.wait()
req = comm.Ibarrier()
del spikes_attr_dict
spikes_attr_dict = dict()
req.wait()
process_time = time.time() - start_time
req = comm.Ibarrier()
total_gid_count = comm.reduce(gid_count, root=0, op=MPI.SUM)
if rank == 0:
logger.info('generated spike trains for %i %s cells in %.2f s' %
(total_gid_count, population, process_time))
req.wait()
if gather:
spike_hist_sum_dict[population] = this_spike_hist_sum
if gather:
this_spike_hist_sum = dict([(key, dict(val.items())) for key, val in viewitems(spike_hist_sum_dict)])
spike_hist_sum = comm.gather(this_spike_hist_sum, root=0)
if rank == 0:
merged_spike_hist_sum = defaultdict(lambda: defaultdict(lambda: np.zeros(spike_hist_resolution)))
for each_spike_hist_sum in spike_hist_sum:
for population in each_spike_hist_sum:
for selectivity_type_name in each_spike_hist_sum[population]:
merged_spike_hist_sum[population][selectivity_type_name] = \
np.add(merged_spike_hist_sum[population][selectivity_type_name],
each_spike_hist_sum[population][selectivity_type_name])
if plot:
if save_fig is not None:
fig_options.saveFig = save_fig
plot_summed_spike_psth(t, trajectory_id, selectivity_type_name, merged_spike_hist_sum,
spike_hist_resolution, fig_options)
if is_interactive and rank == 0:
context.update(locals())
def main(config, template_path, types_path, forest_path, connectivity_path,
connectivity_namespace, coords_path, coords_namespace, io_size,
chunk_size, value_chunk_size, cache_size, write_size, verbose,
dry_run):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config, template_paths=template_path)
configure_hoc_env(env)
gj_config = env.gapjunctions
gj_seed = int(env.model_config['Random Seeds']['Gap Junctions'])
soma_coords = {}
if (not dry_run) and (rank == 0):
if not os.path.isfile(connectivity_path):
input_file = h5py.File(types_path, 'r')
output_file = h5py.File(connectivity_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
population_ranges = read_population_ranges(coords_path)[0]
populations = sorted(population_ranges.keys())
if rank == 0:
logger.info('Reading population coordinates...')
soma_distances = {}
for population in populations:
coords_iter = bcast_cell_attributes(coords_path,
population,
0,
namespace=coords_namespace)
soma_coords[population] = {
k:
(v['X Coordinate'][0], v['Y Coordinate'][0], v['Z Coordinate'][0])
for (k, v) in coords_iter
}
gc.collect()
generate_gj_connections(env,
forest_path,
soma_coords,
gj_config,
gj_seed,
connectivity_namespace,
connectivity_path,
io_size,
chunk_size,
value_chunk_size,
cache_size,
dry_run=dry_run)
MPI.Finalize()
def main(stimulus_path, stimulus_namespace, weights_path,
initial_weights_namespace, structured_weights_namespace,
connections_path, io_size, chunk_size, value_chunk_size, cache_size,
trajectory_id, seed_offset, target_sparsity, debug):
"""
:param stimulus_path: str
:param stimulus_namespace: str
:param weights_path: str
:param initial_weights_namespace: str
:param structured_weights_namespace: str
:param connections_path: str
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param cache_size: int
:param trajectory_id: int
:param seed_offset: int
:param target_sparsity: float
:param debug: bool
"""
# make sure random seeds are not being reused for various types of stochastic sampling
seed_offset *= 2e6
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
print '%s: %i ranks have been allocated' % (script_name, comm.size)
sys.stdout.flush()
stimulus_namespace += ' ' + str(trajectory_id)
stimulus_attrs = {}
source_population_list = ['MPP', 'LPP']
for source in source_population_list:
stimulus_attr_gen = bcast_cell_attributes(comm,
0,
stimulus_path,
source,
namespace=stimulus_namespace)
stimulus_attrs[source] = {
gid: attr_dict
for gid, attr_dict in stimulus_attr_gen
}
trajectory_namespace = 'Trajectory %s' % str(trajectory_id)
arena_dimension = 100. # minimum distance from origin to boundary (cm)
default_run_vel = 30. # cm/s
spatial_resolution = 1. # cm
with h5py.File(stimulus_path, 'a', driver='mpio', comm=comm) as f:
if trajectory_namespace not in f:
print 'Rank: %i; Creating %s datasets' % (rank,
trajectory_namespace)
group = f.create_group(trajectory_namespace)
t, x, y, d = stimulus.generate_trajectory(
arena_dimension=arena_dimension,
velocity=default_run_vel,
spatial_resolution=spatial_resolution)
for key, value in zip(['x', 'y', 'd', 't'], [x, y, d, t]):
dataset = group.create_dataset(key, (value.shape[0], ),
dtype='float32')
with dataset.collective:
dataset[:] = value.astype('float32', copy=False)
else:
print 'Rank: %i; Reading %s datasets' % (rank,
trajectory_namespace)
group = f[trajectory_namespace]
dataset = group['x']
with dataset.collective:
x = dataset[:]
dataset = group['y']
with dataset.collective:
y = dataset[:]
dataset = group['d']
with dataset.collective:
d = dataset[:]
dataset = group['t']
with dataset.collective:
t = dataset[:]
plasticity_window_dur = 4. # s
plasticity_kernel_sigma = plasticity_window_dur * default_run_vel / 3. / np.sqrt(
2.) # cm
plasticity_kernel = lambda d, d_offset: np.exp(-(
(d - d_offset) / plasticity_kernel_sigma)**2.)
plasticity_kernel = np.vectorize(plasticity_kernel, excluded=[1])
max_plasticity_kernel_area = np.sum(plasticity_kernel(
d,
np.max(d) / 2.)) * spatial_resolution
target = 'GC'
pop_ranges, pop_size = read_population_ranges(comm, stimulus_path)
target_gid_offset = pop_ranges[target][0]
count = 0
structured_count = 0
start_time = time.time()
gid_index_map = get_cell_attributes_gid_index_map(
comm, weights_path, target, initial_weights_namespace)
connection_gen_list = []
for source in source_population_list:
connection_gen_list.append(
NeuroH5ProjectionGen(comm,
connections_path,
source,
target,
io_size=io_size,
cache_size=cache_size,
namespaces=['Synapses']))
maxiter = 100 if debug else None
for itercount, attr_gen_package in enumerate(
izip_longest(*connection_gen_list)):
local_time = time.time()
syn_weight_map = {}
source_syn_map = defaultdict(list)
syn_peak_index_map = {}
structured_weights_dict = {}
modulated_inputs = 0
source_gid_array = None
conn_attr_dict = None
target_gid = attr_gen_package[0][0]
if not all([
attr_gen_items[0] == target_gid
for attr_gen_items in attr_gen_package
]):
raise Exception(
'Rank: %i; target: %s; target_gid not matched across multiple attribute generators: %s'
% (rank, target,
[attr_gen_items[0] for attr_gen_items in attr_gen_package]))
sys.stdout.flush()
# else:
# print 'Rank: %i; received target: %s; target_gid: %s' % (rank, target, str(target_gid))
initial_weights_dict = get_cell_attributes_by_gid(
target_gid, comm, weights_path, gid_index_map, target,
initial_weights_namespace, target_gid_offset)
if target_gid is not None:
if initial_weights_dict is None:
raise Exception(
'Rank: %i; target: %s; target_gid: %s; get_cell_attributes_by_gid didn\'t work'
% (rank, target, str(target_gid)))
local_random.seed(int(target_gid + seed_offset))
syn_weight_map = dict(
zip(initial_weights_dict['syn_id'],
initial_weights_dict['weight']))
for this_target_gid, (source_gid_array,
conn_attr_dict) in attr_gen_package:
for i in xrange(len(source_gid_array)):
this_source_gid = source_gid_array[i]
this_syn_id = conn_attr_dict['Synapses'][0][i]
source_syn_map[this_source_gid].append(this_syn_id)
if local_random.uniform() <= target_sparsity:
modify_weights = True
peak_loc = local_random.choice(d)
this_plasticity_kernel = plasticity_kernel(d, peak_loc)
else:
modify_weights = False
for source in stimulus_attrs:
peak_rate = peak_rate_dict[source]
for this_source_gid in stimulus_attrs[source]:
peak_index = stimulus_attrs[source][this_source_gid][
'peak_index'][0]
if modify_weights:
norm_rate = stimulus_attrs[source][this_source_gid][
'rate'] / peak_rate
this_plasticity_signal = np.sum(np.multiply(norm_rate, this_plasticity_kernel)) * \
spatial_resolution / max_plasticity_kernel_area
delta_weight = 2. * this_plasticity_signal
else:
delta_weight = 0.
for this_syn_id in source_syn_map[this_source_gid]:
syn_peak_index_map[this_syn_id] = peak_index
if delta_weight >= 0.1:
modulated_inputs += 1
syn_weight_map[this_syn_id] += delta_weight
structured_weights_dict[target_gid - target_gid_offset] = \
{'syn_id': np.array(syn_peak_index_map.keys()).astype('uint32', copy=False),
'weight': np.array([syn_weight_map[syn_id] for syn_id in syn_peak_index_map]).astype('float32',
copy=False),
'peak_index': np.array(syn_peak_index_map.values()).astype('uint32', copy=False),
'structured': np.array([int(modify_weights)], dtype='uint32')}
if modify_weights:
print 'Rank %i; target: %s; gid %i; generated structured weights for %i/%i inputs in %.2f s' % \
(rank, target, target_gid, modulated_inputs, len(syn_weight_map), time.time() - local_time)
structured_count += 1
else:
print 'Rank %i; target: %s; gid %i; calculated input peak_locs for %i inputs in %.2f s (not selected ' \
'for structured weights)' % (rank, target, target_gid, len(syn_weight_map),
time.time() - local_time)
count += 1
else:
print 'Rank: %i received target_gid as None' % rank
if not debug:
append_cell_attributes(comm,
weights_path,
target,
structured_weights_dict,
namespace=structured_weights_namespace,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
sys.stdout.flush()
del syn_weight_map
del source_syn_map
del syn_peak_index_map
del structured_weights_dict
del modulated_inputs
del source_gid_array
del conn_attr_dict
gc.collect()
if debug:
comm.barrier()
if maxiter is not None and itercount > maxiter:
break
if debug:
print 'Rank: %i exited the loop' % rank
global_count = comm.gather(count, root=0)
global_structured_count = comm.gather(structured_count, root=0)
if rank == 0:
print 'target: %s; %i ranks processed %i cells (%i assigned structured weights) in %.2f s' % \
(target, comm.size, np.sum(global_count), np.sum(global_structured_count), time.time() - start_time)
def main(features_path, connectivity_path, connectivity_namespace, io_size,
chunk_size, value_chunk_size, cache_size, trajectory_id, debug):
"""
:param features_path:
:param connectivity_path:
:param connectivity_namespace:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
:param trajectory_id:
:param debug:
"""
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
print('%i ranks have been allocated' % comm.size)
sys.stdout.flush()
population_range_dict = read_population_ranges(comm, features_path)
features_dict = {}
for population in ['MPP', 'LPP']:
features_dict[population] = bcast_cell_attributes(
comm,
0,
features_path,
population,
namespace='Feature Selectivity')
arena_dimension = 100. # minimum distance from origin to boundary (cm)
run_vel = 30. # cm/s
spatial_resolution = 1. # cm
x = np.arange(-arena_dimension, arena_dimension, spatial_resolution)
y = np.arange(-arena_dimension, arena_dimension, spatial_resolution)
distance = np.insert(
np.cumsum(np.sqrt(np.sum(
[np.diff(x)**2., np.diff(y)**2.], axis=0))), 0, 0.)
interp_distance = np.arange(distance[0], distance[-1], spatial_resolution)
t = old_div(interp_distance, run_vel * 1000.) # ms
interp_x = np.interp(interp_distance, distance, x)
interp_y = np.interp(interp_distance, distance, y)
with h5py.File(features_path, 'a', driver='mpio', comm=comm) as f:
if 'Trajectories' not in f:
f.create_group('Trajectories')
if str(trajectory_id) not in f['Trajectories']:
f['Trajectories'].create_group(str(trajectory_id))
f['Trajectories'][str(trajectory_id)].create_dataset(
'x', dtype='float32', data=interp_x)
f['Trajectories'][str(trajectory_id)].create_dataset(
'y', dtype='float32', data=interp_y)
f['Trajectories'][str(trajectory_id)].create_dataset(
'd', dtype='float32', data=interp_distance)
f['Trajectories'][str(trajectory_id)].create_dataset(
't', dtype='float32', data=t)
x = f['Trajectories'][str(trajectory_id)]['x'][:]
y = f['Trajectories'][str(trajectory_id)]['y'][:]
d = f['Trajectories'][str(trajectory_id)]['d'][:]
prediction_namespace = 'Response Prediction ' + str(trajectory_id)
target_population = 'GC'
count = 0
start_time = time.time()
attr_gen = NeuroH5CellAttrGen(comm,
connectivity_path,
target_population,
io_size=io_size,
cache_size=cache_size,
namespace=connectivity_namespace)
if debug:
attr_gen_wrapper = (next(attr_gen) for i in range(2))
else:
attr_gen_wrapper = attr_gen
for gid, connectivity_dict in attr_gen_wrapper:
local_time = time.time()
source_gid_counts = {}
response_dict = {}
response = np.zeros_like(d, dtype='float32')
if gid is not None:
for population in ['MPP', 'LPP']:
indexes = np.where(
(connectivity_dict[connectivity_namespace]['source_gid'] >=
population_range_dict[population][0])
& (connectivity_dict[connectivity_namespace]['source_gid']
< population_range_dict[population][0] +
population_range_dict[population][1]))[0]
source_gid_counts[population] = \
Counter(connectivity_dict[connectivity_namespace]['source_gid'][indexes])
for population in ['MPP', 'LPP']:
for source_gid in (
source_gid
for source_gid in source_gid_counts[population]
if source_gid in features_dict[population]):
this_feature_dict = features_dict[population][source_gid]
selectivity_type = this_feature_dict['Selectivity Type'][0]
contact_count = source_gid_counts[population][source_gid]
if selectivity_type == selectivity_grid:
ori_offset = this_feature_dict['Grid Orientation'][0]
grid_spacing = this_feature_dict['Grid Spacing'][0]
x_offset = this_feature_dict['X Offset'][0]
y_offset = this_feature_dict['Y Offset'][0]
rate = np.vectorize(
grid_rate(grid_spacing, ori_offset, x_offset,
y_offset))
elif selectivity_type == selectivity_place_field:
field_width = this_feature_dict['Field Width'][0]
x_offset = this_feature_dict['X Offset'][0]
y_offset = this_feature_dict['Y Offset'][0]
rate = np.vectorize(
place_rate(field_width, x_offset, y_offset))
response = np.add(response,
contact_count * rate(x, y),
dtype='float32')
response_dict[gid] = {'waveform': response}
baseline = np.mean(response[np.where(
response <= np.percentile(response, 10.))[0]])
peak = np.mean(response[np.where(
response >= np.percentile(response, 90.))[0]])
modulation = 0. if peak <= 0.1 else old_div(
(peak - baseline), peak)
peak_index = np.where(response == np.max(response))[0][0]
response_dict[gid]['modulation'] = np.array([modulation],
dtype='float32')
response_dict[gid]['peak_index'] = np.array([peak_index],
dtype='uint32')
print('Rank %i: took %.2f s to compute predicted response for %s gid %i' % \
(rank, time.time() - local_time, target_population, gid))
count += 1
if not debug:
append_cell_attributes(comm,
features_path,
target_population,
response_dict,
namespace=prediction_namespace,
io_size=io_size,
chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
sys.stdout.flush()
del response
del response_dict
del source_gid_counts
gc.collect()
global_count = comm.gather(count, root=0)
if rank == 0:
print('%i ranks took %.2f s to compute selectivity parameters for %i %s cells' % \
(comm.size, time.time() - start_time, np.sum(global_count), target_population))
def main(connectivity_path, output_path, coords_path, distances_namespace,
destination, bin_size, cache_size, verbose):
"""
Measures vertex distribution with respect to septo-temporal distance
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
if rank == 0:
logger.info('reading %s distances...' % destination)
destination_soma_distances = bcast_cell_attributes(
coords_path,
destination,
namespace=distances_namespace,
comm=comm,
root=0)
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in destination_soma_distances:
destination_soma_distance_U[k] = v['U Distance'][0]
destination_soma_distance_V[k] = v['V Distance'][0]
del (destination_soma_distances)
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
for s in sources:
if rank == 0:
logger.info('reading %s distances...' % s)
source_soma_distances[s] = bcast_cell_attributes(
coords_path, s, namespace=distances_namespace, comm=comm, root=0)
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
for k, v in source_soma_distances[s]:
this_source_soma_distance_U[k] = v['U Distance'][0]
this_source_soma_distance_V[k] = v['V Distance'][0]
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
del (source_soma_distances)
logger.info('reading connections %s -> %s...' %
(str(sources), destination))
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
for prj_gen_tuple in utils.zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise Exception(
'destination %s: destination_gid %i not matched across multiple projection generators: %s'
% (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
logger.info('reading connections of gid %i' % destination_gid)
for (source, (this_destination_gid,
rest)) in zip(sources, prj_gen_tuple):
this_source_soma_distance_U = source_soma_distance_U[source]
this_source_soma_distance_V = source_soma_distance_V[source]
this_dist_bins = dist_bins[source]
this_dist_u_bins = dist_u_bins[source]
this_dist_v_bins = dist_v_bins[source]
(source_indexes, attr_dict) = rest
dst_U = destination_soma_distance_U[destination_gid]
dst_V = destination_soma_distance_V[destination_gid]
for source_gid in source_indexes:
dist_u = dst_U - this_source_soma_distance_U[source_gid]
dist_v = dst_V - this_source_soma_distance_V[source_gid]
dist = abs(dist_u) + abs(dist_v)
update_bins(this_dist_bins, bin_size, dist)
update_bins(this_dist_u_bins, bin_size, dist_u)
update_bins(this_dist_v_bins, bin_size, dist_v)
comm.barrier()
logger.info('merging distance dictionaries...')
add_bins_op = MPI.Op.Create(add_bins, commute=True)
for source in sources:
dist_bins[source] = comm.reduce(dist_bins[source],
op=add_bins_op,
root=0)
dist_u_bins[source] = comm.reduce(dist_u_bins[source],
op=add_bins_op,
root=0)
dist_v_bins[source] = comm.reduce(dist_v_bins[source],
op=add_bins_op,
root=0)
comm.barrier()
if rank == 0:
color = 1
else:
color = 0
## comm0 includes only rank 0
comm0 = comm.Split(color, 0)
if rank == 0:
if output_path is None:
output_path = connectivity_path
logger.info('writing output to %s...' % output_path)
#f = h5py.File(output_path, 'a', driver='mpio', comm=comm0)
#if 'Nodes' in f:
# nodes_grp = f['Nodes']
#else:
# nodes_grp = f.create_group('Nodes')
#grp = nodes_grp.create_group('Connectivity Distance Histogram')
#dst_grp = grp.create_group(destination)
for source in sources:
dist_histoCount, dist_bin_edges = finalize_bins(
dist_bins[source], bin_size)
dist_u_histoCount, dist_u_bin_edges = finalize_bins(
dist_u_bins[source], bin_size)
dist_v_histoCount, dist_v_bin_edges = finalize_bins(
dist_v_bins[source], bin_size)
np.savetxt('%s Distance U Bin Count.dat' % source,
dist_u_histoCount)
np.savetxt('%s Distance U Bin Edges.dat' % source,
dist_u_bin_edges)
np.savetxt('%s Distance V Bin Count.dat' % source,
dist_v_histoCount)
np.savetxt('%s Distance V Bin Edges.dat' % source,
dist_v_bin_edges)
np.savetxt('%s Distance Bin Count.dat' % source, dist_histoCount)
np.savetxt('%s Distance Bin Edges.dat' % source, dist_bin_edges)
#f.close()
comm.barrier()
from mpi4py import MPI
from neuroh5.io import read_population_ranges, bcast_cell_attributes
# import mkl
comm = MPI.COMM_WORLD
rank = comm.rank
if rank == 0:
print('%i ranks have been allocated' % comm.size)
sys.stdout.flush()
prefix = os.getenv("WORK")
stimulus_path = '%s/Full_Scale_Control/DG_input_spike_trains_20190724_compressed.h5' % prefix
stimulus_namespace = 'Input Spikes A Diag'
sources = ['MPP', 'LPP']
cell_attrs = {}
for source in sources:
if rank == 0:
print("reading attributes for %s" % source)
sys.stdout.flush()
cell_attr_gen = bcast_cell_attributes(stimulus_path,
source,
namespace=stimulus_namespace,
root=0,
comm=comm,
mask=set(["Spike Train"]))
cell_attrs[source] = {gid: attr_dict for gid, attr_dict in cell_attr_gen}
if rank == 0:
print(cell_attrs)
def main(population, forest_path, output_path, index_path, types_path,
index_namespace, coords_namespace, sample_count, io_size, chunk_size,
value_chunk_size, verbose):
"""
:param population: str
:param forest_path: str (path)
:param output_path: str (path)
:param index_path: str (path)
:param io_size: int
:param chunk_size: int
:param value_chunk_size: int
:param verbose: bool
"""
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
random.seed(13)
if rank == 0:
if not os.path.isfile(output_path):
input_file = h5py.File(types_path, 'r')
output_file = h5py.File(output_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
(forest_pop_ranges, _) = read_population_ranges(forest_path)
(forest_population_start,
forest_population_count) = forest_pop_ranges[population]
(pop_ranges, _) = read_population_ranges(output_path)
(population_start, population_count) = pop_ranges[population]
if rank == 0:
logger.info('reading new cell index map...')
reindex_map1 = {}
reindex_map_gen = bcast_cell_attributes(index_path,
population,
namespace=index_namespace,
root=0,
comm=comm)
for gid, attr_dict in reindex_map_gen:
reindex_map1[gid] = attr_dict['New Cell Index'][0]
if rank == 0:
logger.info('reading cell coordinates...')
old_coords_dict = {}
coords_map_gen = bcast_cell_attributes(index_path,
population,
namespace=coords_namespace,
root=0,
comm=comm)
for gid, attr_dict in coords_map_gen:
old_coords_dict[gid] = attr_dict
gc.collect()
if rank == 0:
logger.info('sampling cell population reindex...')
N = len(reindex_map1)
if sample_count is None:
sample_count = min(population_count, N)
else:
sample_count = min(sample_count, N)
reindex_map = None
if rank == 0:
reindex_map = {}
reindex_map = dict(
random_subset(utils.viewitems(reindex_map1), sample_count))
reindex_map = comm.bcast(reindex_map, root=0)
if rank == 0:
logger.info('computing new population index...')
gid_map = {
k: i + population_start
for i, k in enumerate(sorted(reindex_map.keys()))
}
new_coords_dict = {}
new_trees_dict = {}
for gid, old_trees_dict in NeuroH5TreeGen(forest_path,
population,
io_size=io_size,
comm=comm,
topology=False):
if gid is not None and gid in reindex_map:
reindex_gid = reindex_map[gid]
new_gid = gid_map[gid]
new_trees_dict[new_gid] = old_trees_dict
new_coords_dict[new_gid] = old_coords_dict[gid]
logger.info('Rank: %i mapping old gid: %i to new gid: %i' %
(rank, gid, new_gid))
append_cell_trees(output_path,
population,
new_trees_dict,
io_size=io_size,
comm=comm)
append_cell_attributes(output_path, population, new_coords_dict, \
namespace=coords_namespace, io_size=io_size, comm=comm)
comm.barrier()
if rank == 0:
logger.info('Appended reindexed trees to %s' % output_path)
def main(config, config_prefix, include, forest_path, connectivity_path,
connectivity_namespace, coords_path, coords_namespace,
synapses_namespace, distances_namespace, resolution,
interp_chunk_size, io_size, chunk_size, value_chunk_size, cache_size,
write_size, verbose, dry_run):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config, config_prefix=config_prefix)
configure_hoc_env(env)
connection_config = env.connection_config
extent = {}
soma_coords = {}
if (not dry_run) and (rank == 0):
if not os.path.isfile(connectivity_path):
input_file = h5py.File(coords_path, 'r')
output_file = h5py.File(connectivity_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
population_ranges = read_population_ranges(coords_path)[0]
populations = sorted(list(population_ranges.keys()))
soma_distances = {}
for population in populations:
if rank == 0:
logger.info('Reading %s population coordinates...' % population)
coords_iter = bcast_cell_attributes(coords_path,
population,
0,
namespace=coords_namespace)
distances_iter = bcast_cell_attributes(coords_path,
population,
0,
namespace=distances_namespace)
soma_coords[population] = {
k:
(v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0])
for (k, v) in coords_iter
}
distances = {
k: (v['U Distance'][0], v['V Distance'][0])
for (k, v) in distances_iter
}
if len(distances) > 0:
soma_distances[population] = distances
gc.collect()
forest_populations = sorted(read_population_names(forest_path))
if (include is None) or (len(include) == 0):
destination_populations = forest_populations
else:
destination_populations = []
for p in include:
if p in forest_populations:
destination_populations.append(p)
if rank == 0:
logger.info('Generating connectivity for populations %s...' %
str(destination_populations))
if len(soma_distances) == 0:
(origin_ranges, ip_dist_u,
ip_dist_v) = make_distance_interpolant(env,
resolution=resolution,
nsample=nsample)
ip_dist = (origin_ranges, ip_dist_u, ip_dist_v)
soma_distances = measure_distances(env,
soma_coords,
ip_dist,
resolution=resolution)
for destination_population in destination_populations:
if rank == 0:
logger.info(
'Generating connection probabilities for population %s...' %
destination_population)
connection_prob = ConnectionProb(destination_population, soma_coords, soma_distances, \
env.connection_extents)
synapse_seed = int(
env.model_config['Random Seeds']['Synapse Projection Partitions'])
connectivity_seed = int(env.model_config['Random Seeds']
['Distance-Dependent Connectivity'])
cluster_seed = int(
env.model_config['Random Seeds']['Connectivity Clustering'])
if rank == 0:
logger.info('Generating connections for population %s...' %
destination_population)
populations_dict = env.model_config['Definitions']['Populations']
generate_uv_distance_connections(comm,
populations_dict,
connection_config,
connection_prob,
forest_path,
synapse_seed,
connectivity_seed,
cluster_seed,
synapses_namespace,
connectivity_namespace,
connectivity_path,
io_size,
chunk_size,
value_chunk_size,
cache_size,
write_size,
dry_run=dry_run)
MPI.Finalize()
def vertex_distribution(connectivity_path,
coords_path,
distances_namespace,
destination,
sources,
bin_size=20.0,
cache_size=100,
comm=None):
"""
Obtain spatial histograms of source vertices connecting to a given destination population.
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
if comm is None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
if rank == 0:
logger.info('reading %s distances...' % destination)
destination_soma_distances = bcast_cell_attributes(
coords_path,
destination,
namespace=distances_namespace,
comm=comm,
root=0)
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in destination_soma_distances:
destination_soma_distance_U[k] = v['U Distance'][0]
destination_soma_distance_V[k] = v['V Distance'][0]
del (destination_soma_distances)
if sources == ():
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
for s in sources:
if rank == 0:
logger.info('reading %s distances...' % s)
source_soma_distances[s] = bcast_cell_attributes(
coords_path, s, namespace=distances_namespace, comm=comm, root=0)
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
for k, v in source_soma_distances[s]:
this_source_soma_distance_U[k] = v['U Distance'][0]
this_source_soma_distance_V[k] = v['V Distance'][0]
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
del (source_soma_distances)
if rank == 0:
logger.info('reading connections %s -> %s...' %
(str(sources), destination))
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
for prj_gen_tuple in zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if rank == 0 and destination_gid is not None:
logger.info('%d' % destination_gid)
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise RuntimeError(
'destination %s: destination_gid %i not matched across multiple projection generators: '
'%s' % (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
for (source, (this_destination_gid,
rest)) in zip_longest(sources, prj_gen_tuple):
this_source_soma_distance_U = source_soma_distance_U[source]
this_source_soma_distance_V = source_soma_distance_V[source]
this_dist_bins = dist_bins[source]
this_dist_u_bins = dist_u_bins[source]
this_dist_v_bins = dist_v_bins[source]
(source_indexes, attr_dict) = rest
dst_U = destination_soma_distance_U[destination_gid]
dst_V = destination_soma_distance_V[destination_gid]
for source_gid in source_indexes:
dist_u = dst_U - this_source_soma_distance_U[source_gid]
dist_v = dst_V - this_source_soma_distance_V[source_gid]
dist = abs(dist_u) + abs(dist_v)
update_bins(this_dist_bins, bin_size, dist)
update_bins(this_dist_u_bins, bin_size, dist_u)
update_bins(this_dist_v_bins, bin_size, dist_v)
add_bins_op = MPI.Op.Create(add_bins, commute=True)
for source in sources:
dist_bins[source] = comm.reduce(dist_bins[source], op=add_bins_op)
dist_u_bins[source] = comm.reduce(dist_u_bins[source], op=add_bins_op)
dist_v_bins[source] = comm.reduce(dist_v_bins[source], op=add_bins_op)
dist_hist_dict = defaultdict(dict)
dist_u_hist_dict = defaultdict(dict)
dist_v_hist_dict = defaultdict(dict)
if rank == 0:
for source in sources:
dist_hist_dict[destination][source] = finalize_bins(
dist_bins[source], bin_size)
dist_u_hist_dict[destination][source] = finalize_bins(
dist_u_bins[source], bin_size)
dist_v_hist_dict[destination][source] = finalize_bins(
dist_v_bins[source], bin_size)
return {
'Total distance': dist_hist_dict,
'U distance': dist_u_hist_dict,
'V distance': dist_v_hist_dict
}
def spatial_bin_graph(connectivity_path,
coords_path,
distances_namespace,
destination,
sources,
extents,
bin_size=20.0,
cache_size=100,
comm=None):
"""
Obtain reduced graphs of the specified projections by binning nodes according to their spatial position.
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
import networkx as nx
if comm is None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
if rank == 0:
logger.info('reading %s distances...' % destination)
destination_soma_distances = bcast_cell_attributes(
coords_path,
destination,
namespace=distances_namespace,
comm=comm,
root=0)
((x_min, x_max), (y_min, y_max)) = extents
u_bins = np.arange(x_min, x_max, bin_size)
v_bins = np.arange(y_min, y_max, bin_size)
dest_u_bins = {}
dest_v_bins = {}
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in destination_soma_distances:
dist_u = v['U Distance'][0]
dist_v = v['V Distance'][0]
dest_u_bins[k] = np.searchsorted(u_bins, dist_u, side='left')
dest_v_bins[k] = np.searchsorted(v_bins, dist_v, side='left')
destination_soma_distance_U[k] = dist_u
destination_soma_distance_V[k] = dist_v
del (destination_soma_distances)
if (sources == ()) or (sources == []) or (sources is None):
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
for s in sources:
if rank == 0:
logger.info('reading %s distances...' % s)
source_soma_distances[s] = bcast_cell_attributes(
coords_path, s, namespace=distances_namespace, comm=comm, root=0)
source_u_bins = {}
source_v_bins = {}
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
this_source_u_bins = {}
this_source_v_bins = {}
for k, v in source_soma_distances[s]:
dist_u = v['U Distance'][0]
dist_v = v['V Distance'][0]
this_source_u_bins[k] = np.searchsorted(u_bins,
dist_u,
side='left')
this_source_v_bins[k] = np.searchsorted(v_bins,
dist_v,
side='left')
this_source_soma_distance_U[k] = dist_u
this_source_soma_distance_V[k] = dist_v
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
source_u_bins[s] = this_source_u_bins
source_v_bins[s] = this_source_v_bins
del (source_soma_distances)
if rank == 0:
logger.info('reading connections %s -> %s...' %
(str(sources), destination))
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
local_u_bin_graph = defaultdict(dict)
local_v_bin_graph = defaultdict(dict)
for prj_gen_tuple in zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise RuntimeError(
'destination %s: destination_gid %i not matched across multiple projection generators: '
'%s' % (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
dest_u_bin = dest_u_bins[destination_gid]
dest_v_bin = dest_v_bins[destination_gid]
for (source, (this_destination_gid,
rest)) in zip_longest(sources, prj_gen_tuple):
this_source_u_bins = source_u_bins[source]
this_source_v_bins = source_v_bins[source]
(source_indexes, attr_dict) = rest
source_u_bin_dict = defaultdict(int)
source_v_bin_dict = defaultdict(int)
for source_gid in source_indexes:
source_u_bin = this_source_u_bins[source_gid]
source_v_bin = this_source_v_bins[source_gid]
source_u_bin_dict[source_u_bin] += 1
source_v_bin_dict[source_v_bin] += 1
local_u_bin_graph[dest_u_bin][source] = source_u_bin_dict
local_v_bin_graph[dest_v_bin][source] = source_v_bin_dict
local_u_bin_graphs = comm.gather(dict(local_u_bin_graph), root=0)
local_v_bin_graphs = comm.gather(dict(local_v_bin_graph), root=0)
u_bin_graph = None
v_bin_graph = None
nu = None
nv = None
if rank == 0:
u_bin_edges = {destination: dict(ChainMap(*local_u_bin_graphs))}
v_bin_edges = {destination: dict(ChainMap(*local_v_bin_graphs))}
nu = len(u_bins)
u_bin_graph = nx.Graph()
for pop in [destination] + list(sources):
for i in range(nu):
u_bin_graph.add_node((pop, i))
for i, ss in viewitems(u_bin_edges[destination]):
for source, ids in viewitems(ss):
u_bin_graph.add_weighted_edges_from([
((source, j), (destination, i), count)
for j, count in viewitems(ids)
])
nv = len(v_bins)
v_bin_graph = nx.Graph()
for pop in [destination] + list(sources):
for i in range(nv):
v_bin_graph.add_node((pop, i))
for i, ss in viewitems(v_bin_edges[destination]):
for source, ids in viewitems(ss):
v_bin_graph.add_weighted_edges_from([
((source, j), (destination, i), count)
for j, count in viewitems(ids)
])
label = '%s to %s' % (str(sources), destination)
return {
'label': label,
'bin size': bin_size,
'destination': destination,
'sources': sources,
'U graph': u_bin_graph,
'V graph': v_bin_graph
}
def main(config, config_prefix, coords_path, distances_namespace, bin_distance,
selectivity_path, selectivity_namespace, spatial_resolution, arena_id,
populations, io_size, cache_size, verbose, debug, show_fig, save_fig,
save_fig_dir, font_size, fig_size, colormap, fig_format):
"""
:param config: str (.yaml file name)
:param config_prefix: str (path to dir)
:param coords_path: str (path to file)
:param distances_namespace: str
:param bin_distance: float
:param selectivity_path: str
:param arena_id: str
:param populations: tuple of str
:param io_size: int
:param cache_size: int
:param verbose: bool
:param debug: bool
:param show_fig: bool
:param save_fig: str (base file name)
:param save_fig_dir: str (path to dir)
:param font_size: float
:param fig_format: str
"""
comm = MPI.COMM_WORLD
rank = comm.rank
config_logging(verbose)
env = Env(comm=comm,
config_file=config,
config_prefix=config_prefix,
template_paths=None)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info(f'{comm.size} ranks have been allocated')
fig_options = copy.copy(default_fig_options)
fig_options.saveFigDir = save_fig_dir
fig_options.fontSize = font_size
fig_options.figFormat = fig_format
fig_options.showFig = show_fig
fig_options.figSize = fig_size
if save_fig is not None:
save_fig = f'{save_fig} {arena_id}'
fig_options.saveFig = save_fig
population_ranges = read_population_ranges(selectivity_path, comm)[0]
coords_population_ranges = read_population_ranges(coords_path, comm)[0]
if len(populations) == 0:
populations = ('MC', 'ConMC', 'LPP', 'GC', 'MPP', 'CA3c')
valid_selectivity_namespaces = dict()
if rank == 0:
for population in populations:
if population not in population_ranges:
raise RuntimeError(
f'plot_input_selectivity_features: specified population: {population} not found in '
f'provided selectivity_path: {selectivity_path}')
if population not in env.stimulus_config[
'Selectivity Type Probabilities']:
raise RuntimeError(
'plot_input_selectivity_features: selectivity type not specified for '
f'population: {population}')
valid_selectivity_namespaces[population] = []
with h5py.File(selectivity_path, 'r') as selectivity_f:
for this_namespace in selectivity_f['Populations'][population]:
if f'{selectivity_namespace} {arena_id}' in this_namespace:
valid_selectivity_namespaces[population].append(
this_namespace)
if len(valid_selectivity_namespaces[population]) == 0:
raise RuntimeError(
f'plot_input_selectivity_features: no selectivity data in arena: {arena_id} found '
f'for specified population: {population} in provided selectivity_path: {selectivity_path}'
)
valid_selectivity_namespaces = comm.bcast(valid_selectivity_namespaces,
root=0)
selectivity_type_names = dict(
(val, key) for (key, val) in viewitems(env.selectivity_types))
reference_u_arc_distance_bounds = None
reference_v_arc_distance_bounds = None
if rank == 0:
for population in populations:
if population not in coords_population_ranges:
raise RuntimeError(
f'plot_input_selectivity_features: specified population: {population} not found in '
f'provided coords_path: {coords_path}')
with h5py.File(coords_path, 'r') as coords_f:
pop_size = population_ranges[population][1]
unique_gid_count = len(
set(coords_f['Populations'][population]
[distances_namespace]['U Distance']['Cell Index'][:]))
if pop_size != unique_gid_count:
raise RuntimeError(
f'plot_input_selectivity_features: only {unique_gid_count}/{pop_size} unique cell indexes found '
f'for specified population: {population} in provided coords_path: {coords_path}'
)
if reference_u_arc_distance_bounds is None:
try:
reference_u_arc_distance_bounds = \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Min'], \
coords_f['Populations'][population][distances_namespace].attrs['Reference U Max']
except Exception:
raise RuntimeError(
'plot_input_selectivity_features: problem locating attributes '
f'containing reference bounds in namespace: {distances_namespace} '
f'for population: {population} from coords_path: {coords_path}'
)
if reference_v_arc_distance_bounds is None:
try:
reference_v_arc_distance_bounds = \
coords_f['Populations'][population][distances_namespace].attrs['Reference V Min'], \
coords_f['Populations'][population][distances_namespace].attrs['Reference V Max']
except Exception:
raise RuntimeError(
'plot_input_selectivity_features: problem locating attributes '
f'containing reference bounds in namespace: {distances_namespace} '
f'for population: {population} from coords_path: {coords_path}'
)
reference_u_arc_distance_bounds = comm.bcast(
reference_u_arc_distance_bounds, root=0)
reference_v_arc_distance_bounds = comm.bcast(
reference_v_arc_distance_bounds, root=0)
u_edges = np.arange(reference_u_arc_distance_bounds[0],
reference_u_arc_distance_bounds[1] + bin_distance / 2.,
bin_distance)
v_edges = np.arange(reference_v_arc_distance_bounds[0],
reference_v_arc_distance_bounds[1] + bin_distance / 2.,
bin_distance)
if arena_id not in env.stimulus_config['Arena']:
raise RuntimeError(
f'Arena with ID: {arena_id} not specified by configuration at file path: {config_prefix}/{config}'
)
if spatial_resolution is None:
spatial_resolution = env.stimulus_config['Spatial Resolution']
arena = env.stimulus_config['Arena'][arena_id]
arena_x_mesh, arena_y_mesh = None, None
if rank == 0:
arena_x_mesh, arena_y_mesh = \
get_2D_arena_spatial_mesh(arena=arena, spatial_resolution=spatial_resolution)
arena_x_mesh = comm.bcast(arena_x_mesh, root=0)
arena_y_mesh = comm.bcast(arena_y_mesh, root=0)
x0_dict = {}
y0_dict = {}
for population in populations:
start_time = time.time()
u_distances_by_gid = dict()
v_distances_by_gid = dict()
distances_attr_gen = \
bcast_cell_attributes(coords_path, population, root=0, namespace=distances_namespace, comm=comm)
for gid, distances_attr_dict in distances_attr_gen:
u_distances_by_gid[gid] = distances_attr_dict['U Distance'][0]
v_distances_by_gid[gid] = distances_attr_dict['V Distance'][0]
if rank == 0:
logger.info(
f'Reading {len(u_distances_by_gid)} cell positions for population {population} took '
f'{time.time() - start_time:.2f} s')
for this_selectivity_namespace in valid_selectivity_namespaces[
population]:
start_time = time.time()
if rank == 0:
logger.info(
f'Reading from {this_selectivity_namespace} namespace for population {population}...'
)
gid_count = 0
gathered_cell_attributes = defaultdict(list)
gathered_component_attributes = defaultdict(list)
u_distances_by_cell = list()
v_distances_by_cell = list()
u_distances_by_component = list()
v_distances_by_component = list()
rate_map_sum_by_module = defaultdict(
lambda: np.zeros_like(arena_x_mesh))
count_by_module = defaultdict(int)
start_time = time.time()
x0_list_by_module = defaultdict(list)
y0_list_by_module = defaultdict(list)
selectivity_attr_gen = NeuroH5CellAttrGen(
selectivity_path,
population,
namespace=this_selectivity_namespace,
comm=comm,
io_size=io_size,
cache_size=cache_size)
for iter_count, (
gid,
selectivity_attr_dict) in enumerate(selectivity_attr_gen):
if gid is not None:
gid_count += 1
this_selectivity_type = selectivity_attr_dict[
'Selectivity Type'][0]
this_selectivity_type_name = selectivity_type_names[
this_selectivity_type]
input_cell_config = \
get_input_cell_config(selectivity_type=this_selectivity_type,
selectivity_type_names=selectivity_type_names,
selectivity_attr_dict=selectivity_attr_dict)
rate_map = input_cell_config.get_rate_map(x=arena_x_mesh,
y=arena_y_mesh)
u_distances_by_cell.append(u_distances_by_gid[gid])
v_distances_by_cell.append(v_distances_by_gid[gid])
this_cell_attrs, component_count, this_component_attrs = input_cell_config.gather_attributes(
)
for attr_name, attr_val in viewitems(this_cell_attrs):
gathered_cell_attributes[attr_name].append(attr_val)
gathered_cell_attributes['Mean Rate'].append(
np.mean(rate_map))
if component_count > 0:
u_distances_by_component.extend(
[u_distances_by_gid[gid]] * component_count)
v_distances_by_component.extend(
[v_distances_by_gid[gid]] * component_count)
for attr_name, attr_val in viewitems(
this_component_attrs):
gathered_component_attributes[attr_name].extend(
attr_val)
this_module_id = this_cell_attrs['Module ID']
if debug and rank == 0:
fig_title = f'{population} {this_selectivity_type_name} cell {gid}'
if save_fig is not None:
fig_options.saveFig = f'{save_fig} {fig_title}'
plot_2D_rate_map(
x=arena_x_mesh,
y=arena_y_mesh,
rate_map=rate_map,
peak_rate=env.stimulus_config['Peak Rate']
[population][this_selectivity_type],
title=f'{fig_title}\nModule: {this_module_id}',
**fig_options())
x0_list_by_module[this_module_id].append(
selectivity_attr_dict['X Offset'])
y0_list_by_module[this_module_id].append(
selectivity_attr_dict['Y Offset'])
rate_map_sum_by_module[this_module_id] = np.add(
rate_map, rate_map_sum_by_module[this_module_id])
count_by_module[this_module_id] += 1
if debug and iter_count >= 10:
break
if rank == 0:
logger.info(
f'Done reading from {this_selectivity_namespace} namespace for population {population}...'
)
cell_count_hist, _, _ = np.histogram2d(u_distances_by_cell,
v_distances_by_cell,
bins=[u_edges, v_edges])
component_count_hist, _, _ = np.histogram2d(
u_distances_by_component,
v_distances_by_component,
bins=[u_edges, v_edges])
if debug:
context.update(locals())
gathered_cell_attr_hist = dict()
gathered_component_attr_hist = dict()
for key in gathered_cell_attributes:
gathered_cell_attr_hist[key], _, _ = \
np.histogram2d(u_distances_by_cell, v_distances_by_cell, bins=[u_edges, v_edges],
weights=gathered_cell_attributes[key])
for key in gathered_component_attributes:
gathered_component_attr_hist[key], _, _ = \
np.histogram2d(u_distances_by_component, v_distances_by_component, bins=[u_edges, v_edges],
weights=gathered_component_attributes[key])
gid_count = comm.gather(gid_count, root=0)
cell_count_hist = comm.gather(cell_count_hist, root=0)
component_count_hist = comm.gather(component_count_hist, root=0)
gathered_cell_attr_hist = comm.gather(gathered_cell_attr_hist,
root=0)
gathered_component_attr_hist = comm.gather(
gathered_component_attr_hist, root=0)
x0_list_by_module = dict(x0_list_by_module)
y0_list_by_module = dict(y0_list_by_module)
x0_list_by_module = comm.reduce(x0_list_by_module,
op=mpi_op_merge_list_dict,
root=0)
y0_list_by_module = comm.reduce(y0_list_by_module,
op=mpi_op_merge_list_dict,
root=0)
rate_map_sum_by_module = dict(rate_map_sum_by_module)
rate_map_sum_by_module = comm.gather(rate_map_sum_by_module,
root=0)
count_by_module = dict(count_by_module)
count_by_module = comm.reduce(count_by_module,
op=mpi_op_merge_count_dict,
root=0)
if rank == 0:
gid_count = sum(gid_count)
cell_count_hist = np.sum(cell_count_hist, axis=0)
component_count_hist = np.sum(component_count_hist, axis=0)
merged_cell_attr_hist = defaultdict(
lambda: np.zeros_like(cell_count_hist))
merged_component_attr_hist = defaultdict(
lambda: np.zeros_like(component_count_hist))
for each_cell_attr_hist in gathered_cell_attr_hist:
for key in each_cell_attr_hist:
merged_cell_attr_hist[key] = np.add(
merged_cell_attr_hist[key],
each_cell_attr_hist[key])
for each_component_attr_hist in gathered_component_attr_hist:
for key in each_component_attr_hist:
merged_component_attr_hist[key] = np.add(
merged_component_attr_hist[key],
each_component_attr_hist[key])
merged_rate_map_sum_by_module = defaultdict(
lambda: np.zeros_like(arena_x_mesh))
for each_rate_map_sum_by_module in rate_map_sum_by_module:
for this_module_id in each_rate_map_sum_by_module:
merged_rate_map_sum_by_module[this_module_id] = \
np.add(merged_rate_map_sum_by_module[this_module_id],
each_rate_map_sum_by_module[this_module_id])
logger.info(
f'Processing {gid_count} {population} {this_selectivity_type_name} cells '
f'took {time.time() - start_time:.2f} s')
if debug:
context.update(locals())
fig_title = f'{population} {this_selectivity_type_name} field offsets'
if save_fig is not None:
fig_options.saveFig = f'{save_fig} {fig_title}'
for key in merged_cell_attr_hist:
fig_title = f'{population} {this_selectivity_type_name} cells {key} distribution'
if save_fig is not None:
fig_options.saveFig = f'{save_fig} {fig_title}'
if colormap is not None:
fig_options.colormap = colormap
title = f'{population} {this_selectivity_type_name} cells\n{key} distribution'
fig = plot_2D_histogram(
merged_cell_attr_hist[key],
x_edges=u_edges,
y_edges=v_edges,
norm=cell_count_hist,
ylabel='Transverse position (um)',
xlabel='Septo-temporal position (um)',
title=title,
cbar_label='Mean value per bin',
cbar=True,
**fig_options())
close_figure(fig)
for key in merged_component_attr_hist:
fig_title = f'{population} {this_selectivity_type_name} cells {key} distribution'
if save_fig is not None:
fig_options.saveFig = f'{save_fig} {fig_title}'
title = f'{population} {this_selectivity_type_name} cells\n{key} distribution'
fig = plot_2D_histogram(
merged_component_attr_hist[key],
x_edges=u_edges,
y_edges=v_edges,
norm=component_count_hist,
ylabel='Transverse position (um)',
xlabel='Septo-temporal position (um)',
title=title,
cbar_label='Mean value per bin',
cbar=True,
**fig_options())
close_figure(fig)
for this_module_id in merged_rate_map_sum_by_module:
num_cells = count_by_module[this_module_id]
x0 = np.concatenate(x0_list_by_module[this_module_id])
y0 = np.concatenate(y0_list_by_module[this_module_id])
fig_title = f'{population} {this_selectivity_type_name} Module {this_module_id} rate map'
if save_fig is not None:
fig_options.saveFig = f'{save_fig} {fig_title}'
fig = plot_2D_rate_map(
x=arena_x_mesh,
y=arena_y_mesh,
x0=x0,
y0=y0,
rate_map=merged_rate_map_sum_by_module[this_module_id],
title=
(f'{population} {this_selectivity_type_name} rate map\n'
f'Module {this_module_id} ({num_cells} cells)'),
**fig_options())
close_figure(fig)
if is_interactive and rank == 0:
context.update(locals())
def main(config, forest_path, connectivity_namespace, coords_path, coords_namespace, io_size, chunk_size, value_chunk_size,
cache_size, debug):
"""
:param forest_path:
:param connectivity_namespace:
:param coords_path:
:param coords_namespace:
:param io_size:
:param chunk_size:
:param value_chunk_size:
:param cache_size:
"""
# troubleshooting
if False:
forest_path = '../morphologies/DGC_forest_connectivity_20170508.h5'
coords_path = '../morphologies/dentate_Full_Scale_Control_coords_selectivity_20170615.h5'
coords_namespace = 'Coordinates'
io_size = -1
chunk_size = 1000
value_chunk_size = 1000
cache_size = 50
comm = MPI.COMM_WORLD
rank = comm.rank # The process ID (integer 0-3 for 4-process run)
env = Env(comm=comm, config_file=config)
connection_config = env.connection_config
proportions = connection_config.synapse_proportions
layers = connection_config.synapse_layers
syn_types = connection_config.synapse_types
swc_types = connection_config.synapse_locations
if io_size == -1:
io_size = comm.size
if rank == 0:
print('%i ranks have been allocated' % comm.size)
sys.stdout.flush()
start_time = time.time()
soma_coords = {}
source_populations = list(read_population_ranges(comm, coords_path).keys())
for population in source_populations:
soma_coords[population] = bcast_cell_attributes(comm, 0, coords_path, population,
namespace=coords_namespace)
target = 'GC'
layer_set, swc_type_set, syn_type_set = set(), set(), set()
for source in layers[target]:
layer_set.update(layers[target][source])
swc_type_set.update(swc_types[target][source])
syn_type_set.update(syn_types[target][source])
count = 0
attr_gen = NeuroH5CellAttrGen(comm, forest_path, target, io_size=io_size, cache_size=cache_size,
namespace='Synapse Attributes')
if debug:
attr_gen_wrapper = (next(attr_gen) for i in range(2))
else:
attr_gen_wrapper = attr_gen
for target_gid, attributes_dict in attr_gen_wrapper:
last_time = time.time()
connection_dict = {}
p_dict = {}
source_gid_dict = {}
if target_gid is None:
print('Rank %i target gid is None' % rank)
else:
print('Rank %i received attributes for target: %s, gid: %i' % (rank, target, target_gid))
synapse_dict = attributes_dict['Synapse_Attributes']
connection_dict[target_gid] = {}
local_np_random.seed(target_gid + connectivity_seed_offset)
connection_dict[target_gid]['source_gid'] = np.array([], dtype='uint32')
connection_dict[target_gid]['syn_id'] = np.array([], dtype='uint32')
for layer in layer_set:
for swc_type in swc_type_set:
for syn_type in syn_type_set:
sources, this_proportions = filter_sources(target, layer, swc_type, syn_type, connection_config)
if sources:
if rank == 0 and count == 0:
source_list_str = '[' + ', '.join(['%s' % xi for xi in sources]) + ']'
print('Connections to target: %s in layer: %i ' \
'(swc_type: %i, syn_type: %i): %s' % \
(target, layer, swc_type, syn_type, source_list_str))
p, source_gid = np.array([]), np.array([])
for source, this_proportion in zip(sources, this_proportions):
if source not in source_gid_dict:
this_source_gid = list(soma_coords[source].keys())
this_p = np.ones(len(this_source_gid)) / float(len(this_source_gid))
source_gid_dict[source] = this_source_gid
p_dict[source] = this_p
else:
this_source_gid = source_gid_dict[source]
this_p = p_dict[source]
p = np.append(p, this_p * this_proportion)
source_gid = np.append(source_gid, this_source_gid)
syn_indexes = filter_synapses(synapse_dict, layer, swc_type, syn_type)
connection_dict[target_gid]['syn_id'] = \
np.append(connection_dict[target_gid]['syn_id'],
synapse_dict['syn_id'][syn_indexes]).astype('uint32', copy=False)
this_source_gid = local_np_random.choice(source_gid, len(syn_indexes), p=p)
connection_dict[target_gid]['source_gid'] = \
np.append(connection_dict[target_gid]['source_gid'],
this_source_gid).astype('uint32', copy=False)
count += 1
print('Rank %i took %.2f s to compute connectivity for target: %s, gid: %i' % (rank,
time.time() - last_time,
target, target_gid))
sys.stdout.flush()
if not debug:
append_cell_attributes(comm, forest_path, target, connection_dict,
namespace=connectivity_namespace, io_size=io_size, chunk_size=chunk_size,
value_chunk_size=value_chunk_size)
sys.stdout.flush()
del connection_dict
del p_dict
del source_gid_dict
gc.collect()
global_count = comm.gather(count, root=0)
if rank == 0:
print('%i ranks took took %.2f s to compute connectivity for %i cells' % (comm.size, time.time() - start_time,
np.sum(global_count)))
def main(config, coords_path, coords_namespace, geometry_path, populations, interp_chunk_size, resolution, alpha_radius, nsample, io_size, chunk_size, value_chunk_size, cache_size, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(__file__)
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config)
output_path = coords_path
soma_coords = {}
if rank == 0:
logger.info('Reading population coordinates...')
for population in sorted(populations):
coords = bcast_cell_attributes(coords_path, population, 0, \
namespace=coords_namespace, comm=comm)
soma_coords[population] = { k: (v['U Coordinate'][0], v['V Coordinate'][0], v['L Coordinate'][0])
for (k,v) in coords }
del coords
gc.collect()
has_ip_dist=False
origin_ranges=None
ip_dist_u=None
ip_dist_v=None
ip_dist_path = 'Distance Interpolant/%d/%d/%d' % resolution
if rank == 0:
if geometry_path is not None:
f = h5py.File(geometry_path,"a")
pkl_path = f'{ip_dist_path}/ip_dist.pkl'
if pkl_path in f:
has_ip_dist = True
ip_dist_dset = f[pkl_path]
origin_ranges, ip_dist_u, ip_dist_v = pickle.loads(base64.b64decode(ip_dist_dset[()]))
f.close()
has_ip_dist = env.comm.bcast(has_ip_dist, root=0)
if not has_ip_dist:
if rank == 0:
logger.info('Creating distance interpolant...')
(origin_ranges, ip_dist_u, ip_dist_v) = make_distance_interpolant(env.comm, geometry_config=env.geometry,
make_volume=make_CA1_volume,
resolution=resolution, nsample=nsample)
if rank == 0:
if geometry_path is not None:
f = h5py.File(geometry_path, 'a')
pkl_path = f'{ip_dist_path}/ip_dist.pkl'
pkl = pickle.dumps((origin_ranges, ip_dist_u, ip_dist_v))
pklstr = base64.b64encode(pkl)
f[pkl_path] = pklstr
f.close()
ip_dist = (origin_ranges, ip_dist_u, ip_dist_v)
if rank == 0:
logger.info('Measuring soma distances...')
soma_distances = measure_distances(env.comm, env.geometry, soma_coords, ip_dist, resolution=resolution)
for population in list(sorted(soma_distances.keys())):
if rank == 0:
logger.info(f'Writing distances for population {population}...')
dist_dict = soma_distances[population]
attr_dict = {}
for k, v in viewitems(dist_dict):
attr_dict[k] = { 'U Distance': np.asarray([v[0]],dtype=np.float32), \
'V Distance': np.asarray([v[1]],dtype=np.float32) }
append_cell_attributes(output_path, population, attr_dict,
namespace='Arc Distances', comm=comm,
io_size=io_size, chunk_size=chunk_size,
value_chunk_size=value_chunk_size, cache_size=cache_size)
if rank == 0:
f = h5py.File(output_path, 'a')
f['Populations'][population]['Arc Distances'].attrs['Reference U Min'] = origin_ranges[0][0]
f['Populations'][population]['Arc Distances'].attrs['Reference U Max'] = origin_ranges[0][1]
f['Populations'][population]['Arc Distances'].attrs['Reference V Min'] = origin_ranges[1][0]
f['Populations'][population]['Arc Distances'].attrs['Reference V Max'] = origin_ranges[1][1]
f.close()
comm.Barrier()
