def main(inst_rates_path, inst_rates_namespace, include, bin_size, nstdev,
baseline_fraction, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(script_name)
if not include:
population_names = read_population_names(inst_rates_path)
for pop in population_names:
include.append(pop)
for i, population in enumerate(include):
rate_inst_iter = read_cell_attributes(inst_rates_path,
population,
namespace=inst_rates_namespace)
rate_inst_dict = dict(rate_inst_iter)
spikedata.place_fields(population,
bin_size,
rate_inst_dict,
nstdev,
baseline_fraction=baseline_fraction,
saveData=inst_rates_path)
def main(coords_path, coords_namespace):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
print('Allocated %i ranks' % size)
population_ranges = read_population_ranges(coords_path)[0]
print(population_ranges)
soma_coords = {}
populations = ['GC']
for population in population_ranges.keys():
print('Population %s' % population)
it = read_cell_attributes(coords_path,
population,
namespace=coords_namespace)
print('it = %s' % str(it))
for cell_gid, coords_dict in it:
print(hasattr(coords_dict, 'U Coordinate'))
cell_u = getattr(coords_dict, 'U Coordinate')
cell_u = getattr(coords_dict, 'U Coordinate')
cell_v = getattr(coords_dict, 'V Coordinate')
print('Rank %i: gid = %i u = %f v = %f' %
(rank, cell_gid, cell_u, cell_v))
def main(coords_path, coords_namespace):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
print('Allocated %i ranks' % size)
population_ranges = read_population_ranges(coords_path)[0]
print(population_ranges)
soma_coords = {}
for population in sorted(population_ranges.keys()):
print('Population %s' % population)
it, tuple_info = read_cell_attributes(coords_path,
population,
namespace=coords_namespace,
return_type='tuple')
u_index = tuple_info['U Coordinate']
v_index = tuple_info['V Coordinate']
for cell_gid, coords_tuple in it:
cell_u = coords_tuple[u_index]
cell_v = coords_tuple[v_index]
print('Rank %i: gid = %i u = %f v = %f' %
(rank, cell_gid, cell_u, cell_v))
def main(population, features_path, features_namespace, extra_columns):
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
population_ranges = read_population_ranges(features_path)[0]
soma_coords = {}
extra_columns_list = extra_columns.split(",")
columns = ['Field Width', 'X Offset', 'Y Offset'] + extra_columns_list
df_dict = {}
it = read_cell_attributes(features_path,
population,
namespace=features_namespace)
for cell_gid, features_dict in it:
cell_field_width = features_dict['Field Width'][0]
cell_xoffset = features_dict['X Offset'][0]
cell_yoffset = features_dict['Y Offset'][0]
df_dict[cell_gid] = [cell_field_width, cell_xoffset, cell_yoffset]
df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns)
df = df.reindex(sorted(df_dict.keys()))
df.to_csv('features.%s.csv' % population)
def main(config, config_prefix, coords_path, distances_namespace, population, graph_type, bin_size, verbose):
utils.config_logging(verbose)
env = Env(config_file=config, config_prefix=config_prefix)
soma_distances = read_cell_attributes(coords_path, population, namespace=distances_namespace)
plot.plot_positions (env, population, soma_distances, bin_size=bin_size, graph_type=graph_type, saveFig=True)
def main(features_path, cell_type, arena_id, show_fig, save_fig):
font = {'family': 'normal', 'weight': 'bold', 'size': 6}
matplotlib.rc('font', **font)
comm = MPI.COMM_WORLD
modules = np.arange(10) + 1
if cell_type == 'grid':
mpp_grid = read_cell_attributes(features_path, 'MPP',
'Grid Selectivity %s' % arena_id)
cells_modules_dictionary = gid2module_dictionary([mpp_grid], modules)
elif cell_type == 'place':
lpp_place = read_cell_attributes(features_path, 'LPP',
'Place Selectivity %s' % arena_id)
mpp_place = read_cell_attributes(features_path, 'MPP',
'Place Selectivity %s' % arena_id)
cells_modules_dictionary = gid2module_dictionary(
[mpp_place, lpp_place], modules)
elif cell_type == 'both':
lpp_place = read_cell_attributes(features_path, 'LPP',
'Place Selectivity %s' % arena_id)
mpp_place = read_cell_attributes(features_path, 'MPP',
'Place Selectivity %s' % arena_id)
mpp_grid = read_cell_attributes(features_path, 'MPP',
'Grid Selectivity %s' % arena_id)
cells_modules_dictionary = gid2module_dictionary(
[mpp_grid, mpp_place, lpp_place], modules)
kwargs = {'ctype': cell_type}
plot_group(cells_modules_dictionary,
modules,
plot=show_fig,
save=save_fig,
**kwargs)
def read_spike_events(input_file, population_names, namespace_id, spike_train_attr_name='t', time_range=None,
max_spikes=None, n_trials=-1, merge_trials=False):
"""
Reads spike trains from a NeuroH5 file, and returns a dictionary with spike times and cell indices.
:param input_file: str (path to file)
:param population_names: list of str
:param namespace_id: str
:param spike_train_attr_name: str
:param time_range: list of float
:param max_spikes: float
:param n_trials: int
:param merge_trials: bool
:return: dict
"""
assert((n_trials >= 1) | (n_trials == -1))
spkpoplst = []
spkindlst = []
spktlst = []
spktrials = []
num_cell_spks = {}
pop_active_cells = {}
tmin = float('inf')
tmax = 0.
for pop_name in population_names:
if time_range is None or time_range[1] is None:
logger.info('Reading spike data for population %s...' % pop_name)
else:
logger.info('Reading spike data for population %s in time range %s...' % (pop_name, str(time_range)))
spkiter = read_cell_attributes(input_file, pop_name, namespace=namespace_id)
this_num_cell_spks = 0
active_set = set([])
pop_spkindlst = []
pop_spktlst = []
pop_spktriallst = []
# Time Range
if time_range is None or time_range[1] is None:
for spkind, spkts in spkiter:
slen = len(spkts[spike_train_attr_name])
trial_dur = spkts.get('Trial Duration', np.asarray([0.]))
trial_ind = spkts.get('Trial Index', np.zeros((slen,),dtype=np.uint8))
if n_trials == -1:
n_trials = trial_dur.shape[0]
for spk_i, spkt in enumerate(spkts[spike_train_attr_name]):
trial_i = trial_ind[spk_i]
if trial_i >= n_trials:
continue
if merge_trials:
spkt += np.sum(trial_dur[:trial_i])
pop_spkindlst.append(spkind)
pop_spktlst.append(spkt)
pop_spktriallst.append(trial_i)
if spkt < tmin:
tmin = spkt
if spkt > tmax:
tmax = spkt
this_num_cell_spks += 1
active_set.add(spkind)
else:
if time_range[0] is None:
time_range[0] = 0.0
time_span = time_range[1] - time_range[0]
for spkind, spkts in spkiter:
trial_dur = spkts.get('Trial Duration', np.asarray([0.]))
trial_dur[1:] = np.asarray([max(x, time_span) for x in trial_dur[1:] ],
dtype=np.float32)
trial_ind = spkts.get('Trial Index', np.zeros((len(spkts[spike_train_attr_name])),dtype=np.int))
if n_trials == -1:
n_trials = trial_dur.shape[0]
for spk_i, spkt in enumerate(spkts[spike_train_attr_name]):
trial_i = trial_ind[spk_i]
if trial_i >= n_trials:
continue
if time_range[0] <= spkt <= time_range[1]:
if merge_trials:
spkt += np.sum(trial_dur[:trial_i])
pop_spkindlst.append(spkind)
pop_spktlst.append(spkt)
pop_spktriallst.append(trial_i)
if spkt < tmin:
tmin = spkt
if spkt > tmax:
tmax = spkt
this_num_cell_spks += 1
active_set.add(spkind)
if not active_set:
continue
pop_active_cells[pop_name] = active_set
num_cell_spks[pop_name] = this_num_cell_spks
pop_spkts = np.asarray(pop_spktlst, dtype=np.float32)
del (pop_spktlst)
pop_spkinds = np.asarray(pop_spkindlst, dtype=np.uint32)
del (pop_spkindlst)
pop_spktrials = np.asarray(pop_spktriallst, dtype=np.uint32)
del (pop_spktriallst)
# Limit to max_spikes
if (max_spikes is not None) and (len(pop_spkts) > max_spikes):
logger.warn(' Reading only randomly sampled %i out of %i spikes for population %s' %
(max_spikes, len(pop_spkts), pop_name))
sample_inds = np.random.randint(0, len(pop_spkinds) - 1, size=int(max_spikes))
pop_spkts = pop_spkts[sample_inds]
pop_spkinds = pop_spkinds[sample_inds]
pop_spktrials = pop_spkinds[sample_inds]
tmax = max(tmax, max(pop_spkts))
spkpoplst.append(pop_name)
pop_trial_spkindlst = []
pop_trial_spktlst = []
for trial_i in range(n_trials):
trial_idxs = np.where(pop_spktrials == trial_i)[0]
sorted_trial_idxs = np.argsort(pop_spkts[trial_idxs])
pop_trial_spktlst.append(np.take(pop_spkts[trial_idxs], sorted_trial_idxs))
pop_trial_spkindlst.append(np.take(pop_spkinds[trial_idxs], sorted_trial_idxs))
del pop_spkts
del pop_spkinds
del pop_spktrials
if merge_trials:
pop_spkinds = np.concatenate(pop_trial_spkindlst)
pop_spktlst = np.concatenate(pop_trial_spktlst)
spkindlst.append(pop_spkinds)
spktlst.append(pop_spktlst)
else:
spkindlst.append(pop_trial_spkindlst)
spktlst.append(pop_trial_spktlst)
logger.info(' Read %i spikes for population %s' % (this_num_cell_spks, pop_name))
return {'spkpoplst': spkpoplst, 'spktlst': spktlst, 'spkindlst': spkindlst, 'tmin': tmin, 'tmax': tmax,
'pop_active_cells': pop_active_cells, 'num_cell_spks': num_cell_spks,
'n_trials': n_trials}
def read_state(input_file,
population_names,
namespace_id,
time_variable='t',
state_variable='v',
time_range=None,
max_units=None,
gid=None,
comm=None,
n_trials=-1):
if comm is None:
comm = MPI.COMM_WORLD
pop_state_dict = {}
logger.info(
'Reading state data from populations %s, namespace %s gid = %s...' %
(str(population_names), namespace_id, str(gid)))
attr_info_dict = read_cell_attribute_info(input_file,
populations=population_names,
read_cell_index=True)
for pop_name in population_names:
cell_index = None
pop_state_dict[pop_name] = {}
for attr_name, attr_cell_index in attr_info_dict[pop_name][
namespace_id]:
if state_variable == attr_name:
cell_index = attr_cell_index
if cell_index is None:
raise RuntimeError(
'read_state: Unable to find recordings for state variable %s in population %s namespace %s'
% (state_variable, pop_name, str(namespace_id)))
cell_set = set(cell_index)
# Limit to max_units
if gid is None:
if (max_units is not None) and (len(cell_set) > max_units):
logger.info(
' Reading only randomly sampled %i out of %i units for population %s'
% (max_units, len(cell_set), pop_name))
sample_inds = np.random.randint(0,
len(cell_set) - 1,
size=int(max_units))
cell_set_lst = list(cell_set)
gid_set = set([cell_set_lst[i] for i in sample_inds])
else:
gid_set = cell_set
else:
gid_set = set(gid)
state_dict = {}
if gid is None:
valiter = read_cell_attributes(input_file,
pop_name,
namespace=namespace_id,
comm=comm)
else:
valiter = read_cell_attribute_selection(input_file,
pop_name,
namespace=namespace_id,
selection=list(gid_set),
comm=comm)
if time_range is None:
for cellind, vals in valiter:
if cellind is not None:
trial_dur = vals.get('trial duration', None)
distance = vals.get('distance', [None])[0]
section = vals.get('section', [None])[0]
loc = vals.get('loc', [None])[0]
tvals = np.asarray(vals[time_variable], dtype=np.float32)
svals = np.asarray(vals[state_variable], dtype=np.float32)
trial_bounds = list(
np.where(np.isclose(tvals, tvals[0], atol=1e-4))[0])
n_trial_bounds = len(trial_bounds)
trial_bounds.append(len(tvals))
if n_trials == -1:
this_n_trials = n_trial_bounds
else:
this_n_trials = min(n_trial_bounds, n_trials)
if this_n_trials > 1:
state_dict[cellind] = (np.split(
tvals, trial_bounds[1:n_trials]),
np.split(
svals,
trial_bounds[1:n_trials]),
distance, section, loc)
else:
state_dict[cellind] = ([tvals[:trial_bounds[1]]
], [svals[:trial_bounds[1]]],
distance, section, loc)
else:
for cellind, vals in valiter:
if cellind is not None:
distance = vals.get('distance', [None])[0]
section = vals.get('section', [None])[0]
loc = vals.get('loc', [None])[0]
tinds = np.argwhere((vals[time_variable] <= time_range[1])
&
(vals[time_variable] >= time_range[0]))
tvals = np.asarray(vals[time_variable][tinds],
dtype=np.float32).reshape((-1, ))
svals = np.asarray(vals[state_variable][tinds],
dtype=np.float32).reshape((-1, ))
trial_bounds = list(
np.where(np.isclose(tvals, tvals[0], atol=1e-4))[0])
n_trial_bounds = len(trial_bounds)
trial_bounds.append(len(tvals))
if n_trials == -1:
this_n_trials = n_trial_bounds
else:
this_n_trials = min(n_trial_bounds, n_trials)
if this_n_trials > 1:
state_dict[cellind] = (np.split(
tvals, trial_bounds[1:n_trials]),
np.split(
svals,
trial_bounds[1:n_trials]),
distance, section, loc)
else:
state_dict[cellind] = ([tvals[:trial_bounds[1]]
], [svals[:trial_bounds[1]]],
distance, section, loc)
pop_state_dict[pop_name] = state_dict
return {
'states': pop_state_dict,
'time_variable': time_variable,
'state_variable': state_variable
}
def main(config, config_prefix, include, forest_path, connectivity_path,
connectivity_namespace, coords_path, coords_namespace,
synapses_namespace, distances_namespace, resolution,
interp_chunk_size, io_size, chunk_size, value_chunk_size, cache_size,
write_size, verbose, dry_run, debug):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm, config_file=config, config_prefix=config_prefix)
configure_hoc_env(env)
connection_config = env.connection_config
extent = {}
if (not dry_run) and (rank == 0):
if not os.path.isfile(connectivity_path):
input_file = h5py.File(coords_path, 'r')
output_file = h5py.File(connectivity_path, 'w')
input_file.copy('/H5Types', output_file)
input_file.close()
output_file.close()
comm.barrier()
population_ranges = read_population_ranges(coords_path)[0]
populations = sorted(list(population_ranges.keys()))
color = 0
if rank == 0:
color = 1
comm0 = comm.Split(color, 0)
soma_distances = {}
soma_coords = {}
for population in populations:
if rank == 0:
logger.info(f'Reading {population} coordinates...')
coords_iter = read_cell_attributes(
coords_path,
population,
comm=comm0,
mask=set(['U Coordinate', 'V Coordinate', 'L Coordinate']),
namespace=coords_namespace)
distances_iter = read_cell_attributes(
coords_path,
population,
comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
soma_coords[population] = {
k: (float(v['U Coordinate'][0]), float(v['V Coordinate'][0]),
float(v['L Coordinate'][0]))
for (k, v) in coords_iter
}
distances = {
k: (float(v['U Distance'][0]), float(v['V Distance'][0]))
for (k, v) in distances_iter
}
if len(distances) > 0:
soma_distances[population] = distances
gc.collect()
comm.barrier()
comm0.Free()
soma_distances = comm.bcast(soma_distances, root=0)
soma_coords = comm.bcast(soma_coords, root=0)
forest_populations = sorted(read_population_names(forest_path))
if (include is None) or (len(include) == 0):
destination_populations = forest_populations
else:
destination_populations = []
for p in include:
if p in forest_populations:
destination_populations.append(p)
if rank == 0:
logger.info(
f'Generating connectivity for populations {destination_populations}...'
)
if len(soma_distances) == 0:
(origin_ranges, ip_dist_u,
ip_dist_v) = make_distance_interpolant(env,
resolution=resolution,
nsample=nsample)
ip_dist = (origin_ranges, ip_dist_u, ip_dist_v)
soma_distances = measure_distances(env,
soma_coords,
ip_dist,
resolution=resolution)
for destination_population in destination_populations:
if rank == 0:
logger.info(
f'Generating connection probabilities for population {destination_population}...'
)
connection_prob = ConnectionProb(destination_population, soma_coords, soma_distances, \
env.connection_extents)
synapse_seed = int(
env.model_config['Random Seeds']['Synapse Projection Partitions'])
connectivity_seed = int(env.model_config['Random Seeds']
['Distance-Dependent Connectivity'])
cluster_seed = int(
env.model_config['Random Seeds']['Connectivity Clustering'])
if rank == 0:
logger.info(
f'Generating connections for population {destination_population}...'
)
populations_dict = env.model_config['Definitions']['Populations']
generate_uv_distance_connections(comm,
populations_dict,
connection_config,
connection_prob,
forest_path,
synapse_seed,
connectivity_seed,
cluster_seed,
synapses_namespace,
connectivity_namespace,
connectivity_path,
io_size,
chunk_size,
value_chunk_size,
cache_size,
write_size,
dry_run=dry_run,
debug=debug)
MPI.Finalize()
def main(include, coords_path, coords_namespace, distances_namespace, resolution, io_size,
verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
connection_config = env.connection_config
extent = {}
population_ranges = read_population_ranges(coords_path)[0]
populations = sorted(list(population_ranges.keys()))
color = 0
if rank == 0:
color = 1
comm0 = comm.Split(color, 0)
soma_distances = {}
soma_coords = {}
for population in populations:
if rank == 0:
logger.info(f'Reading {population} coordinates...')
coords_iter = read_cell_attributes(coords_path, population, comm=comm0,
mask=set(['U Coordinate', 'V Coordinate', 'L Coordinate']),
namespace=coords_namespace)
distances_iter = read_cell_attributes(coords_path, population, comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
soma_coords[population] = { k: (float(v['U Coordinate'][0]),
float(v['V Coordinate'][0]),
float(v['L Coordinate'][0])) for (k,v) in coords_iter }
distances = { k: (float(v['U Distance'][0]),
float(v['V Distance'][0])) for (k,v) in distances_iter }
if len(distances) > 0:
soma_distances[population] = distances
gc.collect()
comm.barrier()
comm0.Free()
soma_distances = comm.bcast(soma_distances, root=0)
soma_coords = comm.bcast(soma_coords, root=0)
extra_columns_list = extra_columns.split(",")
columns = ['Field Width', 'X Offset', 'Y Offset']+extra_columns_list
df_dict = {}
it = read_cell_attributes(features_path, population,
namespace=features_namespace)
for cell_gid, features_dict in it:
cell_field_width = features_dict['Field Width'][0]
cell_xoffset = features_dict['X Offset'][0]
cell_yoffset = features_dict['Y Offset'][0]
df_dict[cell_gid] = [cell_field_width, cell_xoffset, cell_yoffset]
df = pd.DataFrame.from_dict(df_dict, orient='index', columns=columns)
df = df.reindex(sorted(df_dict.keys()))
df.to_csv('features.%s.csv' % population)
MPI.Finalize()
def vertex_distribution(connectivity_path,
coords_path,
distances_namespace,
destination,
sources,
bin_size=20.0,
cache_size=100,
comm=None):
"""
Obtain spatial histograms of source vertices connecting to a given destination population.
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
"""
if comm is None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
color = 0
if rank == 0:
color = 1
comm0 = comm.Split(color, 0)
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
destination_soma_distances = {}
if rank == 0:
logger.info(f'Reading {destination} distances...')
distances_iter = read_cell_attributes(
coords_path,
destination,
comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
destination_soma_distances = {
k: (float(v['U Distance'][0]), float(v['V Distance'][0]))
for (k, v) in distances_iter
}
gc.collect()
comm.barrier()
destination_soma_distances = comm.bcast(destination_soma_distances, root=0)
destination_soma_distance_U = {}
destination_soma_distance_V = {}
for k, v in viewitems(destination_soma_distances):
destination_soma_distance_U[k] = v[0]
destination_soma_distance_V[k] = v[1]
del (destination_soma_distances)
if sources == ():
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
source_soma_distances = {}
if rank == 0:
for s in sources:
logger.info(f'Reading {s} distances...')
distances_iter = read_cell_attributes(
coords_path,
s,
comm=comm0,
mask=set(['U Distance', 'V Distance']),
namespace=distances_namespace)
source_soma_distances[s] = {
k: (float(v['U Distance'][0]), float(v['V Distance'][0]))
for (k, v) in distances_iter
}
gc.collect()
comm.barrier()
comm0.Free()
source_soma_distances = comm.bcast(source_soma_distances, root=0)
source_soma_distance_U = {}
source_soma_distance_V = {}
for s in sources:
this_source_soma_distance_U = {}
this_source_soma_distance_V = {}
for k, v in viewitems(source_soma_distances[s]):
this_source_soma_distance_U[k] = v[0]
this_source_soma_distance_V[k] = v[1]
source_soma_distance_U[s] = this_source_soma_distance_U
source_soma_distance_V[s] = this_source_soma_distance_V
del (source_soma_distances)
if rank == 0:
logger.info('Reading connections %s -> %s...' %
(str(sources), destination))
dist_bins = defaultdict(dict)
dist_u_bins = defaultdict(dict)
dist_v_bins = defaultdict(dict)
gg = [
NeuroH5ProjectionGen(connectivity_path,
source,
destination,
cache_size=cache_size,
comm=comm) for source in sources
]
for prj_gen_tuple in zip_longest(*gg):
destination_gid = prj_gen_tuple[0][0]
if rank == 0 and destination_gid is not None:
logger.info('%d' % destination_gid)
if not all([
prj_gen_elt[0] == destination_gid
for prj_gen_elt in prj_gen_tuple
]):
raise RuntimeError(
'destination %s: destination gid %i not matched across multiple projection generators: '
'%s' % (destination, destination_gid,
[prj_gen_elt[0] for prj_gen_elt in prj_gen_tuple]))
if destination_gid is not None:
for (source, (this_destination_gid,
rest)) in zip_longest(sources, prj_gen_tuple):
this_source_soma_distance_U = source_soma_distance_U[source]
this_source_soma_distance_V = source_soma_distance_V[source]
this_dist_bins = dist_bins[source]
this_dist_u_bins = dist_u_bins[source]
this_dist_v_bins = dist_v_bins[source]
(source_indexes, attr_dict) = rest
dst_U = destination_soma_distance_U[destination_gid]
dst_V = destination_soma_distance_V[destination_gid]
for source_gid in source_indexes:
dist_u = dst_U - this_source_soma_distance_U[source_gid]
dist_v = dst_V - this_source_soma_distance_V[source_gid]
dist = abs(dist_u) + abs(dist_v)
update_bins(this_dist_bins, bin_size, dist)
update_bins(this_dist_u_bins, bin_size, dist_u)
update_bins(this_dist_v_bins, bin_size, dist_v)
add_bins_op = MPI.Op.Create(add_bins, commute=True)
for source in sources:
dist_bins[source] = comm.reduce(dist_bins[source], op=add_bins_op)
dist_u_bins[source] = comm.reduce(dist_u_bins[source], op=add_bins_op)
dist_v_bins[source] = comm.reduce(dist_v_bins[source], op=add_bins_op)
dist_hist_dict = defaultdict(dict)
dist_u_hist_dict = defaultdict(dict)
dist_v_hist_dict = defaultdict(dict)
if rank == 0:
for source in sources:
dist_hist_dict[destination][source] = finalize_bins(
dist_bins[source], bin_size)
dist_u_hist_dict[destination][source] = finalize_bins(
dist_u_bins[source], bin_size)
dist_v_hist_dict[destination][source] = finalize_bins(
dist_v_bins[source], bin_size)
return {
'Total distance': dist_hist_dict,
'U distance': dist_u_hist_dict,
'V distance': dist_v_hist_dict
}
def vertex_metrics(connectivity_path,
coords_path,
vertex_metrics_namespace,
distances_namespace,
destination,
sources,
bin_size=50.,
metric='Indegree'):
"""
Obtain vertex metrics with respect to septo-temporal position (longitudinal and transverse arc distances to reference points).
:param connectivity_path:
:param coords_path:
:param distances_namespace:
:param destination:
:param source:
:param bin_size:
:param metric:
"""
(population_ranges, _) = read_population_ranges(coords_path)
destination_start = population_ranges[destination][0]
destination_count = population_ranges[destination][1]
if sources == ():
sources = []
for (src, dst) in read_projection_names(connectivity_path):
if dst == destination:
sources.append(src)
degrees_dict = {}
with h5py.File(connectivity_path, 'r') as f:
for source in sources:
degrees_dict[source] = f['Nodes'][vertex_metrics_namespace][
'%s %s -> %s' %
(metric, source,
destination)]['Attribute Value'][0:destination_count]
for source in sources:
logger.info('projection: %s -> %s: max: %i min: %i mean: %i stdev: %i (%d units)' % \
(source, destination, \
np.max(degrees_dict[source]), \
np.min(degrees_dict[source]), \
np.mean(degrees_dict[source]), \
np.std(degrees_dict[source]), \
len(degrees_dict[source])))
if metric == 'Indegree':
distances = read_cell_attributes(coords_path,
destination,
namespace=distances_namespace)
soma_distances = {
k: (v['U Distance'][0], v['V Distance'][0])
for (k, v) in distances
}
del distances
elif metric == 'Outdegree':
distances = read_cell_attributes(coords_path,
sources[0],
namespace=distances_namespace)
soma_distances = {
k: (v['U Distance'][0], v['V Distance'][0])
for (k, v) in distances
}
del distances
gids = sorted(soma_distances.keys())
distance_U = np.asarray([soma_distances[gid][0] for gid in gids])
distance_V = np.asarray([soma_distances[gid][1] for gid in gids])
return (distance_U, distance_V, degrees_dict)
def main(arena_id, config, config_prefix, dataset_prefix, distances_namespace, distance_limits, spike_input_path, spike_input_namespace, spike_input_attr, output_path, io_size, trajectory_id, write_selection, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
env = Env(comm=comm, config_file=config,
config_prefix=config_prefix, dataset_prefix=dataset_prefix,
results_path=output_path, spike_input_path=spike_input_path,
spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr,
arena_id=arena_id, trajectory_id=trajectory_id, io_size=io_size)
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm)
distance_U_dict = {}
distance_V_dict = {}
range_U_dict = {}
range_V_dict = {}
selection_dict = defaultdict(set)
comm0 = env.comm.Split(2 if rank == 0 else 0, 0)
if rank == 0:
for population in pop_ranges:
distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0)
soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k,v) in distances }
del distances
numitems = len(list(soma_distances.keys()))
logger.info('read %s distances (%i elements)' % (population, numitems))
if numitems == 0:
continue
distance_U_array = np.asarray([soma_distances[gid][0] for gid in soma_distances])
distance_V_array = np.asarray([soma_distances[gid][1] for gid in soma_distances])
U_min = np.min(distance_U_array)
U_max = np.max(distance_U_array)
V_min = np.min(distance_V_array)
V_max = np.max(distance_V_array)
range_U_dict[population] = (U_min, U_max)
range_V_dict[population] = (V_min, V_max)
distance_U = { gid: soma_distances[gid][0] for gid in soma_distances }
distance_V = { gid: soma_distances[gid][1] for gid in soma_distances }
distance_U_dict[population] = distance_U
distance_V_dict[population] = distance_V
min_dist = U_min
max_dist = U_max
if distance_limits:
min_dist = distance_limits[0]
max_dist = distance_limits[1]
selection_dict[population] = set([ k for k in distance_U if (distance_U[k] >= min_dist) and
(distance_U[k] <= max_dist) ])
yaml_output_dict = {}
for k, v in utils.viewitems(selection_dict):
yaml_output_dict[k] = list(v)
yaml_output_path = '%s/DG_slice.yaml' % output_path
with open(yaml_output_path, 'w') as outfile:
yaml.dump(yaml_output_dict, outfile)
del(yaml_output_dict)
env.comm.barrier()
write_selection_file_path = None
if write_selection:
write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName)
if write_selection_file_path is not None:
if rank == 0:
io_utils.mkout(env, write_selection_file_path)
env.comm.barrier()
selection_dict = env.comm.bcast(dict(selection_dict), root=0)
env.cell_selection = selection_dict
io_utils.write_cell_selection(env, write_selection_file_path)
input_selection = io_utils.write_connection_selection(env, write_selection_file_path)
io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path)
mapping = {name: idx for name, start, count, idx in defs}
dt = h5py.special_dtype(enum=(np.uint16, mapping))
h5[path_population_labels] = dt
dt = np.dtype([("Start", np.uint64), ("Count", np.uint32),
("Population", h5[path_population_labels].dtype)])
h5[path_population_range] = dt
# create an HDF5 compound type for population ranges
dt = h5[path_population_range].dtype
g = h5_get_group(h5, grp_h5types)
dset = h5_get_dataset(g, grp_populations, maxshape=(n_pop, ), dtype=dt)
dset.resize((n_pop, ))
a = np.zeros(n_pop, dtype=dt)
idx = 0
for name, start, count, idx in defs:
a[idx]["Start"] = start
a[idx]["Count"] = count
a[idx]["Population"] = idx
idx += 1
dset[:] = a
write_cell_attributes(output_path,
pop_name,
attr_dict,
namespace='Test Attributes')
print(list(read_cell_attributes(output_path, pop_name, 'Test Attributes')))
def main(arena_id, config, config_prefix, dataset_prefix, distances_namespace, spike_input_path, spike_input_namespace, spike_input_attr, input_features_namespaces, input_features_path, selection_path, output_path, io_size, trajectory_id, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
if io_size == -1:
io_size = comm.size
env = Env(comm=comm, config_file=config,
config_prefix=config_prefix, dataset_prefix=dataset_prefix,
results_path=output_path, spike_input_path=spike_input_path,
spike_input_namespace=spike_input_namespace, spike_input_attr=spike_input_attr,
arena_id=arena_id, trajectory_id=trajectory_id, io_size=io_size)
selection = []
f = open(selection_path, 'r')
for line in f.readlines():
selection.append(int(line))
f.close()
selection = set(selection)
pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path, comm=comm)
distance_U_dict = {}
distance_V_dict = {}
range_U_dict = {}
range_V_dict = {}
selection_dict = defaultdict(set)
comm0 = env.comm.Split(2 if rank == 0 else 0, 0)
if rank == 0:
for population in pop_ranges:
distances = read_cell_attributes(env.data_file_path, population, namespace=distances_namespace, comm=comm0)
soma_distances = { k: (v['U Distance'][0], v['V Distance'][0]) for (k,v) in distances }
del distances
numitems = len(list(soma_distances.keys()))
if numitems == 0:
continue
distance_U_array = np.asarray([soma_distances[gid][0] for gid in soma_distances])
distance_V_array = np.asarray([soma_distances[gid][1] for gid in soma_distances])
U_min = np.min(distance_U_array)
U_max = np.max(distance_U_array)
V_min = np.min(distance_V_array)
V_max = np.max(distance_V_array)
range_U_dict[population] = (U_min, U_max)
range_V_dict[population] = (V_min, V_max)
distance_U = { gid: soma_distances[gid][0] for gid in soma_distances }
distance_V = { gid: soma_distances[gid][1] for gid in soma_distances }
distance_U_dict[population] = distance_U
distance_V_dict[population] = distance_V
min_dist = U_min
max_dist = U_max
selection_dict[population] = set([ k for k in distance_U if k in selection ])
env.comm.barrier()
write_selection_file_path = "%s/%s_selection.h5" % (env.results_path, env.modelName)
if rank == 0:
io_utils.mkout(env, write_selection_file_path)
env.comm.barrier()
selection_dict = env.comm.bcast(dict(selection_dict), root=0)
env.cell_selection = selection_dict
io_utils.write_cell_selection(env, write_selection_file_path)
input_selection = io_utils.write_connection_selection(env, write_selection_file_path)
if spike_input_path:
io_utils.write_input_cell_selection(env, input_selection, write_selection_file_path)
if input_features_path:
for this_input_features_namespace in sorted(input_features_namespaces):
for population in sorted(input_selection):
logger.info(f"Extracting input features {this_input_features_namespace} for population {population}...")
it = read_cell_attribute_selection(input_features_path, population,
namespace=f"{this_input_features_namespace} {arena_id}",
selection=input_selection[population], comm=env.comm)
output_features_dict = { cell_gid : cell_features_dict for cell_gid, cell_features_dict in it }
append_cell_attributes(write_selection_file_path, population, output_features_dict,
namespace=f"{this_input_features_namespace} {arena_id}",
io_size=io_size, comm=env.comm)
env.comm.barrier()
def main(arena_id, bin_sample_count, config, config_prefix, dataset_prefix,
distances_namespace, distance_bin_extent, input_features_path,
input_features_namespaces, populations, spike_input_path,
spike_input_namespace, spike_input_attr, output_path, io_size,
trajectory_id, write_selection, verbose):
utils.config_logging(verbose)
logger = utils.get_script_logger(os.path.basename(__file__))
comm = MPI.COMM_WORLD
rank = comm.rank
env = Env(comm=comm,
config_file=config,
config_prefix=config_prefix,
dataset_prefix=dataset_prefix,
results_path=output_path,
spike_input_path=spike_input_path,
spike_input_namespace=spike_input_namespace,
spike_input_attr=spike_input_attr,
arena_id=arena_id,
trajectory_id=trajectory_id)
if io_size == -1:
io_size = comm.size
if rank == 0:
logger.info('%i ranks have been allocated' % comm.size)
pop_ranges, pop_size = read_population_ranges(env.connectivity_file_path,
comm=comm)
distance_U_dict = {}
distance_V_dict = {}
range_U_dict = {}
range_V_dict = {}
selection_dict = defaultdict(set)
comm0 = env.comm.Split(2 if rank == 0 else 0, 0)
local_random = np.random.RandomState()
local_random.seed(1000)
if len(populations) == 0:
populations = sorted(pop_ranges.keys())
if rank == 0:
for population in populations:
distances = read_cell_attributes(env.data_file_path,
population,
namespace=distances_namespace,
comm=comm0)
soma_distances = {}
if input_features_path is not None:
num_fields_dict = {}
for input_features_namespace in input_features_namespaces:
if arena_id is not None:
this_features_namespace = '%s %s' % (
input_features_namespace, arena_id)
else:
this_features_namespace = input_features_namespace
input_features_iter = read_cell_attributes(
input_features_path,
population,
namespace=this_features_namespace,
mask=set(['Num Fields']),
comm=comm0)
count = 0
for gid, attr_dict in input_features_iter:
num_fields_dict[gid] = attr_dict['Num Fields']
count += 1
logger.info(
'Read feature data from namespace %s for %i cells in population %s'
% (this_features_namespace, count, population))
for (gid, v) in distances:
num_fields = num_fields_dict.get(gid, 0)
if num_fields > 0:
soma_distances[gid] = (v['U Distance'][0],
v['V Distance'][0])
else:
for (gid, v) in distances:
soma_distances[gid] = (v['U Distance'][0],
v['V Distance'][0])
numitems = len(list(soma_distances.keys()))
logger.info('read %s distances (%i elements)' %
(population, numitems))
if numitems == 0:
continue
gid_array = np.asarray([gid for gid in soma_distances])
distance_U_array = np.asarray(
[soma_distances[gid][0] for gid in gid_array])
distance_V_array = np.asarray(
[soma_distances[gid][1] for gid in gid_array])
U_min = np.min(distance_U_array)
U_max = np.max(distance_U_array)
V_min = np.min(distance_V_array)
V_max = np.max(distance_V_array)
range_U_dict[population] = (U_min, U_max)
range_V_dict[population] = (V_min, V_max)
distance_U = {
gid: soma_distances[gid][0]
for gid in soma_distances
}
distance_V = {
gid: soma_distances[gid][1]
for gid in soma_distances
}
distance_U_dict[population] = distance_U
distance_V_dict[population] = distance_V
min_dist = U_min
max_dist = U_max
distance_bins = np.arange(U_min, U_max, distance_bin_extent)
distance_bin_array = np.digitize(distance_U_array, distance_bins)
selection_set = set([])
for bin_index in range(len(distance_bins) + 1):
bin_gids = gid_array[np.where(
distance_bin_array == bin_index)[0]]
if len(bin_gids) > 0:
selected_bin_gids = local_random.choice(
bin_gids, replace=False, size=bin_sample_count)
for gid in selected_bin_gids:
selection_set.add(int(gid))
selection_dict[population] = selection_set
yaml_output_dict = {}
for k, v in utils.viewitems(selection_dict):
yaml_output_dict[k] = list(sorted(v))
yaml_output_path = '%s/DG_slice.yaml' % output_path
with open(yaml_output_path, 'w') as outfile:
yaml.dump(yaml_output_dict, outfile)
del (yaml_output_dict)
env.comm.barrier()
write_selection_file_path = None
if write_selection:
write_selection_file_path = "%s/%s_selection.h5" % (env.results_path,
env.modelName)
if write_selection_file_path is not None:
if rank == 0:
io_utils.mkout(env, write_selection_file_path)
env.comm.barrier()
selection_dict = env.comm.bcast(dict(selection_dict), root=0)
env.cell_selection = selection_dict
io_utils.write_cell_selection(env,
write_selection_file_path,
populations=populations)
input_selection = io_utils.write_connection_selection(
env, write_selection_file_path, populations=populations)
if env.spike_input_ns is not None:
io_utils.write_input_cell_selection(env,
input_selection,
write_selection_file_path,
populations=populations)
env.comm.barrier()
MPI.Finalize()
from mpi4py import MPI
from neuroh5.io import read_trees, read_cell_attributes
import numpy as np
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
va = read_cell_attributes(
"/home/igr/src/model/dentate/datasets/Full_Scale_Control/dentate_Full_Scale_Control_coords_20170614.h5",
"MEC",
namespace="Sampled Coordinates")
ks = va.keys()
print ks
if rank == 0:
print "rank ", rank, ": len va.keys = ", len(ks)
print "rank ", rank, ": va[", ks[0], " = ", va[ks[0]].keys()
for k in va[ks[0]].keys():
print "rank ", rank, ": ", k, " size = ", va[ks[0]][k].size
print "rank ", rank, ": ", k, " = ", va[ks[0]][k]
if rank == 1:
print "rank ", rank, ": len va.keys = ", len(ks)
print "rank ", rank, ": va[", ks[0], " = ", va[ks[0]].keys()
for k in va[ks[0]].keys():
print "rank ", rank, ": ", k, " size = ", va[ks[0]][k].size
print "rank ", rank, ": ", k, " = ", va[ks[0]][k]
