def _find_nodes(population, config=None, nodes_file=None, node_types_file=None):
if nodes_file is not None:
network = sonata.File(data_files=nodes_file, data_type_files=node_types_file)
if population not in network.nodes.population_names:
raise ValueError('node population "{}" not found in {}'.format(population, nodes_file))
return network.nodes[population]
elif config is not None:
for nodes_grp in config.nodes:
network = sonata.File(data_files=nodes_grp['nodes_file'], data_type_files=nodes_grp['node_types_file'])
if population in network.nodes.population_names:
return network.nodes[population]
raise ValueError('Could not find nodes file with node population "{}".'.format(population))
def from_config(cls, conf, **properties):
"""Generates a graph structure from a json config file or dictionary.
:param conf: name of json config file, or a dictionary with config parameters
:param properties: optional properties.
:return: A graph object of type cls
"""
graph = cls(**properties)
# The simulation run script should create a config-dict since it's likely to vary based on the simulator engine,
# however in the case the user doesn't we will try a generic conversion from dict/json to ConfigDict
if isinstance(conf, ConfigDict):
config = conf
else:
try:
config = ConfigDict.load(conf)
except Exception as e:
graph.io.log_exception('Could not convert {} (type "{}") to json.'.format(conf, type(conf)))
if not config.with_networks:
graph.io.log_exception('Could not find any network files. Unable to build network.')
# TODO: These are simulator specific
graph.spike_threshold = config.spike_threshold
graph.dL = config.dL
# load components
for name, value in config.components.items():
graph.add_component(name, value)
graph._validate_components()
# load nodes
gid_map = config.gid_mappings
for node_dict in config.nodes:
nodes_net = sonata.File(data_files=node_dict['nodes_file'], data_type_files=node_dict['node_types_file'],
gid_table=gid_map)
graph.add_nodes(nodes_net)
# load edges
for edge_dict in config.edges:
target_network = edge_dict['target'] if 'target' in edge_dict else None
source_network = edge_dict['source'] if 'source' in edge_dict else None
edge_net = sonata.File(data_files=edge_dict['edges_file'], data_type_files=edge_dict['edge_types_file'])
graph.add_edges(edge_net, source_pop=target_network, target_pop=source_network)
graph._node_sets['all'] = NodeSetAll(graph)
for ns_name, ns_filter in conf.node_sets.items():
graph._node_sets[ns_name] = NodeSet(ns_filter, graph)
return graph
def import_nodes(self,
nodes_file_name,
node_types_file_name,
population=None):
sonata_file = sonata.File(data_files=nodes_file_name,
data_type_files=node_types_file_name)
if sonata_file.nodes is None:
raise Exception('nodes file {} does not have any nodes.'.format(
nodes_file_name))
populations = sonata_file.nodes.populations
if len(populations) == 1:
node_pop = populations[0]
elif population is None:
raise Exception('The nodes file {} contains multiple populations.'.
format(nodes_file_name) +
'Please specify population parameter.')
else:
for pop in populations:
if pop.name == population:
node_pop = pop
break
else:
raise Exception(
'Nodes file {} does not contain population {}.'.format(
nodes_file_name, population))
for node_type_props in node_pop.node_types_table:
self._add_node_type(node_type_props)
for node in node_pop:
self._node_id_gen.remove_id(node.node_id)
self._nodes.append(
Node(node.node_id, node.group_props,
node.node_type_properties))
def select_nodes():
sonata_file = sonata.File(
data_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5'
],
data_type_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv'
])
nodes_df = sonata_file.nodes['v1'].to_dataframe(index_by_id=False)[[
'node_id', 'pop_name'
]]
edges_df = pd.read_csv('metadata/combined_edges.csv', sep=' ')
l4_l2_edges_df = edges_df[edges_df['src_location'] == 'VisL4'][[
'source_node_id', 'target_node_id'
]]
l4_l2_edges_df = l4_l2_edges_df.rename(columns={
'source_node_id': 'l4_node_id',
'target_node_id': 'l2_node_id'
})
l4_l2_counts_df = l4_l2_edges_df.groupby(
['l4_node_id', 'l2_node_id']).size().to_frame('->').reset_index()
l4_l2_counts_df = l4_l2_counts_df[l4_l2_counts_df['->'] > 3]
l2_l4_edges_df = edges_df[edges_df['src_location'] == 'VisL23']
l2_l4_edges_df = l2_l4_edges_df.rename(columns={
'source_node_id': 'l2_node_id',
'target_node_id': 'l4_node_id'
})
l2_l4_counts_df = l2_l4_edges_df.groupby(
['l4_node_id', 'l2_node_id']).size().to_frame('<-').reset_index()
sym_counts_df = l4_l2_counts_df.merge(l2_l4_counts_df,
how='inner',
on=['l4_node_id', 'l2_node_id'])
sym_counts_df = sym_counts_df.merge(nodes_df,
how='left',
left_on='l4_node_id',
right_on='node_id')
sym_counts_df = sym_counts_df.rename(columns={'pop_name': 'src_pop_name'})
sym_counts_df = sym_counts_df.merge(nodes_df,
how='left',
left_on='l2_node_id',
right_on='node_id')
sym_counts_df = sym_counts_df.rename(columns={'pop_name': 'trg_pop_name'})
sym_counts_df = sym_counts_df.sort_values('->')
sym_counts_df.to_csv('metadata/symmetric_edge_counts.csv',
sep=' ',
index=False)
# print(l4_l2_counts_df.merge(l2_l4_counts_df, how='inner', on=['l4_node_id', 'l2_node_id']).sort_values('->'))
print(sym_counts_df)
def load(cls,
edges_h5,
edge_types_csv,
selected_populations=None,
adaptor=EdgeAdaptor):
sonata_file = sonata.File(data_files=edges_h5,
data_type_files=edge_types_csv)
edge_populations = []
for edge_pop in sonata_file.edges.populations:
edge_populations.append(cls(edge_pop, adaptor))
return edge_populations
def sonata2csv():
sonata_file = sonata.File(
data_files=['network/v1_nodes.h5', 'network/v1_v1_edges.h5'],
data_type_files=[
'network/v1_node_types.csv', 'network/v1_v1_edge_types.csv'
])
v1_nodes = sonata_file.nodes['v1']
nodes_df = v1_nodes.to_dataframe(index_by_id=False)
nodes_df.to_csv('network/csv/nodes.csv', sep=' ', index=False)
v1_edges = sonata_file.edges['v1_to_v1']
edges_df = v1_edges.get_group(0).to_dataframe()
edges_df.to_csv('network/csv/edges.csv', sep=' ', index=False)
def load(cls,
nodes_h5,
node_types_csv,
gid_table=None,
selected_nodes=None,
adaptor=NodeAdaptor):
sonata_file = sonata.File(data_files=nodes_h5,
data_type_files=node_types_csv,
gid_table=gid_table)
node_populations = []
for node_pop in sonata_file.nodes.populations:
node_populations.append(cls(node_pop, adaptor))
return node_populations
def save_nodes_csv(circuit_config, population):
config = SonataConfig.from_json(circuit_config)
morphology_dir = config['components']['morphologies_dir']
config_dir = config['manifest']['configdir']
nodes_h5 = [n['nodes_file'] for n in config['networks']['nodes']]
node_types_csv = [
n['node_types_file'] for n in config['networks']['nodes']
]
l4_net = sonata.File(data_files=nodes_h5, data_type_files=node_types_csv)
net_df = l4_net.nodes[population].to_dataframe()
for rot_axis in [
'rotation_angle_xaxis', 'rotation_angle_yaxis',
'rotation_angle_zaxis'
]:
if rot_axis not in net_df.columns:
net_df[rot_axis] = 0.0
net_df[rot_axis] = net_df[rot_axis].fillna(0.0)
net_df = net_df[[
'x', 'y', 'z', 'rotation_angle_xaxis', 'rotation_angle_yaxis',
'rotation_angle_zaxis', 'morphology', 'model_processing', 'model_name',
'model_type'
]]
p = PurePath(morphology_dir)
morp_rel_path = p.relative_to(config_dir)
net_df['morphology'] = net_df.apply(
lambda r: os.path.join(morp_rel_path, r['morphology'])
if isinstance(r['morphology'], string_types) else None,
axis=1)
net_df.to_csv(os.path.join(config_dir, 'network_cells.csv'),
sep=' ',
na_rep="None")
def save_synapses_csv(circuit_config, population):
config = SonataConfig.from_json(circuit_config)
config_dir = config['manifest']['configdir']
edges_h5 = [n['edges_file'] for n in config['networks']['edges']]
edge_types_csv = [
n['edge_types_file'] for n in config['networks']['edges']
]
l4_net = sonata.File(data_files=edges_h5, data_type_files=edge_types_csv)
l4_edges = l4_net.edges['{0}_to_{0}'.format(population)]
l4_nodes_df = pd.read_csv(os.path.join(config_dir, 'network_cells.csv'),
sep=' ')
cells = {}
for nid, r in l4_nodes_df.iterrows():
cells[nid] = CellSections.load_row(node_id=nid,
row=r,
config_dir=config_dir)
src_node_ids = []
trg_node_ids = []
sec_ids = []
sec_xs = []
syn_x = []
syn_y = []
syn_z = []
for e in l4_edges:
src_node_ids.append(e.source_node_id)
trg_node_ids.append(e.target_node_id)
cell = cells[e.target_node_id]
print(cell.node_id, cell.model_type, cell.is_biophysical)
if cell.is_biophysical:
cell_secs = cells[e.target_node_id]
sec_id = e['sec_id']
sec_x = e['sec_x']
syn_coords = cell_secs.get_coords(sec_id, sec_x)
sec_ids.append(sec_id)
sec_xs.append(sec_x)
syn_x.append(syn_coords[0])
syn_y.append(syn_coords[1])
syn_z.append(syn_coords[2])
else:
sec_ids.append(-1)
sec_xs.append(-1)
syn_x.append(np.nan)
syn_y.append(np.nan)
syn_z.append(np.nan)
edges_df = pd.DataFrame({
'target_node_id': trg_node_ids,
'source_node_id': src_node_ids,
'section_id': sec_ids,
'section_x': sec_xs,
'afferent_x': syn_x,
'afferent_y': syn_y,
'afferent_z': syn_z
})
edges_df.to_csv(os.path.join(config_dir, 'network_synapses.csv'),
sep=' ',
index=False,
na_rep=np.nan)
from bmtk.utils import sonata
sf = sonata.File(data_files='network/v1_nodes.h5',
data_type_files='network/v1_node_types.csv')
sf.nodes.generate_gids('network/gids.h5')
def build_edges(source_node_id, target_node_id):
sonata_file = sonata.File(
data_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5',
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5'
],
data_type_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv',
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edge_types.csv'
])
v1_edges = sonata_file.edges['v1_to_v1']
v1_nodes = sonata_file.nodes['v1']
l4_node = v1_nodes.get_node_id(source_node_id)
l4_node_morphology_path = os.path.join('biophys_components/morphologies',
l4_node['morphology'])
l4_node_props = l4_node.node_type_properties
l4_node_props.update({str(k): [v] for k, v in l4_node.group_props.items()})
l4_node_props['rotation_angle_zaxis'] *= -1
l2_node = v1_nodes.get_node_id(target_node_id)
l2_node_morphology_path = os.path.join('biophys_components/morphologies',
l2_node['morphology'])
l2_node_props = l2_node.node_type_properties
l2_node_props.update({str(k): [v] for k, v in l2_node.group_props.items()})
l2_node_props['rotation_angle_zaxis'] *= -1
swc_parsers = {
source_node_id: SWCParser(l4_node_morphology_path),
target_node_id: SWCParser(l2_node_morphology_path)
}
v1 = NetworkBuilder('v1')
v1.add_nodes(**l4_node_props)
v1.add_nodes(**l2_node_props)
def query_str2dict(qry_str):
qry_dict = {}
for cond in qry_str.split('&'):
k, v = cond.split('==')
v = v.strip("\'")
try:
v = int(v)
except ValueError:
pass
qry_dict[k] = v
return qry_dict
for src_id, trg_id in permutations([source_node_id, target_node_id]):
sec_ids = []
sec_xs = []
syn_weights = []
edge_props = {}
trg_swc = swc_parsers[trg_id]
trg_swc_ids = []
trg_swc_dists = []
for e in v1_edges.get_source(src_id):
if e.target_node_id == trg_id:
sec_id = e._group_props['sec_id']
sec_x = e._group_props['sec_x']
swc_id, swc_dist = trg_swc.get_swc_id(sec_id, sec_x)
trg_swc_ids.append(swc_id)
trg_swc_dists.append(swc_dist)
edge_props[
e._edge_type_props['edge_type_id']] = e._edge_type_props
sec_ids.append(sec_id)
sec_xs.append(sec_x)
syn_weights.append(e._group_props['syn_weight'])
for edge_type_id, edge_type_props in edge_props.items():
trg_query = query_str2dict(edge_type_props['target_query'])
del edge_type_props['target_query']
src_query = query_str2dict(edge_type_props['source_query'])
del edge_type_props['source_query']
cm = v1.add_edges(source=src_query,
target=trg_query,
connection_rule=len(syn_weights),
**edge_type_props)
# cm.add_properties('syn_weight', rule=lambda *_: np.random.uniform(0.0, 100.0), dtypes=np.float)
cm.add_properties('afferent_section_id',
rule=sec_ids,
dtypes=np.int)
cm.add_properties('afferent_section_pos',
rule=sec_xs,
dtypes=np.float)
cm.add_properties('syn_weight', rule=syn_weights, dtypes=np.float)
cm.add_properties('afferent_swc_id',
rule=trg_swc_ids,
dtypes=np.int)
cm.add_properties('afferent_swc_pos',
rule=trg_swc_dists,
dtypes=np.float)
v1.build()
v1.save(output_dir='network')
def sample_df():
sonata_file = sonata.File(
data_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5',
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5'
],
data_type_files=[
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv',
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edge_types.csv'
])
nodes_df = sonata_file.nodes['v1'].to_dataframe(index_by_id=False)
nodes_df = nodes_df[['node_id', 'location', 'ei', 'model_type']]
h5 = h5py.File(
'/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5',
'r')
edges_h5 = h5['/edges/v1_to_v1']
edges_df = pd.DataFrame({
'target_node_id': edges_h5['target_node_id'],
'source_node_id': edges_h5['source_node_id']
})
# edges_df = sonata_file.edges['v1_to_v1'].get_group(0).to_dataframe()
edges_df = edges_df.merge(nodes_df,
how='left',
left_on='target_node_id',
right_on='node_id')
edges_df = edges_df[edges_df['model_type'] == 'biophysical']
edges_df = edges_df.drop(columns=['node_id', 'model_type'])
edges_df = edges_df.rename(columns={
'location': 'trg_location',
'ei': 'trg_ei',
'morphology': 'trg_morphology'
})
print(edges_df)
edges_df = edges_df.merge(nodes_df,
how='left',
left_on='source_node_id',
right_on='node_id')
edges_df = edges_df[edges_df['model_type'] == 'biophysical']
edges_df = edges_df.drop(columns=['node_id', 'model_type'])
edges_df = edges_df.rename(columns={
'location': 'src_location',
'ei': 'src_ei',
'morphology': 'src_morphology'
})
print(edges_df)
e2e_edges_df = edges_df[(edges_df['src_ei'] == 'e')
& (edges_df['trg_ei'] == 'e')]
l4_l2_edges_df = e2e_edges_df[(e2e_edges_df['src_location'] == 'VisL4')
& (e2e_edges_df['trg_location'] == 'VisL23')]
l2_l4_edges_df = e2e_edges_df[(e2e_edges_df['src_location'] == 'VisL23')
& (e2e_edges_df['trg_location'] == 'VisL4')]
combined_edges_df = pd.concat([l2_l4_edges_df, l4_l2_edges_df])
combined_edges_df.to_csv('metadata/combined_edges.csv',
sep=' ',
index=False)
def from_config(cls, conf, **properties):
"""Generates a graph structure from a json config file or dictionary.
:param conf: name of json config file, or a dictionary with config parameters
:param properties: optional properties.
:return: A graph object of type cls
"""
graph = cls(**properties)
if isinstance(conf, basestring):
config = graph._from_json(conf)
elif isinstance(conf, dict):
config = conf
else:
graph.io.log_exception(
'Could not convert {} (type "{}") to json.'.format(
conf, type(conf)))
run_dict = config['run']
if 'spike_threshold' in run_dict:
# TODO: FIX, spike-thresholds should be set by simulation code, allow for diff. values based on node-group
graph.spike_threshold = run_dict['spike_threshold']
if 'dL' in run_dict:
graph.dL = run_dict['dL']
if not config.with_networks:
graph.io.log_exception(
'Could not find any network files. Unable to build network.')
# load components
for name, value in config.components.items():
graph.add_component(name, value)
graph._validate_components()
# load nodes
for node_dict in config.nodes:
nodes_net = sonata.File(
data_files=node_dict['nodes_file'],
data_type_files=node_dict['node_types_file'])
graph.add_nodes(nodes_net)
# load edges
for edge_dict in config.edges:
target_network = edge_dict[
'target'] if 'target' in edge_dict else None
source_network = edge_dict[
'source'] if 'source' in edge_dict else None
edge_net = sonata.File(
data_files=edge_dict['edges_file'],
data_type_files=edge_dict['edge_types_file'])
graph.add_edges(edge_net,
source_pop=target_network,
target_pop=source_network)
'''
graph.io.log_info('Building cells.')
graph.build_nodes()
graph.io.log_info('Building recurrent connections')
graph.build_recurrent_edges()
'''
return graph
