def _find_nodes(population, config=None, nodes_file=None, node_types_file=None): if nodes_file is not None: network = sonata.File(data_files=nodes_file, data_type_files=node_types_file) if population not in network.nodes.population_names: raise ValueError('node population "{}" not found in {}'.format(population, nodes_file)) return network.nodes[population] elif config is not None: for nodes_grp in config.nodes: network = sonata.File(data_files=nodes_grp['nodes_file'], data_type_files=nodes_grp['node_types_file']) if population in network.nodes.population_names: return network.nodes[population] raise ValueError('Could not find nodes file with node population "{}".'.format(population))
def from_config(cls, conf, **properties): """Generates a graph structure from a json config file or dictionary. :param conf: name of json config file, or a dictionary with config parameters :param properties: optional properties. :return: A graph object of type cls """ graph = cls(**properties) # The simulation run script should create a config-dict since it's likely to vary based on the simulator engine, # however in the case the user doesn't we will try a generic conversion from dict/json to ConfigDict if isinstance(conf, ConfigDict): config = conf else: try: config = ConfigDict.load(conf) except Exception as e: graph.io.log_exception('Could not convert {} (type "{}") to json.'.format(conf, type(conf))) if not config.with_networks: graph.io.log_exception('Could not find any network files. Unable to build network.') # TODO: These are simulator specific graph.spike_threshold = config.spike_threshold graph.dL = config.dL # load components for name, value in config.components.items(): graph.add_component(name, value) graph._validate_components() # load nodes gid_map = config.gid_mappings for node_dict in config.nodes: nodes_net = sonata.File(data_files=node_dict['nodes_file'], data_type_files=node_dict['node_types_file'], gid_table=gid_map) graph.add_nodes(nodes_net) # load edges for edge_dict in config.edges: target_network = edge_dict['target'] if 'target' in edge_dict else None source_network = edge_dict['source'] if 'source' in edge_dict else None edge_net = sonata.File(data_files=edge_dict['edges_file'], data_type_files=edge_dict['edge_types_file']) graph.add_edges(edge_net, source_pop=target_network, target_pop=source_network) graph._node_sets['all'] = NodeSetAll(graph) for ns_name, ns_filter in conf.node_sets.items(): graph._node_sets[ns_name] = NodeSet(ns_filter, graph) return graph
def import_nodes(self, nodes_file_name, node_types_file_name, population=None): sonata_file = sonata.File(data_files=nodes_file_name, data_type_files=node_types_file_name) if sonata_file.nodes is None: raise Exception('nodes file {} does not have any nodes.'.format( nodes_file_name)) populations = sonata_file.nodes.populations if len(populations) == 1: node_pop = populations[0] elif population is None: raise Exception('The nodes file {} contains multiple populations.'. format(nodes_file_name) + 'Please specify population parameter.') else: for pop in populations: if pop.name == population: node_pop = pop break else: raise Exception( 'Nodes file {} does not contain population {}.'.format( nodes_file_name, population)) for node_type_props in node_pop.node_types_table: self._add_node_type(node_type_props) for node in node_pop: self._node_id_gen.remove_id(node.node_id) self._nodes.append( Node(node.node_id, node.group_props, node.node_type_properties))
def select_nodes(): sonata_file = sonata.File( data_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5' ], data_type_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv' ]) nodes_df = sonata_file.nodes['v1'].to_dataframe(index_by_id=False)[[ 'node_id', 'pop_name' ]] edges_df = pd.read_csv('metadata/combined_edges.csv', sep=' ') l4_l2_edges_df = edges_df[edges_df['src_location'] == 'VisL4'][[ 'source_node_id', 'target_node_id' ]] l4_l2_edges_df = l4_l2_edges_df.rename(columns={ 'source_node_id': 'l4_node_id', 'target_node_id': 'l2_node_id' }) l4_l2_counts_df = l4_l2_edges_df.groupby( ['l4_node_id', 'l2_node_id']).size().to_frame('->').reset_index() l4_l2_counts_df = l4_l2_counts_df[l4_l2_counts_df['->'] > 3] l2_l4_edges_df = edges_df[edges_df['src_location'] == 'VisL23'] l2_l4_edges_df = l2_l4_edges_df.rename(columns={ 'source_node_id': 'l2_node_id', 'target_node_id': 'l4_node_id' }) l2_l4_counts_df = l2_l4_edges_df.groupby( ['l4_node_id', 'l2_node_id']).size().to_frame('<-').reset_index() sym_counts_df = l4_l2_counts_df.merge(l2_l4_counts_df, how='inner', on=['l4_node_id', 'l2_node_id']) sym_counts_df = sym_counts_df.merge(nodes_df, how='left', left_on='l4_node_id', right_on='node_id') sym_counts_df = sym_counts_df.rename(columns={'pop_name': 'src_pop_name'}) sym_counts_df = sym_counts_df.merge(nodes_df, how='left', left_on='l2_node_id', right_on='node_id') sym_counts_df = sym_counts_df.rename(columns={'pop_name': 'trg_pop_name'}) sym_counts_df = sym_counts_df.sort_values('->') sym_counts_df.to_csv('metadata/symmetric_edge_counts.csv', sep=' ', index=False) # print(l4_l2_counts_df.merge(l2_l4_counts_df, how='inner', on=['l4_node_id', 'l2_node_id']).sort_values('->')) print(sym_counts_df)
def load(cls, edges_h5, edge_types_csv, selected_populations=None, adaptor=EdgeAdaptor): sonata_file = sonata.File(data_files=edges_h5, data_type_files=edge_types_csv) edge_populations = [] for edge_pop in sonata_file.edges.populations: edge_populations.append(cls(edge_pop, adaptor)) return edge_populations
def sonata2csv(): sonata_file = sonata.File( data_files=['network/v1_nodes.h5', 'network/v1_v1_edges.h5'], data_type_files=[ 'network/v1_node_types.csv', 'network/v1_v1_edge_types.csv' ]) v1_nodes = sonata_file.nodes['v1'] nodes_df = v1_nodes.to_dataframe(index_by_id=False) nodes_df.to_csv('network/csv/nodes.csv', sep=' ', index=False) v1_edges = sonata_file.edges['v1_to_v1'] edges_df = v1_edges.get_group(0).to_dataframe() edges_df.to_csv('network/csv/edges.csv', sep=' ', index=False)
def load(cls, nodes_h5, node_types_csv, gid_table=None, selected_nodes=None, adaptor=NodeAdaptor): sonata_file = sonata.File(data_files=nodes_h5, data_type_files=node_types_csv, gid_table=gid_table) node_populations = [] for node_pop in sonata_file.nodes.populations: node_populations.append(cls(node_pop, adaptor)) return node_populations
def save_nodes_csv(circuit_config, population): config = SonataConfig.from_json(circuit_config) morphology_dir = config['components']['morphologies_dir'] config_dir = config['manifest']['configdir'] nodes_h5 = [n['nodes_file'] for n in config['networks']['nodes']] node_types_csv = [ n['node_types_file'] for n in config['networks']['nodes'] ] l4_net = sonata.File(data_files=nodes_h5, data_type_files=node_types_csv) net_df = l4_net.nodes[population].to_dataframe() for rot_axis in [ 'rotation_angle_xaxis', 'rotation_angle_yaxis', 'rotation_angle_zaxis' ]: if rot_axis not in net_df.columns: net_df[rot_axis] = 0.0 net_df[rot_axis] = net_df[rot_axis].fillna(0.0) net_df = net_df[[ 'x', 'y', 'z', 'rotation_angle_xaxis', 'rotation_angle_yaxis', 'rotation_angle_zaxis', 'morphology', 'model_processing', 'model_name', 'model_type' ]] p = PurePath(morphology_dir) morp_rel_path = p.relative_to(config_dir) net_df['morphology'] = net_df.apply( lambda r: os.path.join(morp_rel_path, r['morphology']) if isinstance(r['morphology'], string_types) else None, axis=1) net_df.to_csv(os.path.join(config_dir, 'network_cells.csv'), sep=' ', na_rep="None")
def save_synapses_csv(circuit_config, population): config = SonataConfig.from_json(circuit_config) config_dir = config['manifest']['configdir'] edges_h5 = [n['edges_file'] for n in config['networks']['edges']] edge_types_csv = [ n['edge_types_file'] for n in config['networks']['edges'] ] l4_net = sonata.File(data_files=edges_h5, data_type_files=edge_types_csv) l4_edges = l4_net.edges['{0}_to_{0}'.format(population)] l4_nodes_df = pd.read_csv(os.path.join(config_dir, 'network_cells.csv'), sep=' ') cells = {} for nid, r in l4_nodes_df.iterrows(): cells[nid] = CellSections.load_row(node_id=nid, row=r, config_dir=config_dir) src_node_ids = [] trg_node_ids = [] sec_ids = [] sec_xs = [] syn_x = [] syn_y = [] syn_z = [] for e in l4_edges: src_node_ids.append(e.source_node_id) trg_node_ids.append(e.target_node_id) cell = cells[e.target_node_id] print(cell.node_id, cell.model_type, cell.is_biophysical) if cell.is_biophysical: cell_secs = cells[e.target_node_id] sec_id = e['sec_id'] sec_x = e['sec_x'] syn_coords = cell_secs.get_coords(sec_id, sec_x) sec_ids.append(sec_id) sec_xs.append(sec_x) syn_x.append(syn_coords[0]) syn_y.append(syn_coords[1]) syn_z.append(syn_coords[2]) else: sec_ids.append(-1) sec_xs.append(-1) syn_x.append(np.nan) syn_y.append(np.nan) syn_z.append(np.nan) edges_df = pd.DataFrame({ 'target_node_id': trg_node_ids, 'source_node_id': src_node_ids, 'section_id': sec_ids, 'section_x': sec_xs, 'afferent_x': syn_x, 'afferent_y': syn_y, 'afferent_z': syn_z }) edges_df.to_csv(os.path.join(config_dir, 'network_synapses.csv'), sep=' ', index=False, na_rep=np.nan)
from bmtk.utils import sonata sf = sonata.File(data_files='network/v1_nodes.h5', data_type_files='network/v1_node_types.csv') sf.nodes.generate_gids('network/gids.h5')
def build_edges(source_node_id, target_node_id): sonata_file = sonata.File( data_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5', '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5' ], data_type_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv', '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edge_types.csv' ]) v1_edges = sonata_file.edges['v1_to_v1'] v1_nodes = sonata_file.nodes['v1'] l4_node = v1_nodes.get_node_id(source_node_id) l4_node_morphology_path = os.path.join('biophys_components/morphologies', l4_node['morphology']) l4_node_props = l4_node.node_type_properties l4_node_props.update({str(k): [v] for k, v in l4_node.group_props.items()}) l4_node_props['rotation_angle_zaxis'] *= -1 l2_node = v1_nodes.get_node_id(target_node_id) l2_node_morphology_path = os.path.join('biophys_components/morphologies', l2_node['morphology']) l2_node_props = l2_node.node_type_properties l2_node_props.update({str(k): [v] for k, v in l2_node.group_props.items()}) l2_node_props['rotation_angle_zaxis'] *= -1 swc_parsers = { source_node_id: SWCParser(l4_node_morphology_path), target_node_id: SWCParser(l2_node_morphology_path) } v1 = NetworkBuilder('v1') v1.add_nodes(**l4_node_props) v1.add_nodes(**l2_node_props) def query_str2dict(qry_str): qry_dict = {} for cond in qry_str.split('&'): k, v = cond.split('==') v = v.strip("\'") try: v = int(v) except ValueError: pass qry_dict[k] = v return qry_dict for src_id, trg_id in permutations([source_node_id, target_node_id]): sec_ids = [] sec_xs = [] syn_weights = [] edge_props = {} trg_swc = swc_parsers[trg_id] trg_swc_ids = [] trg_swc_dists = [] for e in v1_edges.get_source(src_id): if e.target_node_id == trg_id: sec_id = e._group_props['sec_id'] sec_x = e._group_props['sec_x'] swc_id, swc_dist = trg_swc.get_swc_id(sec_id, sec_x) trg_swc_ids.append(swc_id) trg_swc_dists.append(swc_dist) edge_props[ e._edge_type_props['edge_type_id']] = e._edge_type_props sec_ids.append(sec_id) sec_xs.append(sec_x) syn_weights.append(e._group_props['syn_weight']) for edge_type_id, edge_type_props in edge_props.items(): trg_query = query_str2dict(edge_type_props['target_query']) del edge_type_props['target_query'] src_query = query_str2dict(edge_type_props['source_query']) del edge_type_props['source_query'] cm = v1.add_edges(source=src_query, target=trg_query, connection_rule=len(syn_weights), **edge_type_props) # cm.add_properties('syn_weight', rule=lambda *_: np.random.uniform(0.0, 100.0), dtypes=np.float) cm.add_properties('afferent_section_id', rule=sec_ids, dtypes=np.int) cm.add_properties('afferent_section_pos', rule=sec_xs, dtypes=np.float) cm.add_properties('syn_weight', rule=syn_weights, dtypes=np.float) cm.add_properties('afferent_swc_id', rule=trg_swc_ids, dtypes=np.int) cm.add_properties('afferent_swc_pos', rule=trg_swc_dists, dtypes=np.float) v1.build() v1.save(output_dir='network')
def sample_df(): sonata_file = sonata.File( data_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_nodes.h5', '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5' ], data_type_files=[ '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_node_types.csv', '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edge_types.csv' ]) nodes_df = sonata_file.nodes['v1'].to_dataframe(index_by_id=False) nodes_df = nodes_df[['node_id', 'location', 'ei', 'model_type']] h5 = h5py.File( '/data/work_files/V1_network_update/Biophysical_network/network_fixed_synapses/v1_v1_edges.h5', 'r') edges_h5 = h5['/edges/v1_to_v1'] edges_df = pd.DataFrame({ 'target_node_id': edges_h5['target_node_id'], 'source_node_id': edges_h5['source_node_id'] }) # edges_df = sonata_file.edges['v1_to_v1'].get_group(0).to_dataframe() edges_df = edges_df.merge(nodes_df, how='left', left_on='target_node_id', right_on='node_id') edges_df = edges_df[edges_df['model_type'] == 'biophysical'] edges_df = edges_df.drop(columns=['node_id', 'model_type']) edges_df = edges_df.rename(columns={ 'location': 'trg_location', 'ei': 'trg_ei', 'morphology': 'trg_morphology' }) print(edges_df) edges_df = edges_df.merge(nodes_df, how='left', left_on='source_node_id', right_on='node_id') edges_df = edges_df[edges_df['model_type'] == 'biophysical'] edges_df = edges_df.drop(columns=['node_id', 'model_type']) edges_df = edges_df.rename(columns={ 'location': 'src_location', 'ei': 'src_ei', 'morphology': 'src_morphology' }) print(edges_df) e2e_edges_df = edges_df[(edges_df['src_ei'] == 'e') & (edges_df['trg_ei'] == 'e')] l4_l2_edges_df = e2e_edges_df[(e2e_edges_df['src_location'] == 'VisL4') & (e2e_edges_df['trg_location'] == 'VisL23')] l2_l4_edges_df = e2e_edges_df[(e2e_edges_df['src_location'] == 'VisL23') & (e2e_edges_df['trg_location'] == 'VisL4')] combined_edges_df = pd.concat([l2_l4_edges_df, l4_l2_edges_df]) combined_edges_df.to_csv('metadata/combined_edges.csv', sep=' ', index=False)
def from_config(cls, conf, **properties): """Generates a graph structure from a json config file or dictionary. :param conf: name of json config file, or a dictionary with config parameters :param properties: optional properties. :return: A graph object of type cls """ graph = cls(**properties) if isinstance(conf, basestring): config = graph._from_json(conf) elif isinstance(conf, dict): config = conf else: graph.io.log_exception( 'Could not convert {} (type "{}") to json.'.format( conf, type(conf))) run_dict = config['run'] if 'spike_threshold' in run_dict: # TODO: FIX, spike-thresholds should be set by simulation code, allow for diff. values based on node-group graph.spike_threshold = run_dict['spike_threshold'] if 'dL' in run_dict: graph.dL = run_dict['dL'] if not config.with_networks: graph.io.log_exception( 'Could not find any network files. Unable to build network.') # load components for name, value in config.components.items(): graph.add_component(name, value) graph._validate_components() # load nodes for node_dict in config.nodes: nodes_net = sonata.File( data_files=node_dict['nodes_file'], data_type_files=node_dict['node_types_file']) graph.add_nodes(nodes_net) # load edges for edge_dict in config.edges: target_network = edge_dict[ 'target'] if 'target' in edge_dict else None source_network = edge_dict[ 'source'] if 'source' in edge_dict else None edge_net = sonata.File( data_files=edge_dict['edges_file'], data_type_files=edge_dict['edge_types_file']) graph.add_edges(edge_net, source_pop=target_network, target_pop=source_network) ''' graph.io.log_info('Building cells.') graph.build_nodes() graph.io.log_info('Building recurrent connections') graph.build_recurrent_edges() ''' return graph