def _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth,
expand, compute_risk, cable_spread, path_confluence):
""" Assumes all skeleton_ids belong to project_id. """
skeletons_string = ",".join(str(int(x)) for x in skeleton_ids)
cursor = connection.cursor()
# Fetch all treenodes of all skeletons
cursor.execute('''
SELECT id, parent_id, confidence, skeleton_id,
location_x, location_y, location_z
FROM treenode
WHERE skeleton_id IN (%s)
''' % skeletons_string)
rows = tuple(cursor.fetchall())
# Each skeleton is represented with a DiGraph
arbors = defaultdict(nx.DiGraph)
# Get reviewers for the requested skeletons
reviews = get_treenodes_to_reviews(skeleton_ids=skeleton_ids)
# Create a DiGraph for every skeleton
for row in rows:
arbors[row[3]].add_node(row[0],
{'reviewer_ids': reviews.get(row[0], [])})
# Dictionary of skeleton IDs vs list of DiGraph instances
arbors = split_by_confidence_and_add_edges(confidence_threshold, arbors,
rows)
# Fetch all synapses
relations = {'presynaptic_to': -1, 'postsynaptic_to': -1}
for r in Relation.objects.filter(
relation_name__in=('presynaptic_to', 'postsynaptic_to'),
project_id=project_id).values_list('relation_name', 'id'):
relations[r[0]] = r[1]
cursor.execute('''
SELECT connector_id, relation_id, treenode_id, skeleton_id
FROM treenode_connector
WHERE skeleton_id IN (%s)
''' % skeletons_string)
connectors = defaultdict(partial(defaultdict, list))
skeleton_synapses = defaultdict(partial(defaultdict, list))
for row in cursor.fetchall():
connectors[row[0]][row[1]].append((row[2], row[3]))
skeleton_synapses[row[3]][row[1]].append(row[2])
# Cluster by synapses
minis = defaultdict(list) # skeleton_id vs list of minified graphs
locations = None
whole_arbors = arbors
if expand and bandwidth > 0:
locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
treenode_connector = defaultdict(list)
for connector_id, pp in connectors.iteritems():
for treenode_id in chain.from_iterable(
pp[relations['presynaptic_to']]):
treenode_connector[treenode_id].append(
(connector_id, "presynaptic_to"))
for treenode_id in chain.from_iterable(
pp[relations['postsynaptic_to']]):
treenode_connector[treenode_id].append(
(connector_id, "postsynaptic_to"))
arbors_to_expand = {
skid: ls
for skid, ls in arbors.iteritems() if skid in expand
}
expanded_arbors, minis = split_by_synapse_domain(
bandwidth, locations, arbors_to_expand, treenode_connector, minis)
arbors.update(expanded_arbors)
# Obtain neuron names
cursor.execute('''
SELECT cici.class_instance_a, ci.name
FROM class_instance ci,
class_instance_class_instance cici,
relation r
WHERE cici.class_instance_a IN (%s)
AND cici.class_instance_b = ci.id
AND cici.relation_id = r.id
AND r.relation_name = 'model_of'
''' % skeletons_string)
names = dict(cursor.fetchall())
# A DiGraph representing the connections between the arbors (every node is an arbor)
circuit = nx.DiGraph()
for skid, digraphs in arbors.iteritems():
base_label = names[skid]
tag = len(digraphs) > 1
i = 0
for g in digraphs:
if g.number_of_nodes() == 0:
#print "no nodes in g, from skeleton ID #%s" % skid
continue
if tag:
label = "%s [%s]" % (base_label, i + 1)
else:
label = base_label
circuit.add_node(
g,
{
'id':
"%s_%s" % (skid, i + 1),
'label':
label,
'skeleton_id':
skid,
'node_count':
len(g),
'node_reviewed_count':
sum(
1 for v in g.node.itervalues()
if 0 != len(v.get('reviewer_ids', []))
), # TODO when bandwidth > 0, not all nodes are included. They will be included when the bandwidth is computed with an O(n) algorithm rather than the current O(n^2)
'branch':
False
})
i += 1
# Define edges between arbors, with number of synapses as an edge property
for c in connectors.itervalues():
for pre_treenode, pre_skeleton in c[relations['presynaptic_to']]:
for pre_arbor in arbors.get(pre_skeleton, ()):
if pre_treenode in pre_arbor:
# Found the DiGraph representing an arbor derived from the skeleton to which the presynaptic treenode belongs.
for post_treenode, post_skeleton in c[
relations['postsynaptic_to']]:
for post_arbor in arbors.get(post_skeleton, ()):
if post_treenode in post_arbor:
# Found the DiGraph representing an arbor derived from the skeleton to which the postsynaptic treenode belongs.
edge_props = circuit.get_edge_data(
pre_arbor, post_arbor)
if edge_props:
edge_props['c'] += 1
edge_props['pre_treenodes'].append(
pre_treenode)
edge_props['post_treenodes'].append(
post_treenode)
else:
circuit.add_edge(
pre_arbor, post_arbor, {
'c': 1,
'pre_treenodes': [pre_treenode],
'post_treenodes': [post_treenode],
'arrow': 'triangle',
'directed': True
})
break
break
if compute_risk and bandwidth <= 0:
# Compute synapse risk:
# Compute synapse centrality of every node in every arbor that has synapses
for skeleton_id, arbors in whole_arbors.iteritems():
synapses = skeleton_synapses[skeleton_id]
pre = synapses[relations['presynaptic_to']]
post = synapses[relations['postsynaptic_to']]
for arbor in arbors:
# The subset of synapses that belong to the fraction of the original arbor
pre_sub = tuple(treenodeID for treenodeID in pre
if treenodeID in arbor)
post_sub = tuple(treenodeID for treenodeID in post
if treenodeID in arbor)
totalInputs = len(pre_sub)
totalOutputs = len(post_sub)
tc = {treenodeID: Counts() for treenodeID in arbor}
for treenodeID in pre_sub:
tc[treenodeID].outputs += 1
for treenodeID in post_sub:
tc[treenodeID].inputs += 1
# Update the nPossibleIOPaths field in the Counts instance of each treenode
_node_centrality_by_synapse(arbor, tc, totalOutputs,
totalInputs)
arbor.treenode_synapse_counts = tc
if not locations:
locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
# Estimate the risk factor of the edge between two arbors,
# as a function of the number of synapses and their location within the arbor.
# Algorithm by Casey Schneider-Mizell
# Implemented by Albert Cardona
for pre_arbor, post_arbor, edge_props in circuit.edges_iter(data=True):
if pre_arbor == post_arbor:
# Signal autapse
edge_props['risk'] = -2
continue
try:
spanning = spanning_tree(post_arbor,
edge_props['post_treenodes'])
#for arbor in whole_arbors[circuit[post_arbor]['skeleton_id']]:
# if post_arbor == arbor:
# tc = arbor.treenode_synapse_counts
tc = post_arbor.treenode_synapse_counts
count = spanning.number_of_nodes()
if count < 3:
median_synapse_centrality = sum(
tc[treenodeID].synapse_centrality
for treenodeID in spanning.nodes_iter()) / count
else:
median_synapse_centrality = sorted(
tc[treenodeID].synapse_centrality
for treenodeID in spanning.nodes_iter())[count / 2]
cable = cable_length(spanning, locations)
if -1 == median_synapse_centrality:
# Signal not computable
edge_props['risk'] = -1
else:
edge_props['risk'] = 1.0 / sqrt(
pow(cable / cable_spread, 2) +
pow(median_synapse_centrality / path_confluence, 2)
) # NOTE: should subtract 1 from median_synapse_centrality, but not doing it here to avoid potential divisions by zero
except Exception as e:
print >> sys.stderr, e
# Signal error when computing
edge_props['risk'] = -3
if expand and bandwidth > 0:
# Add edges between circuit nodes that represent different domains of the same neuron
for skeleton_id, list_mini in minis.iteritems():
for mini in list_mini:
for node in mini.nodes_iter():
g = mini.node[node]['g']
if 1 == len(g) and g.nodes_iter(
data=True).next()[1].get('branch'):
# A branch node that was preserved in the minified arbor
circuit.add_node(
g,
{
'id': '%s-%s' % (skeleton_id, node),
'skeleton_id': skeleton_id,
'label':
"", # "%s [%s]" % (names[skeleton_id], node),
'node_count': 1,
'branch': True
})
for node1, node2 in mini.edges_iter():
g1 = mini.node[node1]['g']
g2 = mini.node[node2]['g']
circuit.add_edge(g1, g2, {
'c': 10,
'arrow': 'none',
'directed': False
})
return circuit
def _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth,
expand, compute_risk, cable_spread, path_confluence,
pre_rel='presynaptic_to', post_rel='postsynaptic_to'):
""" Assumes all skeleton_ids belong to project_id. """
skeletons_string = ",".join(str(int(x)) for x in skeleton_ids)
cursor = connection.cursor()
# Fetch all treenodes of all skeletons
cursor.execute('''
SELECT id, parent_id, confidence, skeleton_id,
location_x, location_y, location_z
FROM treenode
WHERE skeleton_id IN (%s)
''' % skeletons_string)
rows = tuple(cursor.fetchall())
# Each skeleton is represented with a DiGraph
arbors = defaultdict(nx.DiGraph)
# Get reviewers for the requested skeletons
reviews = get_treenodes_to_reviews(skeleton_ids=skeleton_ids)
# Create a DiGraph for every skeleton
for row in rows:
arbors[row[3]].add_node(row[0], {'reviewer_ids': reviews.get(row[0], [])})
# Dictionary of skeleton IDs vs list of DiGraph instances
arbors = split_by_confidence_and_add_edges(confidence_threshold, arbors, rows)
# Fetch all synapses
relations = get_relation_to_id_map(project_id, cursor=cursor)
cursor.execute('''
SELECT connector_id, relation_id, treenode_id, skeleton_id
FROM treenode_connector
WHERE skeleton_id IN (%s)
AND (relation_id = %s OR relation_id = %s)
''' % (skeletons_string, relations[pre_rel], relations[post_rel]))
connectors = defaultdict(partial(defaultdict, list))
skeleton_synapses = defaultdict(partial(defaultdict, list))
for row in cursor.fetchall():
connectors[row[0]][row[1]].append((row[2], row[3]))
skeleton_synapses[row[3]][row[1]].append(row[2])
# Cluster by synapses
minis = defaultdict(list) # skeleton_id vs list of minified graphs
locations = None
whole_arbors = arbors
if expand and bandwidth > 0:
locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
treenode_connector = defaultdict(list)
for connector_id, pp in connectors.items():
for treenode_id in chain.from_iterable(pp[relations[pre_rel]]):
treenode_connector[treenode_id].append((connector_id, pre_rel))
for treenode_id in chain.from_iterable(pp[relations[post_rel]]):
treenode_connector[treenode_id].append((connector_id, post_rel))
arbors_to_expand = {skid: ls for skid, ls in arbors.items() if skid in expand}
expanded_arbors, minis = split_by_synapse_domain(bandwidth, locations, arbors_to_expand, treenode_connector, minis)
arbors.update(expanded_arbors)
# Obtain neuron names
cursor.execute('''
SELECT cici.class_instance_a, ci.name
FROM class_instance ci,
class_instance_class_instance cici
WHERE cici.class_instance_a IN (%s)
AND cici.class_instance_b = ci.id
AND cici.relation_id = %s
''' % (skeletons_string, relations['model_of']))
names = dict(cursor.fetchall())
# A DiGraph representing the connections between the arbors (every node is an arbor)
circuit = nx.DiGraph()
for skid, digraphs in arbors.items():
base_label = names[skid]
tag = len(digraphs) > 1
i = 0
for g in digraphs:
if g.number_of_nodes() == 0:
continue
if tag:
label = "%s [%s]" % (base_label, i+1)
else:
label = base_label
circuit.add_node(g, {'id': "%s_%s" % (skid, i+1),
'label': label,
'skeleton_id': skid,
'node_count': len(g),
'node_reviewed_count': sum(1 for v in g.node.values() if 0 != len(v.get('reviewer_ids', []))), # TODO when bandwidth > 0, not all nodes are included. They will be included when the bandwidth is computed with an O(n) algorithm rather than the current O(n^2)
'branch': False})
i += 1
# Define edges between arbors, with number of synapses as an edge property
for c in connectors.values():
for pre_treenode, pre_skeleton in c[relations[pre_rel]]:
for pre_arbor in arbors.get(pre_skeleton, ()):
if pre_treenode in pre_arbor:
# Found the DiGraph representing an arbor derived from the skeleton to which the presynaptic treenode belongs.
for post_treenode, post_skeleton in c[relations[post_rel]]:
for post_arbor in arbors.get(post_skeleton, ()):
if post_treenode in post_arbor:
# Found the DiGraph representing an arbor derived from the skeleton to which the postsynaptic treenode belongs.
edge_props = circuit.get_edge_data(pre_arbor, post_arbor)
if edge_props:
edge_props['c'] += 1
edge_props['pre_treenodes'].append(pre_treenode)
edge_props['post_treenodes'].append(post_treenode)
else:
circuit.add_edge(pre_arbor, post_arbor, {'c': 1, 'pre_treenodes': [pre_treenode], 'post_treenodes': [post_treenode], 'arrow': 'triangle', 'directed': True})
break
break
if compute_risk and bandwidth <= 0:
# Compute synapse risk:
# Compute synapse centrality of every node in every arbor that has synapses
for skeleton_id, arbors in whole_arbors.items():
synapses = skeleton_synapses[skeleton_id]
pre = synapses[relations[pre_rel]]
post = synapses[relations[post_rel]]
for arbor in arbors:
# The subset of synapses that belong to the fraction of the original arbor
pre_sub = tuple(treenodeID for treenodeID in pre if treenodeID in arbor)
post_sub = tuple(treenodeID for treenodeID in post if treenodeID in arbor)
totalInputs = len(pre_sub)
totalOutputs = len(post_sub)
tc = {treenodeID: Counts() for treenodeID in arbor}
for treenodeID in pre_sub:
tc[treenodeID].outputs += 1
for treenodeID in post_sub:
tc[treenodeID].inputs += 1
# Update the nPossibleIOPaths field in the Counts instance of each treenode
_node_centrality_by_synapse(arbor, tc, totalOutputs, totalInputs)
arbor.treenode_synapse_counts = tc
if not locations:
locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
# Estimate the risk factor of the edge between two arbors,
# as a function of the number of synapses and their location within the arbor.
# Algorithm by Casey Schneider-Mizell
# Implemented by Albert Cardona
for pre_arbor, post_arbor, edge_props in circuit.edges_iter(data=True):
if pre_arbor == post_arbor:
# Signal autapse
edge_props['risk'] = -2
continue
try:
spanning = spanning_tree(post_arbor, edge_props['post_treenodes'])
#for arbor in whole_arbors[circuit[post_arbor]['skeleton_id']]:
# if post_arbor == arbor:
# tc = arbor.treenode_synapse_counts
tc = post_arbor.treenode_synapse_counts
count = spanning.number_of_nodes()
if count < 3:
median_synapse_centrality = sum(tc[treenodeID].synapse_centrality for treenodeID in spanning.nodes_iter()) / count
else:
median_synapse_centrality = sorted(tc[treenodeID].synapse_centrality for treenodeID in spanning.nodes_iter())[count / 2]
cable = cable_length(spanning, locations)
if -1 == median_synapse_centrality:
# Signal not computable
edge_props['risk'] = -1
else:
edge_props['risk'] = 1.0 / sqrt(pow(cable / cable_spread, 2) + pow(median_synapse_centrality / path_confluence, 2)) # NOTE: should subtract 1 from median_synapse_centrality, but not doing it here to avoid potential divisions by zero
except Exception as e:
logging.getLogger(__name__).error(e)
# Signal error when computing
edge_props['risk'] = -3
if expand and bandwidth > 0:
# Add edges between circuit nodes that represent different domains of the same neuron
for skeleton_id, list_mini in minis.items():
for mini in list_mini:
for node in mini.nodes_iter():
g = mini.node[node]['g']
if 1 == len(g) and next(g.nodes_iter(data=True))[1].get('branch'):
# A branch node that was preserved in the minified arbor
circuit.add_node(g, {'id': '%s-%s' % (skeleton_id, node),
'skeleton_id': skeleton_id,
'label': "", # "%s [%s]" % (names[skeleton_id], node),
'node_count': 1,
'branch': True})
for node1, node2 in mini.edges_iter():
g1 = mini.node[node1]['g']
g2 = mini.node[node2]['g']
circuit.add_edge(g1, g2, {'c': 10, 'arrow': 'none', 'directed': False})
return circuit