Exemple #1
0
def _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth,
                    expand, compute_risk, cable_spread, path_confluence):
    """ Assumes all skeleton_ids belong to project_id. """
    skeletons_string = ",".join(str(int(x)) for x in skeleton_ids)
    cursor = connection.cursor()

    # Fetch all treenodes of all skeletons
    cursor.execute('''
    SELECT id, parent_id, confidence, skeleton_id,
           location_x, location_y, location_z
    FROM treenode
    WHERE skeleton_id IN (%s)
    ''' % skeletons_string)
    rows = tuple(cursor.fetchall())
    # Each skeleton is represented with a DiGraph
    arbors = defaultdict(nx.DiGraph)

    # Get reviewers for the requested skeletons
    reviews = get_treenodes_to_reviews(skeleton_ids=skeleton_ids)

    # Create a DiGraph for every skeleton
    for row in rows:
        arbors[row[3]].add_node(row[0],
                                {'reviewer_ids': reviews.get(row[0], [])})

    # Dictionary of skeleton IDs vs list of DiGraph instances
    arbors = split_by_confidence_and_add_edges(confidence_threshold, arbors,
                                               rows)

    # Fetch all synapses
    relations = {'presynaptic_to': -1, 'postsynaptic_to': -1}
    for r in Relation.objects.filter(
            relation_name__in=('presynaptic_to', 'postsynaptic_to'),
            project_id=project_id).values_list('relation_name', 'id'):
        relations[r[0]] = r[1]
    cursor.execute('''
    SELECT connector_id, relation_id, treenode_id, skeleton_id
    FROM treenode_connector
    WHERE skeleton_id IN (%s)
    ''' % skeletons_string)
    connectors = defaultdict(partial(defaultdict, list))
    skeleton_synapses = defaultdict(partial(defaultdict, list))
    for row in cursor.fetchall():
        connectors[row[0]][row[1]].append((row[2], row[3]))
        skeleton_synapses[row[3]][row[1]].append(row[2])

    # Cluster by synapses
    minis = defaultdict(list)  # skeleton_id vs list of minified graphs
    locations = None
    whole_arbors = arbors
    if expand and bandwidth > 0:
        locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
        treenode_connector = defaultdict(list)
        for connector_id, pp in connectors.iteritems():
            for treenode_id in chain.from_iterable(
                    pp[relations['presynaptic_to']]):
                treenode_connector[treenode_id].append(
                    (connector_id, "presynaptic_to"))
            for treenode_id in chain.from_iterable(
                    pp[relations['postsynaptic_to']]):
                treenode_connector[treenode_id].append(
                    (connector_id, "postsynaptic_to"))
        arbors_to_expand = {
            skid: ls
            for skid, ls in arbors.iteritems() if skid in expand
        }
        expanded_arbors, minis = split_by_synapse_domain(
            bandwidth, locations, arbors_to_expand, treenode_connector, minis)
        arbors.update(expanded_arbors)

    # Obtain neuron names
    cursor.execute('''
    SELECT cici.class_instance_a, ci.name
    FROM class_instance ci,
         class_instance_class_instance cici,
         relation r
    WHERE cici.class_instance_a IN (%s)
      AND cici.class_instance_b = ci.id
      AND cici.relation_id = r.id
      AND r.relation_name = 'model_of'
    ''' % skeletons_string)
    names = dict(cursor.fetchall())

    # A DiGraph representing the connections between the arbors (every node is an arbor)
    circuit = nx.DiGraph()

    for skid, digraphs in arbors.iteritems():
        base_label = names[skid]
        tag = len(digraphs) > 1
        i = 0
        for g in digraphs:
            if g.number_of_nodes() == 0:
                #print "no nodes in g, from skeleton ID #%s" % skid
                continue
            if tag:
                label = "%s [%s]" % (base_label, i + 1)
            else:
                label = base_label
            circuit.add_node(
                g,
                {
                    'id':
                    "%s_%s" % (skid, i + 1),
                    'label':
                    label,
                    'skeleton_id':
                    skid,
                    'node_count':
                    len(g),
                    'node_reviewed_count':
                    sum(
                        1 for v in g.node.itervalues()
                        if 0 != len(v.get('reviewer_ids', []))
                    ),  # TODO when bandwidth > 0, not all nodes are included. They will be included when the bandwidth is computed with an O(n) algorithm rather than the current O(n^2)
                    'branch':
                    False
                })
            i += 1

    # Define edges between arbors, with number of synapses as an edge property
    for c in connectors.itervalues():
        for pre_treenode, pre_skeleton in c[relations['presynaptic_to']]:
            for pre_arbor in arbors.get(pre_skeleton, ()):
                if pre_treenode in pre_arbor:
                    # Found the DiGraph representing an arbor derived from the skeleton to which the presynaptic treenode belongs.
                    for post_treenode, post_skeleton in c[
                            relations['postsynaptic_to']]:
                        for post_arbor in arbors.get(post_skeleton, ()):
                            if post_treenode in post_arbor:
                                # Found the DiGraph representing an arbor derived from the skeleton to which the postsynaptic treenode belongs.
                                edge_props = circuit.get_edge_data(
                                    pre_arbor, post_arbor)
                                if edge_props:
                                    edge_props['c'] += 1
                                    edge_props['pre_treenodes'].append(
                                        pre_treenode)
                                    edge_props['post_treenodes'].append(
                                        post_treenode)
                                else:
                                    circuit.add_edge(
                                        pre_arbor, post_arbor, {
                                            'c': 1,
                                            'pre_treenodes': [pre_treenode],
                                            'post_treenodes': [post_treenode],
                                            'arrow': 'triangle',
                                            'directed': True
                                        })
                                break
                    break

    if compute_risk and bandwidth <= 0:
        # Compute synapse risk:
        # Compute synapse centrality of every node in every arbor that has synapses
        for skeleton_id, arbors in whole_arbors.iteritems():
            synapses = skeleton_synapses[skeleton_id]
            pre = synapses[relations['presynaptic_to']]
            post = synapses[relations['postsynaptic_to']]
            for arbor in arbors:
                # The subset of synapses that belong to the fraction of the original arbor
                pre_sub = tuple(treenodeID for treenodeID in pre
                                if treenodeID in arbor)
                post_sub = tuple(treenodeID for treenodeID in post
                                 if treenodeID in arbor)

                totalInputs = len(pre_sub)
                totalOutputs = len(post_sub)
                tc = {treenodeID: Counts() for treenodeID in arbor}

                for treenodeID in pre_sub:
                    tc[treenodeID].outputs += 1

                for treenodeID in post_sub:
                    tc[treenodeID].inputs += 1

                # Update the nPossibleIOPaths field in the Counts instance of each treenode
                _node_centrality_by_synapse(arbor, tc, totalOutputs,
                                            totalInputs)

                arbor.treenode_synapse_counts = tc

        if not locations:
            locations = {row[0]: (row[4], row[5], row[6]) for row in rows}

        # Estimate the risk factor of the edge between two arbors,
        # as a function of the number of synapses and their location within the arbor.
        # Algorithm by Casey Schneider-Mizell
        # Implemented by Albert Cardona
        for pre_arbor, post_arbor, edge_props in circuit.edges_iter(data=True):
            if pre_arbor == post_arbor:
                # Signal autapse
                edge_props['risk'] = -2
                continue

            try:
                spanning = spanning_tree(post_arbor,
                                         edge_props['post_treenodes'])
                #for arbor in whole_arbors[circuit[post_arbor]['skeleton_id']]:
                #    if post_arbor == arbor:
                #        tc = arbor.treenode_synapse_counts
                tc = post_arbor.treenode_synapse_counts
                count = spanning.number_of_nodes()
                if count < 3:
                    median_synapse_centrality = sum(
                        tc[treenodeID].synapse_centrality
                        for treenodeID in spanning.nodes_iter()) / count
                else:
                    median_synapse_centrality = sorted(
                        tc[treenodeID].synapse_centrality
                        for treenodeID in spanning.nodes_iter())[count / 2]
                cable = cable_length(spanning, locations)
                if -1 == median_synapse_centrality:
                    # Signal not computable
                    edge_props['risk'] = -1
                else:
                    edge_props['risk'] = 1.0 / sqrt(
                        pow(cable / cable_spread, 2) +
                        pow(median_synapse_centrality / path_confluence, 2)
                    )  # NOTE: should subtract 1 from median_synapse_centrality, but not doing it here to avoid potential divisions by zero
            except Exception as e:
                print >> sys.stderr, e
                # Signal error when computing
                edge_props['risk'] = -3

    if expand and bandwidth > 0:
        # Add edges between circuit nodes that represent different domains of the same neuron
        for skeleton_id, list_mini in minis.iteritems():
            for mini in list_mini:
                for node in mini.nodes_iter():
                    g = mini.node[node]['g']
                    if 1 == len(g) and g.nodes_iter(
                            data=True).next()[1].get('branch'):
                        # A branch node that was preserved in the minified arbor
                        circuit.add_node(
                            g,
                            {
                                'id': '%s-%s' % (skeleton_id, node),
                                'skeleton_id': skeleton_id,
                                'label':
                                "",  # "%s [%s]" % (names[skeleton_id], node),
                                'node_count': 1,
                                'branch': True
                            })
                for node1, node2 in mini.edges_iter():
                    g1 = mini.node[node1]['g']
                    g2 = mini.node[node2]['g']
                    circuit.add_edge(g1, g2, {
                        'c': 10,
                        'arrow': 'none',
                        'directed': False
                    })

    return circuit
Exemple #2
0
def _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth,
        expand, compute_risk, cable_spread, path_confluence,
        pre_rel='presynaptic_to', post_rel='postsynaptic_to'):
    """ Assumes all skeleton_ids belong to project_id. """
    skeletons_string = ",".join(str(int(x)) for x in skeleton_ids)
    cursor = connection.cursor()

    # Fetch all treenodes of all skeletons
    cursor.execute('''
    SELECT id, parent_id, confidence, skeleton_id,
           location_x, location_y, location_z
    FROM treenode
    WHERE skeleton_id IN (%s)
    ''' % skeletons_string)
    rows = tuple(cursor.fetchall())
    # Each skeleton is represented with a DiGraph
    arbors = defaultdict(nx.DiGraph)

    # Get reviewers for the requested skeletons
    reviews = get_treenodes_to_reviews(skeleton_ids=skeleton_ids)

    # Create a DiGraph for every skeleton
    for row in rows:
        arbors[row[3]].add_node(row[0], {'reviewer_ids': reviews.get(row[0], [])})

    # Dictionary of skeleton IDs vs list of DiGraph instances
    arbors = split_by_confidence_and_add_edges(confidence_threshold, arbors, rows)

    # Fetch all synapses
    relations = get_relation_to_id_map(project_id, cursor=cursor)
    cursor.execute('''
    SELECT connector_id, relation_id, treenode_id, skeleton_id
    FROM treenode_connector
    WHERE skeleton_id IN (%s)
      AND (relation_id = %s OR relation_id = %s)
    ''' % (skeletons_string, relations[pre_rel], relations[post_rel]))
    connectors = defaultdict(partial(defaultdict, list))
    skeleton_synapses = defaultdict(partial(defaultdict, list))
    for row in cursor.fetchall():
        connectors[row[0]][row[1]].append((row[2], row[3]))
        skeleton_synapses[row[3]][row[1]].append(row[2])

    # Cluster by synapses
    minis = defaultdict(list) # skeleton_id vs list of minified graphs
    locations = None
    whole_arbors = arbors
    if expand and bandwidth > 0:
        locations = {row[0]: (row[4], row[5], row[6]) for row in rows}
        treenode_connector = defaultdict(list)
        for connector_id, pp in connectors.items():
            for treenode_id in chain.from_iterable(pp[relations[pre_rel]]):
                treenode_connector[treenode_id].append((connector_id, pre_rel))
            for treenode_id in chain.from_iterable(pp[relations[post_rel]]):
                treenode_connector[treenode_id].append((connector_id, post_rel))
        arbors_to_expand = {skid: ls for skid, ls in arbors.items() if skid in expand}
        expanded_arbors, minis = split_by_synapse_domain(bandwidth, locations, arbors_to_expand, treenode_connector, minis)
        arbors.update(expanded_arbors)


    # Obtain neuron names
    cursor.execute('''
    SELECT cici.class_instance_a, ci.name
    FROM class_instance ci,
         class_instance_class_instance cici
    WHERE cici.class_instance_a IN (%s)
      AND cici.class_instance_b = ci.id
      AND cici.relation_id = %s
    ''' % (skeletons_string, relations['model_of']))
    names = dict(cursor.fetchall())

    # A DiGraph representing the connections between the arbors (every node is an arbor)
    circuit = nx.DiGraph()

    for skid, digraphs in arbors.items():
        base_label = names[skid]
        tag = len(digraphs) > 1
        i = 0
        for g in digraphs:
            if g.number_of_nodes() == 0:
                continue
            if tag:
                label = "%s [%s]" % (base_label, i+1)
            else:
                label = base_label
            circuit.add_node(g, {'id': "%s_%s" % (skid, i+1),
                                 'label': label,
                                 'skeleton_id': skid,
                                 'node_count': len(g),
                                 'node_reviewed_count': sum(1 for v in g.node.values() if 0 != len(v.get('reviewer_ids', []))), # TODO when bandwidth > 0, not all nodes are included. They will be included when the bandwidth is computed with an O(n) algorithm rather than the current O(n^2)
                                 'branch': False})
            i += 1

    # Define edges between arbors, with number of synapses as an edge property
    for c in connectors.values():
        for pre_treenode, pre_skeleton in c[relations[pre_rel]]:
            for pre_arbor in arbors.get(pre_skeleton, ()):
                if pre_treenode in pre_arbor:
                    # Found the DiGraph representing an arbor derived from the skeleton to which the presynaptic treenode belongs.
                    for post_treenode, post_skeleton in c[relations[post_rel]]:
                        for post_arbor in arbors.get(post_skeleton, ()):
                            if post_treenode in post_arbor:
                                # Found the DiGraph representing an arbor derived from the skeleton to which the postsynaptic treenode belongs.
                                edge_props = circuit.get_edge_data(pre_arbor, post_arbor)
                                if edge_props:
                                    edge_props['c'] += 1
                                    edge_props['pre_treenodes'].append(pre_treenode)
                                    edge_props['post_treenodes'].append(post_treenode)
                                else:
                                    circuit.add_edge(pre_arbor, post_arbor, {'c': 1, 'pre_treenodes': [pre_treenode], 'post_treenodes': [post_treenode], 'arrow': 'triangle', 'directed': True})
                                break
                    break

    if compute_risk and bandwidth <= 0:
        # Compute synapse risk:
        # Compute synapse centrality of every node in every arbor that has synapses
        for skeleton_id, arbors in whole_arbors.items():
            synapses = skeleton_synapses[skeleton_id]
            pre = synapses[relations[pre_rel]]
            post = synapses[relations[post_rel]]
            for arbor in arbors:
                # The subset of synapses that belong to the fraction of the original arbor
                pre_sub = tuple(treenodeID for treenodeID in pre if treenodeID in arbor)
                post_sub = tuple(treenodeID for treenodeID in post if treenodeID in arbor)

                totalInputs = len(pre_sub)
                totalOutputs = len(post_sub)
                tc = {treenodeID: Counts() for treenodeID in arbor}

                for treenodeID in pre_sub:
                    tc[treenodeID].outputs += 1

                for treenodeID in post_sub:
                    tc[treenodeID].inputs += 1

                # Update the nPossibleIOPaths field in the Counts instance of each treenode
                _node_centrality_by_synapse(arbor, tc, totalOutputs, totalInputs)

                arbor.treenode_synapse_counts = tc

        if not locations:
            locations = {row[0]: (row[4], row[5], row[6]) for row in rows}

        # Estimate the risk factor of the edge between two arbors,
        # as a function of the number of synapses and their location within the arbor.
        # Algorithm by Casey Schneider-Mizell
        # Implemented by Albert Cardona
        for pre_arbor, post_arbor, edge_props in circuit.edges_iter(data=True):
            if pre_arbor == post_arbor:
                # Signal autapse
                edge_props['risk'] = -2
                continue

            try:
                spanning = spanning_tree(post_arbor, edge_props['post_treenodes'])
                #for arbor in whole_arbors[circuit[post_arbor]['skeleton_id']]:
                #    if post_arbor == arbor:
                #        tc = arbor.treenode_synapse_counts
                tc = post_arbor.treenode_synapse_counts
                count = spanning.number_of_nodes()
                if count < 3:
                    median_synapse_centrality = sum(tc[treenodeID].synapse_centrality for treenodeID in spanning.nodes_iter()) / count
                else:
                    median_synapse_centrality = sorted(tc[treenodeID].synapse_centrality for treenodeID in spanning.nodes_iter())[count / 2]
                cable = cable_length(spanning, locations)
                if -1 == median_synapse_centrality:
                    # Signal not computable
                    edge_props['risk'] = -1
                else:
                    edge_props['risk'] = 1.0 / sqrt(pow(cable / cable_spread, 2) + pow(median_synapse_centrality / path_confluence, 2)) # NOTE: should subtract 1 from median_synapse_centrality, but not doing it here to avoid potential divisions by zero
            except Exception as e:
                logging.getLogger(__name__).error(e)
                # Signal error when computing
                edge_props['risk'] = -3


    if expand and bandwidth > 0:
        # Add edges between circuit nodes that represent different domains of the same neuron
        for skeleton_id, list_mini in minis.items():
            for mini in list_mini:
                for node in mini.nodes_iter():
                    g = mini.node[node]['g']
                    if 1 == len(g) and next(g.nodes_iter(data=True))[1].get('branch'):
                        # A branch node that was preserved in the minified arbor
                        circuit.add_node(g, {'id': '%s-%s' % (skeleton_id, node),
                                             'skeleton_id': skeleton_id,
                                             'label': "", # "%s [%s]" % (names[skeleton_id], node),
                                             'node_count': 1,
                                             'branch': True})
                for node1, node2 in mini.edges_iter():
                    g1 = mini.node[node1]['g']
                    g2 = mini.node[node2]['g']
                    circuit.add_edge(g1, g2, {'c': 10, 'arrow': 'none', 'directed': False})

    return circuit