def node_update(request, project_id=None): treenodes = get_request_list(request.POST, "t") or [] connectors = get_request_list(request.POST, "c") or [] cursor = connection.cursor() nodes = treenodes + connectors if nodes: node_ids = [int(n[0]) for n in nodes] state.validate_state(node_ids, request.POST.get('state'), multinode=True, lock=True, cursor=cursor) now = timezone.now() old_treenodes = _update_location("treenode", treenodes, now, request.user, cursor) old_connectors = _update_location("connector", connectors, now, request.user, cursor) num_updated_nodes = len(treenodes) + len(connectors) return JsonResponse({ 'updated': num_updated_nodes, 'old_treenodes': old_treenodes, 'old_connectors': old_connectors })
def insert_treenode(request, project_id=None): """ Create a new treenode between two existing nodes. Its creator and creation_date information will be set to information of child node. No node will be created, if the node on the edge between the given child and parent node. """ # Use creation time, if part of parameter set params = {} float_values = {'x': 0, 'y': 0, 'z': 0, 'radius': 0} int_values = {'confidence': 0, 'parent_id': -1, 'child_id': -1} for p in float_values.keys(): params[p] = float(request.POST.get(p, float_values[p])) for p in int_values.keys(): params[p] = int(request.POST.get(p, int_values[p])) # If siblings should be taken over, all children of the parent node will be # come children of the inserted node. This requires extra state # information: the child state for the paren. takeover_child_ids = get_request_list(request.POST, 'takeover_child_ids', None, int) # Get optional initial links to connectors, expect each entry to be a list # of connector ID and relation ID. try: links = get_request_list(request.POST, 'links', [], int) except Exception, e: raise ValueError("Couldn't parse list parameter: {}".format(e))
def node_list_tuples(request, project_id=None, provider=None): ''' Retrieve an JSON array with four entries: [0] an array of arrays, each array representing a treenode [1] an array of arrays, each array representing a connector and containing arrays inside that specify the relations between the connector and treenodes. In this function tuples are used as much as possible for immutable list, and uses directly the tuples returned by the database cursor. [2] the labels, if requested. [3] a boolean which is true when the node limit has been reached. The returned JSON data is therefore sensitive to indices in the array, so care must be taken never to alter the order of the variables in the SQL statements without modifying the accesses to said data both in this function and in the client that consumes it. ''' project_id = int(project_id) # sanitize params = {} treenode_ids = get_request_list(request.POST, 'treenode_ids', map_fn=int) connector_ids = get_request_list(request.POST, 'connector_ids', map_fn=int) for p in ('top', 'left', 'bottom', 'right', 'z1', 'z2'): params[p] = float(request.POST.get(p, 0)) # Limit the number of retrieved treenodes within the section params['limit'] = settings.NODE_LIST_MAXIMUM_COUNT params['project_id'] = project_id include_labels = (request.POST.get('labels', None) == 'true') provider = get_treenodes_postgis return node_list_tuples_query(params, project_id, treenode_ids, connector_ids, include_labels, provider)
def circles_of_hell(request: HttpRequest, project_id) -> JsonResponse: """ Given a set of one or more skeleton IDs, find all skeletons that connect them (n_circles=1), or that connect to others that connect them (n_circles=2), etc. Returns a list of unique skeleton IDs that exclude the ones provided as argument. --- parameters: - name: skeleton_ids[] description: IDs of the skeletons to start expanding from. required: true type: array items: type: integer paramType: form - name: n_circles description: (Optional) The numbers of recursive expansions. required: false defaultValue: 1 type: integer paramType: form - name: allowed_connector_ids[] description: (Optional) IDs of connector nodes that are allowed to be used for expansion. required: false type: array items: type: integer paramType: form """ n_circles = int(request.POST.get('n_circles', 1)) if n_circles < 1: raise Exception("Requires at least one circle.") first_circle = set( get_request_list(request.POST, 'skeleton_ids', map_fn=int)) if not first_circle: raise Exception("No skeletons were provided.") cursor = connection.cursor() mins, relations = _clean_mins(request, cursor, int(project_id)) allowed_connector_ids = get_request_list(request.POST, 'allowed_connector_ids', None) current_circle = first_circle all_circles = first_circle while n_circles > 0 and current_circle: n_circles -= 1 connections = _next_circle(current_circle, relations, cursor, allowed_connector_ids) next_circle = set(skID for c in connections.values() \ for relationID, cs in c.items() \ for skID, count in cs.items() if count >= mins[relationID]) current_circle = next_circle - all_circles all_circles = all_circles.union(next_circle) skeleton_ids = tuple(all_circles - first_circle) return JsonResponse( [skeleton_ids, _neuronnames(skeleton_ids, project_id)], safe=False)
def add_multiple_sampler_domains(request, project_id, sampler_id): """Create a new domain for a sampler. --- parameters: - name: domains description: List of domains to add type: array: items: type: string required: true """ sampler_id = int(sampler_id) domains = get_request_list(request.POST, 'domains', map_fn) result_domains = [] for domain in domains: domain_type_id = domain.get('domain_type_id') if domain_type_id: domain_type_id = int(domain_type_id) else: raise ValueError("Need domain_type_id parameter") start_node_id = domain.get('start_node_id') if start_node_id: start_node_id = int(start_node_id) else: raise ValueError("Need start_node_id parameter") end_node_ids = get_request_list(domain, 'end_node_ids', map_fn=int) if not end_node_ids: raise ValueError("Need at least one valid end point") parent_interval_id = domain.get('parent_interval_id') if parent_interval_id: parent_interval_id = int(parent_interval_id) d = SamplerDomain.objects.create( sampler_id=sampler_id, start_node=start_node, domain_type=domain_type, parent_interval_id=parent_interval_id, user=request.user, project_id=project_id) result_domains.append({ "id": d.id, "sampler_id": d.sampler_id, "type_id": d.domain_type_id, "parent_interval": d.parent_interval_id, "start_node_id": d.start_node_id, "user_id": d.user_id, "project_id": d.project_id, "ends": [{ "id": e.id, "node_id": e.end_node_id } for e in domain_ends] }) return JsonResponse(result_domains, safe=False)
def annotate_entities(request, project_id=None): p = get_object_or_404(Project, pk=project_id) # Read keys in a sorted manner sorted_keys = sorted(request.POST.keys()) annotations = get_request_list(request.POST, 'annotations', []) meta_annotations = get_request_list(request.POST, 'meta_annotations', []) entity_ids = get_request_list(request.POST, 'entity_ids', [], map_fn=int) skeleton_ids = get_request_list(request.POST, 'skeleton_ids', [], map_fn=int) if any(skeleton_ids): skid_to_eid = dict( ClassInstance.objects.filter( project=p, class_column__class_name='neuron', cici_via_b__relation__relation_name='model_of', cici_via_b__class_instance_a__in=skeleton_ids).values_list( 'cici_via_b__class_instance_a', 'id')) entity_ids += [skid_to_eid[skid] for skid in skeleton_ids] # Annotate enties annotation_map = {a: request.user.id for a in annotations} annotation_objs, new_annotations = _annotate_entities( project_id, entity_ids, annotation_map) # Annotate annotations if meta_annotations: annotation_ids = [a.id for a in annotation_objs.keys()] meta_annotation_map = {ma: request.user.id for ma in meta_annotations} meta_annotation_objs, new_meta_annotations = _annotate_entities( project_id, annotation_ids, meta_annotation_map) # Keep track of new annotations new_annotations.update(new_meta_annotations) # Update used annotation objects set for ma, me in meta_annotation_objs.items(): entities = annotation_objs.get(ma) if entities: entities.update(me) else: annotation_objs[ma] = me result = { 'message': 'success', 'annotations': [{ 'name': a.name, 'id': a.id, 'entities': list(e) } for a, e in annotation_objs.items()], 'new_annotations': list(new_annotations) } return JsonResponse(result)
def connectors_info(request, project_id): """ Given a list of connectors, a list of presynaptic skeletons and a list of postsynatic skeletons, return a list of rows, one per synaptic connection, in the same format as one_to_many_synapses. The list of connectors is optional. """ int_to_str = lambda x: str(int(x)) cids = get_request_list(request.POST, 'cids', map_fn=int_to_str) skids_pre = get_request_list(request.POST, 'pre', map_fn=int_to_str) skids_post = get_request_list(request.POST, 'post', map_fn=int_to_str) cursor = connection.cursor() relations = get_relation_to_id_map(project_id, ('presynaptic_to', 'postsynaptic_to'), cursor) pre = relations['presynaptic_to'] post = relations['postsynaptic_to'] cursor.execute(''' SELECT DISTINCT tc1.connector_id, c.location_x, c.location_y, c.location_z, tc1.treenode_id, tc1.skeleton_id, tc1.confidence, tc1.user_id, t1.location_x, t1.location_y, t1.location_z, tc2.treenode_id, tc2.skeleton_id, tc2.confidence, tc2.user_id, t2.location_x, t2.location_y, t2.location_z FROM treenode_connector tc1, treenode_connector tc2, treenode t1, treenode t2, connector c WHERE %s tc1.connector_id = c.id AND tc1.connector_id = tc2.connector_id AND tc1.skeleton_id IN (%s) AND tc2.skeleton_id IN (%s) AND tc1.relation_id = %s AND tc2.relation_id = %s AND tc1.id != tc2.id AND tc1.treenode_id = t1.id AND tc2.treenode_id = t2.id ORDER BY tc2.skeleton_id ''' % ("c.id IN (%s) AND" % ",".join(cids) if cids else "", ",".join(skids_pre), ",".join(skids_post), pre, post)) rows = tuple((row[0], (row[1], row[2], row[3]), row[4], row[5], row[6], row[7], (row[8], row[9], row[10]), row[11], row[12], row[13], row[14], (row[15], row[16], row[17])) for row in cursor.fetchall()) return HttpResponse(json.dumps(rows))
def annotate_entities(request, project_id = None): p = get_object_or_404(Project, pk = project_id) # Read keys in a sorted manner sorted_keys = sorted(request.POST.keys()) annotations = get_request_list(request.POST, 'annotations', []) meta_annotations = get_request_list(request.POST, 'meta_annotations', []) entity_ids = get_request_list(request.POST, 'entity_ids', [], map_fn=int) skeleton_ids = get_request_list(request.POST, 'skeleton_ids', [], map_fn=int) if any(skeleton_ids): skid_to_eid = dict(ClassInstance.objects.filter(project = p, class_column__class_name = 'neuron', cici_via_b__relation__relation_name = 'model_of', cici_via_b__class_instance_a__in = skeleton_ids).values_list( 'cici_via_b__class_instance_a', 'id')) entity_ids += [skid_to_eid[skid] for skid in skeleton_ids] # Annotate enties annotation_map = {a: request.user.id for a in annotations} annotation_objs, new_annotations = _annotate_entities(project_id, entity_ids, annotation_map) # Annotate annotations if meta_annotations: annotation_ids = [a.id for a in annotation_objs.keys()] meta_annotation_map = {ma: request.user.id for ma in meta_annotations} meta_annotation_objs, new_meta_annotations = _annotate_entities( project_id, annotation_ids, meta_annotation_map) # Keep track of new annotations new_annotations.update(new_meta_annotations) # Update used annotation objects set for ma, me in meta_annotation_objs.items(): entities = annotation_objs.get(ma) if entities: entities.update(me) else: annotation_objs[ma] = me result = { 'message': 'success', 'annotations': [{ 'name': a.name, 'id': a.id, 'entities': list(e) } for a,e in annotation_objs.items()], 'new_annotations': list(new_annotations) } return JsonResponse(result)
def skeleton_graph(request: HttpRequest, project_id=None) -> JsonResponse: project_id = int(project_id) skeleton_ids = set( int(v) for k, v in request.POST.items() if k.startswith('skeleton_list[')) confidence_threshold = int(request.POST.get('confidence_threshold', 0)) bandwidth = float(request.POST.get('bandwidth', 0)) # in nanometers cable_spread = float(request.POST.get('cable_spread', 2500)) # in nanometers path_confluence = int(request.POST.get('path_confluence', 10)) # a count compute_risk = 1 == int(request.POST.get('risk', 0)) expand = set( int(v) for k, v in request.POST.items() if k.startswith('expand[')) link_types = get_request_list(request.POST, 'link_types', None) by_link_type = bool(link_types) if not by_link_type: link_types = ['synaptic-connector'] result = {} # type: ignore for link_type in link_types: pair = KNOWN_LINK_PAIRS.get(link_type) if not pair: raise ValueError(f"Unknown link type: {link_type}") source_rel = pair['source'] target_rel = pair['target'] circuit = _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth, expand, compute_risk, cable_spread, path_confluence, source_rel, target_rel) package: Dict[str, Any] = { 'nodes': [{ 'data': props } for props in circuit.nodes.values()], 'edges': [] } edges: List = package['edges'] for g1, g2, props in circuit.edges(data=True): id1 = circuit.nodes[g1]['id'] id2 = circuit.nodes[g2]['id'] data = { 'id': '%s_%s' % (id1, id2), 'source': id1, 'target': id2, 'weight': props['c'], 'label': str(props['c']) if props['directed'] else None, 'directed': props['directed'], 'arrow': props['arrow'] } if compute_risk: data['risk'] = props.get('risk') edges.append({'data': data}) if by_link_type: result[link_type] = package else: result = package return JsonResponse(result, safe=False)
def volume_collection(request: HttpRequest, project_id) -> JsonResponse: """Get a collection of all available volumes. --- parameters: - name: project_id description: Project to operate in type: integer paramType: path required: true - name: volume_ids description: Only return specified volumes paramType: form type: array items: type: integer required: false """ if request.method == 'GET': data = request.GET elif request.method == 'POST': data = request.POST else: raise ValueError("Unsupported HTTP method" + request.method) volume_ids = get_request_list(data, 'volume_ids', [], map_fn=int) return JsonResponse(_volume_collection(project_id, volume_ids))
def remove_annotation(request, project_id=None, annotation_id=None): """ Removes an annotation from one or more entities. """ entity_ids = get_request_list(request.POST, 'entity_ids', [], map_fn=int) cicis_to_delete, missed_cicis, deleted, num_left = _remove_annotation( request.user, project_id, entity_ids, annotation_id) if len(cicis_to_delete) > 1: message = "Removed annotation from %s entities." % len(cicis_to_delete) elif len(cicis_to_delete) == 1: message = "Removed annotation from one entity." else: message = "No annotation removed." if missed_cicis: message += " Couldn't de-annotate %s entities, due to the lack of " \ "permissions." % len(missed_cicis) if deleted: message += " Also removed annotation instance, because it isn't used " \ "anywhere else." else: message += " There are %s links left to this annotation." % num_left return JsonResponse({ 'message': message, 'deleted_annotation': deleted, 'left_uses': num_left })
def get_partners(request, project_id=None): # todo: document, test syn_ids = get_request_list(request.POST, 'synapse_object_ids', tuple(), int) response = { 'columns': ['synapse_object_id', 'tnids', 'skid', 'contact_px'], 'data': [] } if not syn_ids: return JsonResponse(response) cursor = connection.cursor() cursor.execute(''' SELECT ss_so.synapse_object_id, array_agg(tn.id), tn.skeleton_id, sum(ss_tn.contact_px) FROM synapse_slice_synapse_object ss_so INNER JOIN unnest(%(syns)s::bigint[]) AS syns(id) ON ss_so.synapse_object_id = syns.id INNER JOIN synapse_slice_treenode ss_tn ON ss_tn.synapse_slice_id = ss_so.synapse_slice_id INNER JOIN treenode tn ON tn.id = ss_tn.treenode_id WHERE tn.project_id = %(pid)s GROUP BY ss_so.synapse_object_id, tn.skeleton_id; ''', {'pid': project_id, 'syns': syn_ids}) response['data'] = cursor.fetchall() return JsonResponse(response)
def post(self, request: Request, project_id) -> Response: """Create a new project token. The request requires admin permissions in the project. --- serializer: SimpleProjectTokenSerializer """ project = get_object_or_404(Project, pk=project_id) name = request.POST.get('name', '') needs_approval = get_request_bool(request.POST, 'needs_approval', False) default_permissions = set( get_request_list(request.POST, 'default_permissions', [])) allowed_permissions = set( get_perms_for_model(Project).values_list('codename', flat=True)) unknown_permissions = default_permissions - allowed_permissions if unknown_permissions: raise ValueError( f'Unknown permissions: {", ".join(unknown_permissions)}') token = ProjectToken.objects.create( **{ 'name': name, 'user_id': request.user.id, 'project_id': project.id, 'needs_approval': needs_approval, 'default_permissions': default_permissions, }) if not name: token.name = f'Project token {token.id}' token.save() serializer = SimpleProjectTokenSerializer(token) return Response(serializer.data)
def treenodes_by_label(request, project_id=None): """Return all treenodes in the project associated with the given tags. --- parameters: - name: tags description: label names to find treenodes associated with required: true type: array items: type: str paramType: form """ # todo: test labels = get_request_list(request.GET, 'tags', tuple()) columns = ['tag_name', 'treenode_id', 'xp', 'yp', 'zp'] if not labels: return JsonResponse({'columns': columns, 'data': []}) labeled_as_relation = Relation.objects.get(project=project_id, relation_name='labeled_as') cursor = connection.cursor() cursor.execute(""" SELECT ci.name, t.id, t.location_x, t.location_y, t.location_z FROM class_instance ci JOIN treenode_class_instance tci ON tci.class_instance_id = ci.id JOIN treenode t ON tci.treenode_id = t.id WHERE ci.project_id = %s AND tci.relation_id = %s AND ci.name = ANY(%s); """, (project_id, labeled_as_relation.id, labels)) return JsonResponse({'columns': columns, 'data': cursor.fetchall()})
def many_to_many_synapses(request:HttpRequest, project_id=None) -> JsonResponse: """ Return the list of synapses of a specific kind between one list of skeletons and a list of other skeletons. """ skids1 = get_request_list(request.POST, 'skids1', map_fn=int) if not skids1: raise ValueError("No skeleton IDs for first list of 'many' provided") skids2 = get_request_list(request.POST, 'skids2', map_fn=int) if not skids2: raise ValueError("No skeleton IDs for second list 'many' provided") relation_name = request.POST.get('relation') rows = _many_to_many_synapses(skids1, skids2, relation_name, project_id) return JsonResponse(rows, safe=False)
def many_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one list of skeletons and a list of other skeletons. """ skids1 = get_request_list(request.POST, 'skids1', map_fn=int) if not skids1: raise ValueError("No skeleton IDs for first list of 'many' provided") skids2 = get_request_list(request.POST, 'skids2', map_fn=int) if not skids2: raise ValueError("No skeleton IDs for second list 'many' provided") relation_name = request.POST.get('relation') # expecting presynaptic_to, postsynaptic_to, or gapjunction_with rows = _many_to_many_synapses(skids1, skids2, relation_name, project_id) return HttpResponse(json.dumps(rows))
def get_volume_entities(request, project_id): """Retrieve a mapping of volume IDs to entity (class instance) IDs. --- parameters: - name: volume_ids description: A list of volume IDs to map paramType: query """ volume_ids = get_request_list(request.POST, 'volume_ids', map_fn=int) cursor = connection.cursor() cursor.execute( """ SELECT vci.volume_id, vci.class_instance_id FROM volume_class_instance vci JOIN UNNEST(%(volume_ids)s::int[]) volume(id) ON volume.id = vci.volume_id WHERE project_id = %(project_id)s AND relation_id = ( SELECT id FROM relation WHERE relation_name = 'model_of' AND project_id = %(project_id)s ) """, { 'volume_ids': volume_ids, 'project_id': project_id }) return JsonResponse(dict(cursor.fetchall()))
def many_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one list of skeletons and a list of other skeletons. """ skids1 = get_request_list(request.POST, 'skids1', map_fn=int) if not skids1: raise ValueError("No skeleton IDs for first list of 'many' provided") skids2 = get_request_list(request.POST, 'skids2', map_fn=int) if not skids2: raise ValueError("No skeleton IDs for second list 'many' provided") relation_name = request.POST.get('relation') rows = _many_to_many_synapses(skids1, skids2, relation_name, project_id) return JsonResponse(rows, safe=False)
def list_broken_section_nodes(request, project_id=None): """List nodes that are located in a broken section. Broken secrions of all stacks linked to the current project are tested if they contain any nodes. Stack orientatins are respected. Optionally, only particular skeletons can be checked. --- parameters: - name: 'skeleton_ids' description: List of skeleton IDs to constrain tests on type: array item: integer required: false type: - type: array items: type: string description: A list of lists, each containing [treenode_id, stack_id, stack_title, orientation, section, section_physical] required: true """ project_id = int(project_id) skeleton_ids = get_request_list(request.POST, 'skeleton_ids', map_fn=int) broken_section_nodes = check_broken_section(project_id, skeleton_ids) return JsonResponse(broken_section_nodes, safe=False)
def list_broken_section_nodes(request, project_id=None) -> JsonResponse: """List nodes that are located in a broken section. Broken secrions of all stacks linked to the current project are tested if they contain any nodes. Stack orientatins are respected. Optionally, only particular skeletons can be checked. --- parameters: - name: 'skeleton_ids' description: List of skeleton IDs to constrain tests on type: array item: integer required: false type: - type: array items: type: string description: A list of lists, each containing [treenode_id, stack_id, stack_title, orientation, section, section_physical] required: true """ project_id = int(project_id) skeleton_ids = get_request_list(request.POST, 'skeleton_ids', map_fn=int) broken_section_nodes = check_broken_section(project_id, skeleton_ids) return JsonResponse(broken_section_nodes, safe=False)
def get_volume_entities(request, project_id): """Retrieve a mapping of volume IDs to entity (class instance) IDs. --- parameters: - name: volume_ids description: A list of volume IDs to map paramType: query """ volume_ids = get_request_list(request.POST, 'volume_ids', map_fn=int) cursor = connection.cursor() cursor.execute(""" SELECT vci.volume_id, vci.class_instance_id FROM volume_class_instance vci JOIN UNNEST(%(volume_ids)s::int[]) volume(id) ON volume.id = vci.volume_id WHERE project_id = %(project_id)s AND relation_id = ( SELECT id FROM relation WHERE relation_name = 'model_of' AND project_id = %(project_id)s ) """, { 'volume_ids': volume_ids, 'project_id': project_id }) return JsonResponse(dict(cursor.fetchall()))
def connectors_from_treenodes(request:HttpRequest, project_id) -> JsonResponse: """Get a list of connectors that are linked to a set of treenodes. --- parameters: - name: project_id description: The project to operate in. type: integeger paramType: path required: true - name: treenode_ids description: Treenode IDs that result nodes are connected to. type: array items: type: integer paramType: form required: true """ treenode_ids = get_request_list(request.POST, 'treenode_ids', map_fn=int) cursor = connection.cursor() cursor.execute(""" SELECT DISTINCT ON (connector_id) connector_id FROM treenode_connector tc JOIN UNNEST(%(treenode_ids)s::bigint[]) query_treenode(id) ON query_treenode.id = tc.treenode_id WHERE project_id = %(project_id)s """, { 'project_id': project_id, 'treenode_ids': treenode_ids, }) return JsonResponse({ 'connector_ids': [c[0] for c in cursor.fetchall()], })
def post(self, request: Request, project_id) -> Response: """ Update the group membership of multiple users at once. Users and groups as well as their memberships are global, therefore this action requires either superuser status or project tokens need to be in use. If the latter is the case, the requesting user is expected to have a) admin permission in the current project and is b) only allowed to change users and groups visible to them. """ action = request.POST.get('action') if action not in ('add', 'revoke'): raise ValueError('Action needs to be "add" or "revoke"') # Collect user and group information source_users = set( get_request_list(request.POST, 'source_users', [], int)) source_groups = set( get_request_list(request.POST, 'source_groups', [], int)) target_users = set( get_request_list(request.POST, 'target_users', [], int)) target_groups = set( get_request_list(request.POST, 'target_groups', [], int)) # Check permissions if settings.PROJECT_TOKEN_USER_VISIBILITY and not request.user.is_superuser: # Find all visible users and groups visible_user_ids = get_token_visible_users(request.user.id) visible_group_ids = get_token_visible_groups(request.user.id) invisible_user_ids = (set(source_users).union( set(target_users))).difference(set(visible_user_ids)) if invisible_user_ids: raise PermissionError( 'This request includes users beyond the allowed scope') elif not request.user.is_superuser: raise PermissionError('Need superuser permission') updated, warnings = update_group_memberships(action, source_users, source_groups, target_users, target_groups) return JsonResponse({ 'updated_users': updated, 'warnings': warnings, })
def post(self, request: HttpRequest, project_id) -> JsonResponse: """List all available point clouds or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point clouds type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_images description: Wheter linked images should returned as well. type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: sample_ratio description: Number in [0,1] to optionally sample point cloud type: number paramType: form required: false - name: pointcloud_ids description: A list of point cloud IDs to which the query is constrained. type: array paramType: path required: false - name: order_by description: The field to order the response list by (name, id). type: string paramType: path required: false defaultValue: 'id' """ with_images = get_request_bool(request.POST, 'with_images', False) with_points = get_request_bool(request.POST, 'with_points', False) sample_ratio = float(request.POST.get('sample_ratio', '1.0')) simple = get_request_bool(request.POST, 'simple', False) pointcloud_ids = get_request_list(request.POST, 'pointcloud_ids', None, map_fn=int) order_by = request.query_params.get('order_by', 'id') pointclouds = list_pointclouds(project_id, request.user.id, simple, with_images, with_points, sample_ratio, pointcloud_ids, order_by) return JsonResponse(pointclouds, safe=False)
def project_user_permission_set(request: HttpRequest, project_id: int) -> JsonResponse: """ If a user is authenticated and has can_administer permissions in this project, a list of user permissions for the current project is returned. If the user has no permission to view this informaiton, a permission error is raised. """ project = Project.objects.get(id=project_id) checker = ObjectPermissionChecker(request.user) has_admin_role = checker.has_perm('can_administer', project) if not request.user.is_authenticated or \ not (request.user.is_superuser or has_admin_role): raise PermissionError("User needs to be logged in ans superuser") if request.method == 'GET': project_perms = get_perms_for_model(Project) perm_names = [perm.codename for perm in project_perms] # Get all user permissions for this project users_with_perms = dict((u.id, {"permissions": l, "user_name": u.username}) for u,l in \ get_users_with_perms(project, attach_perms=True, with_group_users=False).items()) return JsonResponse(users_with_perms) elif request.method == 'POST': target_user_id = request.POST.get('user_id') if target_user_id is None: raise ValueError("Need target user ID (user_id)") target_user = User.objects.get(id=target_user_id) permissions = get_request_list(request.POST, 'permissions') if not permissions: raise ValueError("Need permissions to update") permissions = dict([(perm, b.lower() == 'true') for perm, b in \ get_request_list(request.POST, 'permissions')]) for perm, enabled in permissions.items(): if enabled: assign_perm(perm, target_user, project) else: remove_perm(perm, target_user, project) return JsonResponse({ 'n_updated_permissions': len(permissions), }) else: raise ValueError(f'Unsupported HTTP method: {request.method}')
def many_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one list of skeletons and a list of other skeletons. """ skids1 = get_request_list(request.POST, 'skids1', map_fn=int) if not skids1: raise ValueError("No skeleton IDs for first list of 'many' provided") skids2 = get_request_list(request.POST, 'skids2', map_fn=int) if not skids2: raise ValueError("No skeleton IDs for second list 'many' provided") relation_name = request.POST.get( 'relation' ) # expecting presynaptic_to, postsynaptic_to, or gapjunction_with rows = _many_to_many_synapses(skids1, skids2, relation_name, project_id) return HttpResponse(json.dumps(rows))
def post(self, *args, **kwargs): """Check for superuser permissions in decorator.""" project_ids = get_request_list(self.request.POST, 'ids', map_fn=int) if not project_ids: raise ValueError("No project IDs specified") delete_projects(project_ids) messages.add_message(self.request, messages.INFO, f"{len(project_ids)} project(s) plus all their linked data have been deleted") return HttpResponseRedirect(reverse('admin:index') + 'catmaid/project')
def annotations_for_entities(request, project_id=None): """Query annotations linked to a list of objects. These objects can for instance be neurons, annotations or stack groups. From a database perspective, these objects are class instances. Returned is an object with the fields "entities" and "annotations". The former is an object mapping an entity ID to a list of annotations. Each annotation is represented by an object containing its "id" and "uid", the user who annotated it. The latter maps annotation IDs to annotation names. For instance:: { "entities": { "42": [{id: 1, uid: 12}, {id: 3, uid: 14}] }, "annotations": { 12: "example1", 14: "example2" } } --- parameters: - name: object_ids description: A list of object IDs for which annotations should be returned. paramType: form type: array allowMultiple: true items: type: integer description: A skeleton ID """ # Get 'annotated_with' relation ID object_ids = tuple( get_request_list(request.POST, 'object_ids', [], map_fn=int)) cursor = connection.cursor() cursor.execute(""" SELECT id FROM relation WHERE project_id=%s AND relation_name='annotated_with'""" % int(project_id)) annotated_with_id = cursor.fetchone()[0] # Select pairs of skeleton_id vs annotation name cursor.execute(''' SELECT entity_annotation.class_instance_a, annotation.id, annotation.name, entity_annotation.user_id FROM class_instance_class_instance entity_annotation, class_instance annotation WHERE entity_annotation.class_instance_a IN (%s) AND entity_annotation.relation_id = %s AND entity_annotation.class_instance_b = annotation.id ''' % (",".join(map(str, object_ids)), annotated_with_id)) # Group by entity ID m = defaultdict(list) a = dict() for eid, aid, name, uid in cursor.fetchall(): m[eid].append({'id': aid, 'uid': uid}) a[aid] = name return JsonResponse({ 'entities': m, 'annotations': a }, json_dumps_params={'separators': (',', ':')})
def compact_detail_list(request, project_id=None): """Retrieve node information in a compact form. A list of elements of the following form is returned: [ID, parent ID, x, y, z, confidence, user_id, radius, skeleton_id, user_id]. """ treenode_ids = get_request_list(request.POST, 'treenode_ids', None, int) if not treenode_ids: raise ValueError("No treenode IDs provided") info = _compact_detail_list(int(project_id), treenode_ids) return JsonResponse(info, safe=False)
def annotations_for_entities(request, project_id=None): """Query annotations linked to a list of objects. These objects can for instance be neurons, annotations or stack groups. From a database perspective, these objects are class instances. Returned is an object with the fields "entities" and "annotations". The former is an object mapping an entity ID to a list of annotations. Each annotation is represented by an object containing its "id" and "uid", the user who annotated it. The latter maps annotation IDs to annotation names. For instance:: { "entities": { "42": [{id: 1, uid: 12}, {id: 3, uid: 14}] }, "annotations": { 12: "example1", 14: "example2" } } --- parameters: - name: object_ids description: A list of object IDs for which annotations should be returned. paramType: form type: array allowMultiple: true items: type: integer description: A skeleton ID """ # Get 'annotated_with' relation ID object_ids = tuple(get_request_list(request.POST, 'object_ids', [], map_fn=int)) cursor = connection.cursor() cursor.execute(""" SELECT id FROM relation WHERE project_id=%s AND relation_name='annotated_with'""" % int(project_id)) annotated_with_id = cursor.fetchone()[0] # Select pairs of skeleton_id vs annotation name cursor.execute(''' SELECT entity_annotation.class_instance_a, annotation.id, annotation.name, entity_annotation.user_id FROM class_instance_class_instance entity_annotation, class_instance annotation WHERE entity_annotation.class_instance_a IN (%s) AND entity_annotation.relation_id = %s AND entity_annotation.class_instance_b = annotation.id ''' % (",".join(map(str, object_ids)), annotated_with_id)) # Group by entity ID m = defaultdict(list) a = dict() for eid, aid, name, uid in cursor.fetchall(): m[eid].append({'id': aid, 'uid': uid}) a[aid] = name return JsonResponse({ 'entities': m, 'annotations': a }, json_dumps_params={'separators': (',', ':')})
def compact_detail_list(request, project_id=None): """ Retrieve node information in a compact form. A list of elements of the following form is returned: [ID, parent ID, x, y, z, confidence, radius, skeleton_id, edition_time, user_id] The returned edition time is an epoch number. --- parameters: - name: project_id description: Project to work in required: true - name: treenode_ids description: A list of treeonde IDs to return information on required: false - name: label_ids description: | A list of label IDs that must be linked to result treenodes. Alternative to explicit treenode IDs and label names. required: false - name: label_names description: | A list of label names that must be linked to result treenodes. Alternative to explicit treenode IDs and label IDs required: false - name: skeleton_ids description: | A list of skeleton IDs that result skeletons have to be part of. required: false """ treenode_ids = get_request_list(request.POST, 'treenode_ids', None, int) label_ids = get_request_list(request.POST, 'label_ids', None, int) label_names = get_request_list(request.POST, 'label_names') skeleton_ids = get_request_list(request.POST, 'skeleton_ids', None, int) if not any((treenode_ids, label_ids, label_names, skeleton_ids)): raise ValueError("No treenode IDs, label IDs, label names or skeleton IDs provided") info = _compact_detail_list(int(project_id), treenode_ids, label_ids, label_names, skeleton_ids) return JsonResponse(info, safe=False)
def annotations_for_skeletons(request, project_id=None): """Get annotations and who used them for a set of skeletons. This method focuses only on annotations linked to skeletons and is likely to be faster than the general query. Returns an object with two fields: "annotations", which is itself an object with annotation IDs as fields, giving access to the corresponding annotation names. And the field "skeletons" is also an object, mapping skeleton IDs to lists of annotation-annotator ID pairs. Also, as JSON separator a colon is used instead of a comma. --- parameters: - name: skeleton_ids description: A list of skeleton IDs which are annotated by the resulting annotations. paramType: form type: array items: type: integer description: A skeleton ID """ skids = tuple( get_request_list(request.POST, 'skeleton_ids', [], map_fn=int)) cursor = connection.cursor() cursor.execute( "SELECT id FROM relation WHERE project_id=%s AND relation_name='annotated_with'" % int(project_id)) annotated_with_id = cursor.fetchone()[0] # Select pairs of skeleton_id vs annotation name cursor.execute(''' SELECT skeleton_neuron.class_instance_a, annotation.id, annotation.name, neuron_annotation.user_id FROM class_instance_class_instance skeleton_neuron, class_instance_class_instance neuron_annotation, class_instance annotation WHERE skeleton_neuron.class_instance_a IN (%s) AND skeleton_neuron.class_instance_b = neuron_annotation.class_instance_a AND neuron_annotation.relation_id = %s AND neuron_annotation.class_instance_b = annotation.id ''' % (",".join(map(str, skids)), annotated_with_id)) # Group by skeleton ID m = defaultdict(list) a = dict() for skid, aid, name, uid in cursor.fetchall(): m[skid].append({'id': aid, 'uid': uid}) a[aid] = name return JsonResponse({ 'skeletons': m, 'annotations': a }, json_dumps_params={'separators': (',', ':')})
def graphedge_list(request, project_id=None): """ Assumes that first element of skeletonlist is pre, and second is post """ skeletonlist = get_request_list(request.POST, 'skeletonlist[]') skeletonlist = map(int, skeletonlist) p = get_object_or_404(Project, pk=project_id) edge = {} connectordata = {} qs_tc = TreenodeConnector.objects.filter( project=p, skeleton__in=skeletonlist).select_related('relation__relation_name', 'connector__user', 'connector') for q in qs_tc: # Only look at synapse connectors if q.relation.relation_name not in ('presynaptic_to', 'postsynaptic_to'): continue if not q.connector_id in edge: # has to be a list, not a set, because we need matching treenode id edge[q.connector_id] = { 'pre': [], 'post': [], 'pretreenode': [], 'posttreenode': [] } connectordata[q.connector_id] = { 'connector_id': q.connector_id, 'x': q.connector.location_x, 'y': q.connector.location_y, 'z': q.connector.location_z, 'user': q.connector.user.username } if q.relation.relation_name == 'presynaptic_to': edge[q.connector_id]['pre'].append(q.skeleton_id) edge[q.connector_id]['pretreenode'].append(q.treenode_id) elif q.relation.relation_name == 'postsynaptic_to': edge[q.connector_id]['post'].append(q.skeleton_id) edge[q.connector_id]['posttreenode'].append(q.treenode_id) result = [] for k, v in edge.items(): if skeletonlist[0] in v['pre'] and skeletonlist[1] in v['post']: connectordata[k]['pretreenode'] = v['pretreenode'][v['pre'].index( skeletonlist[0])] connectordata[k]['posttreenode'] = v['posttreenode'][ v['post'].index(skeletonlist[1])] result.append(connectordata[k]) return HttpResponse(json.dumps(result), content_type='application/json')
def get_locations(request, project_id=None): """Get locations for a particular set of nodes in a project. A list of lists is returned. Each inner list represents one location and hast the following format: [id, x, y, z]. --- parameters: - name: node_ids description: A list of node IDs to get the location for required: true type: array items: type: number format: integer required: true paramType: form models: location_element: id: location_element properties: - name: id description: ID of the node. type: integer required: true - name: x description: X coordinate of the node. required: true type: number format: double paramType: form - name: y description: Y coordinate of the node. required: true type: number format: double paramType: form - name: z description: Z coordinate of the node. required: true type: number format: double paramType: form type: - type: array items: $ref: location_element required: true """ node_ids = get_request_list(request.POST, 'node_ids', map_fn=int) locations = _fetch_locations(project_id, node_ids) return JsonResponse(locations, safe=False)
def one_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one skeleton and a list of other skeletons. """ if 'skid' not in request.POST: raise ValueError("No skeleton ID for 'one' provided") skid = int(request.POST.get('skid')) skids = get_request_list(request.POST, 'skids', map_fn=int) if not skids: raise ValueError("No skeleton IDs for 'many' provided") relation_name = request.POST.get('relation') rows = _many_to_many_synapses([skid], skids, relation_name, project_id) return JsonResponse(rows, safe=False)
def one_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one skeleton and a list of other skeletons. """ if 'skid' not in request.POST: raise ValueError("No skeleton ID for 'one' provided") skid = int(request.POST.get('skid')) skids = get_request_list(request.POST, 'skids', map_fn=int) if not skids: raise ValueError("No skeleton IDs for 'many' provided") relation_name = request.POST.get('relation') # expecting presynaptic_to, postsynaptic_to, or gapjunction_with rows = _many_to_many_synapses([skid], skids, relation_name, project_id) return HttpResponse(json.dumps(rows))
def get_neuron_ids_from_models(request, project_id=None): """Retrieve neuron IDs modeled by particular entities, eg skeletons. From a list of source entities (class instances), the IDs of all modeled neurons are returned. There are currently only skeletons that model neurons. --- parameters: - name: model_ids[] description: IDs of models to find neurons for (e.g. skeleton IDs) required: true type: array items: type: integer paramType: form type: '{model_id}': description: ID of modeled neuron type: integer required: true """ model_ids = get_request_list(request.POST, "model_ids", map_fn=int) if not model_ids: raise ValueError("No valid model IDs provided") cursor = connection.cursor() model_template = ",".join(("(%s)",) * len(model_ids)) or "()" params = [project_id] + model_ids + [project_id] cursor.execute( """ WITH allowed_relation AS ( SELECT id FROM relation WHERE project_id = %s AND relation_name = 'model_of' LIMIT 1 ) SELECT cici.class_instance_a, cici.class_instance_b FROM allowed_relation ar, class_instance_class_instance cici JOIN (VALUES {}) model(id) ON cici.class_instance_a = model.id WHERE cici.project_id = %s AND cici.relation_id = ar.id """.format( model_template ), params, ) models = {} for row in cursor.fetchall(): models[row[0]] = row[1] return JsonResponse(models)
def skeleton_graph(request, project_id=None): project_id = int(project_id) skeleton_ids = set(int(v) for k,v in request.POST.items() if k.startswith('skeleton_list[')) confidence_threshold = int(request.POST.get('confidence_threshold', 0)) bandwidth = float(request.POST.get('bandwidth', 0)) # in nanometers cable_spread = float(request.POST.get('cable_spread', 2500)) # in nanometers path_confluence = int(request.POST.get('path_confluence', 10)) # a count compute_risk = 1 == int(request.POST.get('risk', 0)) expand = set(int(v) for k,v in request.POST.items() if k.startswith('expand[')) link_types = get_request_list(request.POST, 'link_types', None) by_link_type = bool(link_types) if not by_link_type: link_types = ['synaptic-connector'] for link_type in link_types: pair = KNOWN_LINK_PAIRS.get(link_type) if not pair: raise ValueError("Unknown link type: " + link_type) source_rel = pair['source'] target_rel = pair['target'] circuit = _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth, expand, compute_risk, cable_spread, path_confluence, source_rel, target_rel) package = {'nodes': [{'data': props} for props in circuit.node.values()], 'edges': []} edges = package['edges'] for g1, g2, props in circuit.edges_iter(data=True): id1 = circuit.node[g1]['id'] id2 = circuit.node[g2]['id'] data = {'id': '%s_%s' % (id1, id2), 'source': id1, 'target': id2, 'weight': props['c'], 'label': str(props['c']) if props['directed'] else None, 'directed': props['directed'], 'arrow': props['arrow']} if compute_risk: data['risk'] = props.get('risk') edges.append({'data': data}) if by_link_type: if not result: result = {} result[link_type] = package else: result = package return JsonResponse(result, safe=False)
def one_to_many_synapses(request:HttpRequest, project_id=None) -> JsonResponse: """ Return the list of synapses of a specific kind between one skeleton and a list of other skeletons. """ if 'skid' not in request.POST: raise ValueError("No skeleton ID for 'one' provided") skid = int(request.POST.get('skid')) skids = get_request_list(request.POST, 'skids', map_fn=int) if not skids: raise ValueError("No skeleton IDs for 'many' provided") relation_name = request.POST.get('relation') rows = _many_to_many_synapses([skid], skids, relation_name, project_id) return JsonResponse(rows, safe=False)
def remove_annotations(request, project_id=None): """ Removes an annotation from one or more entities. """ annotation_ids = get_request_list(request.POST, 'annotation_ids', [], map_fn=int) entity_ids = get_request_list(request.POST, 'entity_ids', [], map_fn=int) if not annotation_ids: raise ValueError("No annotation IDs provided") if not entity_ids: raise ValueError("No entity IDs provided") # Remove individual annotations deleted_annotations = {} deleted_links = [] num_left_annotations = {} for annotation_id in annotation_ids: cicis_to_delete, missed_cicis, deleted, num_left = _remove_annotation( request.user, project_id, entity_ids, annotation_id) # Keep track of results num_left_annotations[str(annotation_id)] = num_left targetIds = [] for cici in cicis_to_delete: deleted_links.append(cici.id) # The target is class_instance_a, because we deal with the # "annotated_with" relation. targetIds.append(cici.class_instance_a_id) if targetIds: deleted_annotations[annotation_id] = {'targetIds': targetIds} return JsonResponse({ 'deleted_annotations': deleted_annotations, 'deleted_links': deleted_links, 'left_uses': num_left_annotations })
def insert_treenode(request, project_id=None): """ Create a new treenode between two existing nodes. Its creator and creation_date information will be set to information of child node. No node will be created, if the node on the edge between the given child and parent node. """ # Use creation time, if part of parameter set params = {} float_values = { 'x': 0, 'y': 0, 'z': 0, 'radius': 0 } int_values = { 'confidence': 0, 'parent_id': -1, 'child_id': -1 } for p in float_values.keys(): params[p] = float(request.POST.get(p, float_values[p])) for p in int_values.keys(): params[p] = int(request.POST.get(p, int_values[p])) # If siblings should be taken over, all children of the parent node will be # come children of the inserted node. This requires extra state # information: the child state for the paren. takeover_child_ids = get_request_list(request.POST, 'takeover_child_ids', None, lambda x: int(x)) # Get optional initial links to connectors, expect each entry to be a list # of connector ID and relation ID. try: links = get_request_list(request.POST, 'links', [], lambda x: int(x)) except Exception, e: raise ValueError("Couldn't parse list parameter: {}".format(e))
def remove_annotations(request, project_id=None): """ Removes an annotation from one or more entities. """ annotation_ids = get_request_list(request.POST, 'annotation_ids', [], map_fn=int) entity_ids = get_request_list(request.POST, 'entity_ids', [], map_fn=int) if not annotation_ids: raise ValueError("No annotation IDs provided") if not entity_ids: raise ValueError("No entity IDs provided") # Remove individual annotations deleted_annotations = {} deleted_links = [] num_left_annotations = {} for annotation_id in annotation_ids: cicis_to_delete, missed_cicis, deleted, num_left = _remove_annotation( request.user, project_id, entity_ids, annotation_id) # Keep track of results num_left_annotations[str(annotation_id)] = num_left targetIds = [] for cici in cicis_to_delete: deleted_links.append(cici.id) # The target is class_instance_a, because we deal with the # "annotated_with" relation. targetIds.append(cici.class_instance_a_id) if targetIds: deleted_annotations[annotation_id] = { 'targetIds': targetIds } return JsonResponse({ 'deleted_annotations': deleted_annotations, 'deleted_links': deleted_links, 'left_uses': num_left_annotations })
def post(self, request, project_id): """List all available point clouds or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point clouds type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_images description: Wheter linked images should returned as well. type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: sample_ratio description: Number in [0,1] to optionally sample point cloud type: number paramType: form required: false - name: pointcloud_ids description: A list of point cloud IDs to which the query is constrained. type: array paramType: path required: false """ with_images = get_request_bool(request.POST, 'with_images', False) with_points = get_request_bool(request.POST, 'with_points', False) sample_ratio = float(request.POST.get('sample_ratio', '1.0')) simple = get_request_bool(request.POST, 'simple', False) pointcloud_ids = get_request_list(request.POST, 'pointcloud_ids', None, map_fn=int) pointclouds = list_pointclouds(project_id, request.user.id, simple, with_images, with_points, sample_ratio, pointcloud_ids) return JsonResponse(pointclouds, safe=False)
def annotations_for_skeletons(request, project_id=None): """Get annotations and who used them for a set of skeletons. This method focuses only on annotations linked to skeletons and is likely to be faster than the general query. Returns an object with two fields: "annotations", which is itself an object with annotation IDs as fields, giving access to the corresponding annotation names. And the field "skeletons" is also an object, mapping skeleton IDs to lists of annotation-annotator ID pairs. Also, as JSON separator a colon is used instead of a comma. --- parameters: - name: skeleton_ids description: A list of skeleton IDs which are annotated by the resulting annotations. paramType: form type: array items: type: integer description: A skeleton ID """ skids = tuple(get_request_list(request.POST, 'skeleton_ids', [], map_fn=int)) cursor = connection.cursor() cursor.execute("SELECT id FROM relation WHERE project_id=%s AND relation_name='annotated_with'" % int(project_id)) annotated_with_id = cursor.fetchone()[0] # Select pairs of skeleton_id vs annotation name cursor.execute(''' SELECT skeleton_neuron.class_instance_a, annotation.id, annotation.name, neuron_annotation.user_id FROM class_instance_class_instance skeleton_neuron, class_instance_class_instance neuron_annotation, class_instance annotation WHERE skeleton_neuron.class_instance_a IN (%s) AND skeleton_neuron.class_instance_b = neuron_annotation.class_instance_a AND neuron_annotation.relation_id = %s AND neuron_annotation.class_instance_b = annotation.id ''' % (",".join(map(str, skids)), annotated_with_id)) # Group by skeleton ID m = defaultdict(list) a = dict() for skid, aid, name, uid in cursor.fetchall(): m[skid].append({'id': aid, 'uid': uid}) a[aid] = name return JsonResponse({ 'skeletons': m, 'annotations': a }, json_dumps_params={'separators': (',', ':')})
def get_skeleton_innervations(request, project_id): """Test environment only contains two skeletons - based on that, sql query always returns list of all SKIDs but all data (about both skeletons) is contained in the first SKID in the list - if this changes, write an else statement for: len(cleanResults) >1. --- parameters: - name: project_id required: true description: The project to operate in type: integer paramType: path - name: skeleton_ids description: Constrain results to these skeletons required: false type: array items: type: integer paramType: form - name: annotation description: An annotation potential target volumes need to have type: string required: false - name: min_nodes description: A minimum number of nodes result skeleton need to have. required: false type: boolean - name: min_cable description: A minimum number of cable length esult skeleton need to have. required: false type: boolean """ skeleton_ids = get_request_list(request.POST, 'skeleton_ids', map_fn=int) if not skeleton_ids: raise ValueError('Need skeleton IDs') volume_annotation = request.POST.get('annotation') min_nodes = request.POST.get('min_nodes') if min_nodes: min_nodes = int(min_nodes) min_cable = request.POST.get('min_cable') if min_cable: min_cable = int(min_cable) volume_intersections = _get_skeleton_innervations(project_id, skeleton_ids, volume_annotation, min_nodes, min_cable) return JsonResponse(volume_intersections, safe=False)
def one_to_many_synapses(request, project_id=None): """ Return the list of synapses of a specific kind between one skeleton and a list of other skeletons. """ if 'skid' not in request.POST: raise ValueError("No skeleton ID for 'one' provided") skid = int(request.POST.get('skid')) skids = get_request_list(request.POST, 'skids', map_fn=int) if not skids: raise ValueError("No skeleton IDs for 'many' provided") relation_name = request.POST.get( 'relation' ) # expecting presynaptic_to, postsynaptic_to, or gapjunction_with rows = _many_to_many_synapses([skid], skids, relation_name, project_id) return HttpResponse(json.dumps(rows))
def add_all_intervals(request, project_id, domain_id): """Create all intervals in a particular domain. --- parameters: - name: domain_id description: Domain to add intervals in type: integer: required: true - name: intervals description: A list of two-element lists, with start and end node each type: array: items: type: string required: true """ domain_id = int(domain_id) domain = SamplerDomain.objects.get(id=domain_id) state = SamplerIntervalState.objects.get(name='untouched') intervals = get_request_list(request.POST, 'intervals', [], map_fn=lambda x: x) result_intervals = [] for i in intervals: start_node = int(i[0]) end_node = int(i[1]) i = SamplerInterval.objects.create( domain=domain, interval_state=state, start_node_id=start_node, end_node_id=end_node, user=request.user, project_id=project_id) result_intervals.append({ "id": i.id, "interval_state_id": i.interval_state_id, "start_node_id": i.start_node_id, "end_node_id": i.end_node_id, "user_id": i.user_id, "project_id": i.project_id }) return JsonResponse(result_intervals, safe=False)
def connector_associated_edgetimes(request, project_id=None): """ See _connector_associated_edgetimes """ connector_ids = get_request_list(request.POST, 'connector_ids', map_fn=int) def default(obj): """Default JSON serializer.""" import calendar, datetime if isinstance(obj, datetime.datetime): if obj.utcoffset() is not None: obj = obj - obj.utcoffset() millis = int( calendar.timegm(obj.timetuple()) * 1000 + obj.microsecond / 1000 ) return millis return HttpResponse(json.dumps(_connector_associated_edgetimes(connector_ids, project_id), default=default))
def graphedge_list(request, project_id=None): """ Assumes that first element of skeletonlist is pre, and second is post """ skeletonlist = get_request_list(request.POST, 'skeletonlist[]') skeletonlist = map(int, skeletonlist) p = get_object_or_404(Project, pk=project_id) edge = {} connectordata = {} qs_tc = TreenodeConnector.objects.filter( project=p, skeleton__in=skeletonlist ).select_related('relation__relation_name', 'connector__user', 'connector') for q in qs_tc: # Only look at synapse connectors if q.relation.relation_name not in ('presynaptic_to', 'postsynaptic_to'): continue if not q.connector_id in edge: # has to be a list, not a set, because we need matching treenode id edge[ q.connector_id ] = {'pre': [], 'post': [], 'pretreenode': [], 'posttreenode': []} connectordata[ q.connector_id ] = { 'connector_id': q.connector_id, 'x': q.connector.location_x, 'y': q.connector.location_y, 'z': q.connector.location_z, 'user': q.connector.user.username } if q.relation.relation_name == 'presynaptic_to': edge[ q.connector_id ]['pre'].append( q.skeleton_id ) edge[ q.connector_id ]['pretreenode'].append( q.treenode_id ) elif q.relation.relation_name == 'postsynaptic_to': edge[ q.connector_id ]['post'].append( q.skeleton_id ) edge[ q.connector_id ]['posttreenode'].append( q.treenode_id ) result = [] for k,v in edge.items(): if skeletonlist[0] in v['pre'] and skeletonlist[1] in v['post']: connectordata[k]['pretreenode'] = v['pretreenode'][ v['pre'].index( skeletonlist[0] ) ] connectordata[k]['posttreenode'] = v['posttreenode'][ v['post'].index( skeletonlist[1] ) ] result.append(connectordata[k]) return HttpResponse(json.dumps( result ), content_type='application/json')
def create_connector(request, project_id=None): query_parameters = {} default_values = {'x': 0, 'y': 0, 'z': 0, 'confidence': 5} for p in default_values.keys(): query_parameters[p] = request.POST.get(p, default_values[p]) project_id = int(project_id) parsed_confidence = int(query_parameters['confidence']) if parsed_confidence < 1 or parsed_confidence > 5: return HttpResponse(json.dumps({'error': 'Confidence not in range 1-5 inclusive.'})) cursor = connection.cursor() # Get optional initial links to connectors, expect each entry to be a list # of connector ID, relation ID and confidence. links = get_request_list(request.POST, 'links', [], map_fn=int) new_connector = Connector( user=request.user, editor=request.user, project=Project.objects.get(id=project_id), location_x=float(query_parameters['x']), location_y=float(query_parameters['y']), location_z=float(query_parameters['z']), confidence=parsed_confidence) new_connector.save() # Create all initial links if links: created_links = create_treenode_links(project_id, request.user.id, new_connector.id, links, cursor) else: created_links = [] return JsonResponse({ 'connector_id': new_connector.id, 'connector_edition_time': new_connector.edition_time, 'created_links': created_links })
def user_info(request, project_id=None): """Return information on a treenode or connector. This function is called pretty often (with every node activation) and should therefore be as fast as possible. """ node_ids = get_request_list(request.POST, 'node_ids', map_fn=int) if not node_ids: raise ValueError('Need at least one node ID') node_template = ','.join('(%s)' for n in node_ids) cursor = connection.cursor() cursor.execute(''' SELECT n.id, n.user_id, n.editor_id, n.creation_time, n.edition_time, array_agg(r.reviewer_id), array_agg(r.review_time) FROM location n JOIN (VALUES {}) req_node(id) ON n.id = req_node.id LEFT OUTER JOIN review r ON r.treenode_id = n.id GROUP BY n.id '''.format(node_template), node_ids) # Build result result = {} for row in cursor.fetchall(): result[row[0]] = { 'user': row[1], 'editor': row[2], 'creation_time': str(row[3].isoformat()), 'edition_time': str(row[4].isoformat()), 'reviewers': [r for r in row[5] if r], 'review_times': [str(rt.isoformat()) for rt in row[6] if rt] } return JsonResponse(result)
def update_confidence(request, project_id=None, treenode_id=None): """Update confidence of edge between a node to either its parent or its connectors. The connection between a node and its parent or the connectors it is linked to can be rated with a confidence value in the range 1-5. If connector links should be updated, one can limit the affected connections to a specific connector. Returned is an object, mapping updated partners to their old confidences. --- parameters: - name: new_confidence description: New confidence, value in range 1-5 type: integer required: true - name: to_connector description: Whether all linked connectors instead of parent should be updated type: boolean required: false - name: partner_ids description: Limit update to a set of connectors if to_connector is true type: array items: integer required: false - name: partner_confidences description: Set different confidences to connectors in <partner_ids> type: array items: integer required: false type: message: type: string required: true updated_partners: type: object required: true """ tnid = int(treenode_id) can_edit_treenode_or_fail(request.user, project_id, tnid) cursor = connection.cursor() state.validate_state(tnid, request.POST.get('state'), node=True, lock=True, cursor=cursor) to_connector = get_request_bool(request.POST, 'to_connector', False) partner_ids = get_request_list(request.POST, 'partner_ids', None, int) partner_confidences = get_request_list(request.POST, 'partner_confidences', None, int) new_confidence = int(request.POST.get('new_confidence', 0)) # If partner confidences are specified, make sure there are exactly as many # as there are partners. Otherwise validate passed in confidence if partner_ids and partner_confidences: if len(partner_confidences) != len(partner_ids): raise ValueError("There have to be as many partner confidences as" "there are partner IDs") else: if new_confidence < 1 or new_confidence > 5: raise ValueError('Confidence not in range 1-5 inclusive.') if partner_ids: # Prepare new confidences for connector query partner_confidences = (new_confidence,) * len(partner_ids) if to_connector: if partner_ids: partner_template = ",".join(("(%s,%s)",) * len(partner_ids)) partner_data = [p for v in zip(partner_ids, partner_confidences) for p in v] cursor.execute(''' UPDATE treenode_connector tc SET confidence = target.new_confidence FROM (SELECT x.id, x.confidence AS old_confidence, new_values.confidence AS new_confidence FROM treenode_connector x JOIN (VALUES {}) new_values(cid, confidence) ON x.connector_id = new_values.cid WHERE x.treenode_id = %s) target WHERE tc.id = target.id RETURNING tc.connector_id, tc.edition_time, target.old_confidence '''.format(partner_template), partner_data + [tnid]) else: cursor.execute(''' UPDATE treenode_connector tc SET confidence = %s FROM (SELECT x.id, x.confidence AS old_confidence FROM treenode_connector x WHERE treenode_id = %s) target WHERE tc.id = target.id RETURNING tc.connector_id, tc.edition_time, target.old_confidence ''', (new_confidence, tnid)) else: cursor.execute(''' UPDATE treenode t SET confidence = %s, editor_id = %s FROM (SELECT x.id, x.confidence AS old_confidence FROM treenode x WHERE id = %s) target WHERE t.id = target.id RETURNING t.parent_id, t.edition_time, target.old_confidence ''', (new_confidence, request.user.id, tnid)) updated_partners = cursor.fetchall() if len(updated_partners) > 0: location = Location.objects.filter(id=tnid).values_list( 'location_x', 'location_y', 'location_z')[0] insert_into_log(project_id, request.user.id, "change_confidence", location, "Changed to %s" % new_confidence) return JsonResponse({ 'message': 'success', 'updated_partners': { r[0]: { 'edition_time': r[1], 'old_confidence': r[2] } for r in updated_partners } }) # Else, signal error if to_connector: raise ValueError('Failed to update confidence between treenode %s and ' 'connector.' % tnid) else: raise ValueError('Failed to update confidence at treenode %s.' % tnid)
def create_treenode(request, project_id=None): """ Add a new treenode to the database ---------------------------------- 1. Add new treenode for a given skeleton id. Parent should not be empty. return: new treenode id If the parent's skeleton has a single node and belongs to the 'Isolated synaptic terminals' group, then reassign ownership of the skeleton and the neuron to the user. The treenode remains property of the original user who created it. 2. Add new treenode (root) and create a new skeleton (maybe for a given neuron) return: new treenode id and skeleton id. If a neuron id is given, use that one to create the skeleton as a model of it. """ params = {} float_values = { 'x': 0, 'y': 0, 'z': 0, 'radius': 0} int_values = { 'confidence': 0, 'useneuron': -1, 'parent_id': -1} string_values = {} for p in float_values.keys(): params[p] = float(request.POST.get(p, float_values[p])) for p in int_values.keys(): params[p] = int(request.POST.get(p, int_values[p])) for p in string_values.keys(): params[p] = request.POST.get(p, string_values[p]) # Get optional initial links to connectors, expect each entry to be a list # of connector ID, relation ID and confidence. links = get_request_list(request.POST, 'links', [], map_fn=int) # Make sure the back-end is in the expected state if the node should have a # parent and will therefore become part of another skeleton. parent_id = int(params['parent_id']) has_parent = parent_id and parent_id != -1 if has_parent: state.validate_state(parent_id, request.POST.get('state'), parent_edittime=has_parent, lock=True) new_treenode = _create_treenode(project_id, request.user, request.user, params['x'], params['y'], params['z'], params['radius'], params['confidence'], params['useneuron'], params['parent_id'], neuron_name=request.POST.get('neuron_name', None)) # Create all initial links if links: created_links = create_connector_link(project_id, request.user.id, new_treenode.treenode_id, new_treenode.skeleton_id, links) else: created_links = [] return JsonResponse({ 'treenode_id': new_treenode.treenode_id, 'skeleton_id': new_treenode.skeleton_id, 'edition_time': new_treenode.edition_time, 'parent_edition_time': new_treenode.parent_edition_time, 'created_links': created_links })
def node_list_tuples(request, project_id=None, provider=None): '''Retrieve all nodes intersecting a bounding box The intersection bounding box is defined in terms of its minimum and maximum project space coordinates. The number of returned nodes can be limited to constrain query execution time. Optionally, lists of treenodes and connector IDs can be provided to make sure they are included in the result set, regardless of intersection. Returned is an array with four entries: [[treenodes], [connectors], {labels}, node_limit_reached, {relation_map}] The list of treenodes has elements of this form: [id, parent_id, location_x, location_y, location_z, confidence, radius, skeleton_id, edition_time, user_id] The list connectors has elements of this form: [id, location_x, location_y, location_z, confidence, edition_time, user_id, [partners]] The partners arrary represents linked partner nodes, each one represented like this: [treenode_id, relation_id, link_confidence, link_edition_time, link_id] If labels are returned, they are represented as an object of the following form, with the labels just being simple strings: {treenode_id: [labels]} The fourth top level entry, node_limit_reached, is a boolean that represents if there are more nodes available than the ones returned. With the last top level element returned the present connector linked relations are mapped to their textural representations: {relation_id: relation_name} --- parameters: - name: treenode_ids description: | Whether linked connectors should be returned. required: false type: array items: type: integer paramType: form - name: connector_ids description: | Whether tags should be returned. required: false type: array items: type: integer paramType: form - name: limit description: | Limit the number of returned nodes. required: false type: integer defaultValue: 3500 paramType: form - name: left description: | Minimum world space X coordinate required: true type: float paramType: form - name: top description: | Minimum world space Y coordinate required: true type: float paramType: form - name: z1 description: | Minimum world space Z coordinate required: true type: float paramType: form - name: right description: | Maximum world space X coordinate required: true type: float paramType: form - name: bottom description: | Maximum world space Y coordinate required: true type: float paramType: form - name: z2 description: | Maximum world space Z coordinate required: true type: float paramType: form type: - type: array items: type: string required: true ''' project_id = int(project_id) # sanitize params = {} treenode_ids = get_request_list(request.POST, 'treenode_ids', map_fn=int) connector_ids = get_request_list(request.POST, 'connector_ids', map_fn=int) for p in ('top', 'left', 'bottom', 'right', 'z1', 'z2'): params[p] = float(request.POST.get(p, 0)) # Limit the number of retrieved treenodes within the section params['limit'] = settings.NODE_LIST_MAXIMUM_COUNT params['project_id'] = project_id include_labels = (request.POST.get('labels', None) == 'true') provider = get_treenodes_postgis return node_list_tuples_query(params, project_id, treenode_ids, connector_ids, include_labels, provider)
def skeleton_graph(request, project_id=None): """Get a synaptic graph between skeletons compartmentalized by confidence. Given a set of skeletons, retrieve presynaptic-to-postsynaptic edges between them, annotated with count. If a confidence threshold is supplied, compartmentalize the skeletons at edges in the arbor below that threshold and report connectivity based on these compartments. When skeletons are split into compartments, nodes in the graph take an string ID like ``{skeleton_id}_{compartment #}``. --- parameters: - name: skeleton_ids[] description: IDs of the skeletons to graph required: true type: array items: type: integer paramType: form - name: confidence_threshold description: Confidence value below which to segregate compartments type: integer paramType: form - name: bandwidth description: Bandwidth in nanometers type: number - name: cable_spread description: Cable spread in nanometers type: number - name: expand[] description: IDs of the skeletons to expand type: array items: type: integer - name: link_types[] description: IDs of link types to respect type: array items: type: string models: skeleton_graph_edge: id: skeleton_graph_edge properties: - description: ID of the presynaptic skeleton or compartment type: integer|string required: true - description: ID of the postsynaptic skeleton or compartment type: integer|string required: true - description: number of synapses constituting this edge $ref: skeleton_graph_edge_count required: true skeleton_graph_edge_count: id: skeleton_graph_edge_count properties: - description: Number of synapses with confidence 1 type: integer required: true - description: Number of synapses with confidence 2 type: integer required: true - description: Number of synapses with confidence 3 type: integer required: true - description: Number of synapses with confidence 4 type: integer required: true - description: Number of synapses with confidence 5 type: integer required: true skeleton_graph_intraedge: id: skeleton_graph_intraedge properties: - description: ID of the presynaptic skeleton or compartment type: integer|string required: true - description: ID of the postsynaptic skeleton or compartment type: integer|string required: true type: edges: type: array items: $ref: skeleton_graph_edge required: true nodes: type: array items: type: integer|string required: false intraedges: type: array items: $ref: skeleton_graph_intraedge required: false branch_nodes: type: array items: type: integer|string required: false """ compute_risk = 1 == int(request.POST.get('risk', 0)) if compute_risk: # TODO port the last bit: computing the synapse risk from graph import skeleton_graph as slow_graph return slow_graph(request, project_id) project_id = int(project_id) skeleton_ids = set(int(v) for k,v in request.POST.items() if k.startswith('skeleton_ids[')) confidence_threshold = min(int(request.POST.get('confidence_threshold', 0)), 5) bandwidth = float(request.POST.get('bandwidth', 0)) # in nanometers cable_spread = float(request.POST.get('cable_spread', 2500)) # in nanometers path_confluence = int(request.POST.get('path_confluence', 10)) # a count expand = set(int(v) for k,v in request.POST.items() if k.startswith('expand[')) with_overall_counts = get_request_bool(request.POST, 'with_overall_counts', False) expand = set(int(v) for k,v in request.POST.items() if k.startswith('expand[')) link_types = get_request_list(request.POST, 'link_types', None) graph = _skeleton_graph(project_id, skeleton_ids, confidence_threshold, bandwidth, expand, compute_risk, cable_spread, path_confluence, with_overall_counts, link_types=link_types) if not graph: raise ValueError("Could not compute graph") return JsonResponse(graph)
def insert_treenode(request, project_id=None): """ Create a new treenode between two existing nodes. Its creator and creation_date information will be set to information of child node. No node will be created, if the node on the edge between the given child and parent node. """ # Use creation time, if part of parameter set params = {} float_values = { 'x': 0, 'y': 0, 'z': 0, 'radius': 0 } int_values = { 'confidence': 0, 'parent_id': -1, 'child_id': -1 } for p in float_values.keys(): params[p] = float(request.POST.get(p, float_values[p])) for p in int_values.keys(): params[p] = int(request.POST.get(p, int_values[p])) # If siblings should be taken over, all children of the parent node will be # come children of the inserted node. This requires extra state # information: the child state for the paren. takeover_child_ids = get_request_list(request.POST, 'takeover_child_ids', None, int) # Get optional initial links to connectors, expect each entry to be a list # of connector ID and relation ID. try: links = get_request_list(request.POST, 'links', [], int) except Exception as e: raise ValueError("Couldn't parse list parameter: {}".format(e)) # Make sure the back-end is in the expected state if the node should have a # parent and will therefore become part of another skeleton. parent_id = params.get('parent_id') child_id = params.get('child_id') if parent_id not in (-1, None): s = request.POST.get('state') # Testing egular edge insertion is assumed if a child ID is provided partial_child_checks = [] if child_id in (-1, None) else [child_id] if takeover_child_ids: partial_child_checks.extend(takeover_child_ids) state.validate_state(parent_id, s, node=True, children=partial_child_checks or False, lock=True), # Find child and parent of new treenode child = Treenode.objects.get(pk=params['child_id']) parent = Treenode.objects.get(pk=params['parent_id']) # Make sure both nodes are actually child and parent if not child.parent == parent: raise ValueError('The provided nodes need to be child and parent') # Make sure the requested location for the new node is on the edge between # both existing nodes if the user has no edit permissions on the neuron. try: can_edit_treenode_or_fail(request.user, project_id, parent.id) user, time = request.user, None except: child_loc = Point3D(child.location_x, child.location_y, child.location_z) parent_loc = Point3D(parent.location_x, parent.location_y, parent.location_z) new_node_loc = Point3D(params['x'], params['y'], params['z']) if not is_collinear(child_loc, parent_loc, new_node_loc, True, 0.001): raise ValueError('New node location has to be between child and parent') # Use creator and creation time for neighboring node that was created last. if child.creation_time < parent.creation_time: user, time = parent.user, parent.creation_time else: user, time = child.user, child.creation_time # Create new treenode new_treenode = _create_treenode(project_id, user, request.user, params['x'], params['y'], params['z'], params['radius'], params['confidence'], -1, params['parent_id'], time) # Update parent of child to new treenode, do this in raw SQL to also get the # updated edition time Update also takeover children cursor = connection.cursor() params = [new_treenode.treenode_id, child.id] if takeover_child_ids: params.extend(takeover_child_ids) child_template = ",".join(("%s",) * (len(takeover_child_ids) + 1)) else: child_template = "%s" cursor.execute(""" UPDATE treenode SET parent_id = %s WHERE id IN ({}) RETURNING id, edition_time """.format(child_template), params) result = cursor.fetchall() if not result or (len(params) - 1) != len(result): raise ValueError("Couldn't update parent of inserted node's child: " + child.id) child_edition_times = [[k,v] for k,v in result] # Create all initial links if links: created_links = create_connector_link(project_id, request.user.id, new_treenode.treenode_id, new_treenode.skeleton_id, links) else: created_links = [] return JsonResponse({ 'treenode_id': new_treenode.treenode_id, 'skeleton_id': new_treenode.skeleton_id, 'edition_time': new_treenode.edition_time, 'parent_edition_time': new_treenode.parent_edition_time, 'child_edition_times': child_edition_times, 'created_links': created_links })