def insert_into_log(project_id, user_id, op_type, location=None, freetext=None): """ Inserts a new entry into the log table. If the location parameter is passed, it is expected to be an iteratable (list, tuple). """ # valid operation types operation_type_array = [ "rename_root", "create_neuron", "rename_neuron", "remove_neuron", "move_neuron", "create_group", "rename_group", "remove_group", "move_group", "create_skeleton", "rename_skeleton", "remove_skeleton", "move_skeleton", "split_skeleton", "join_skeleton", "reroot_skeleton", "change_confidence", "reset_reviews" ] if not op_type in operation_type_array: return {'error': 'Operation type {0} not valid'.format(op_type)} new_log = Log() new_log.user_id = user_id new_log.project_id = project_id new_log.operation_type = op_type if not location is None: new_log.location = Double3D(*location) if not freetext is None: new_log.freetext = freetext new_log.save()
def insert_into_log(project_id: Union[int, str], user_id, op_type: str, location=None, freetext=None) -> Optional[Dict[str, str]]: """ Inserts a new entry into the log table. If the location parameter is passed, it is expected to be an iteratable (list, tuple). """ # valid operation types operation_type_array = [ "rename_root", "create_neuron", "rename_neuron", "remove_neuron", "move_neuron", "create_group", "rename_group", "remove_group", "move_group", "create_skeleton", "rename_skeleton", "remove_skeleton", "move_skeleton", "split_skeleton", "join_skeleton", "reroot_skeleton", "change_confidence", "reset_reviews" ] if op_type not in operation_type_array: raise ValueError(f'Operation type {op_type} not valid') new_log = Log() new_log.user_id = user_id new_log.project_id = project_id new_log.operation_type = op_type if location is not None: new_log.location = Double3D(*location) if freetext is not None: new_log.freetext = freetext new_log.save() return None
def _export_review_skeleton(project_id=None, skeleton_id=None, format=None): treenodes = Treenode.objects.filter(skeleton_id=skeleton_id).values_list( 'id', 'location', 'parent_id', 'reviewer_id') g = nx.DiGraph() reviewed = set() for t in treenodes: loc = Double3D.from_str(t[1]) # While at it, send the reviewer ID, which is useful to iterate fwd # to the first unreviewed node in the segment. g.add_node(t[0], { 'id': t[0], 'x': loc.x, 'y': loc.y, 'z': loc.z, 'rid': t[3] }) if -1 != t[3]: reviewed.add(t[0]) if t[2]: # if parent g.add_edge(t[2], t[0]) # edge from parent to child else: root_id = t[0] # Create all sequences, as long as possible and always from end towards root distances = edge_count_to_root( g, root_node=root_id) # distance in number of edges from root seen = set() sequences = [] # Iterate end nodes sorted from highest to lowest distance to root endNodeIDs = (nID for nID in g.nodes() if 0 == len(g.successors(nID))) for nodeID in sorted(endNodeIDs, key=distances.get, reverse=True): sequence = [g.node[nodeID]] parents = g.predecessors(nodeID) while parents: parentID = parents[0] sequence.append(g.node[parentID]) if parentID in seen: break seen.add(parentID) parents = g.predecessors(parentID) if len(sequence) > 1: sequences.append(sequence) segments = [] for sequence in sorted(sequences, key=len, reverse=True): segments.append({ 'id': len(segments), 'sequence': sequence, 'status': '%.2f' % (100.0 * sum(1 for node in sequence if node['id'] in reviewed) / len(sequence)), 'nr_nodes': len(sequence) }) return segments
def link_roi_to_class_instance(request, project_id=None, relation_id=None, stack_id=None, ci_id=None): """ With the help of this method one can link a region of interest (ROI) to a class instance. The information about the ROI is passed as POST variables. """ # Try to get all needed POST parameters x_min = float(request.POST['x_min']) x_max = float(request.POST['x_max']) y_min = float(request.POST['y_min']) y_max = float(request.POST['y_max']) z = float(request.POST['z']) zoom_level = int(request.POST['zoom_level']) rotation_cw = int(request.POST['rotation_cw']) # Get related objects project = Project.objects.get(id=project_id) stack = Stack.objects.get(id=stack_id) ci = ClassInstance.objects.get(id=ci_id) rel = Relation.objects.get(id=relation_id) # Calculate ROI center and extent cx = (x_max + x_min) * 0.5 cy = (y_max + y_min) * 0.5 cz = z width = abs(x_max - x_min) height = abs(y_max - y_min) # Create a new ROI class instance roi = RegionOfInterest() roi.user = request.user roi.editor = request.user roi.project = project roi.stack = stack roi.zoom_level = zoom_level roi.location = Double3D(cx, cy, cz) roi.width = width roi.height = height roi.rotation_cw = rotation_cw roi.save() # Link ROI and class instance roi_ci = RegionOfInterestClassInstance() roi_ci.user = request.user roi_ci.project = project roi_ci.relation = rel roi_ci.region_of_interest = roi roi_ci.class_instance = ci roi_ci.save() # Create cropped image, if wanted if settings.ROI_AUTO_CREATE_IMAGE: file_name, file_path = create_roi_path(roi.id) create_roi_image(request.user, project_id, roi.id, file_path) # Build result data set status = {'status': "Created new ROI with ID %s." % roi.id} return HttpResponse(json.dumps(status))
def _create_interpolated_treenode(request, params, project_id, skip_last): """ Create interpolated treenodes between the 'parent_id' and the clicked x,y,z coordinate. The skip_last is to prevent the creation of the last node, used by the join_skeletons_interpolated. """ response_on_error = 'Could not create interpolated treenode' try: parent = Treenode.objects.get(pk=params['parent_id']) parent_skeleton_id = parent.skeleton_id loc = parent.location parent_x = decimal.Decimal(loc.x) parent_y = decimal.Decimal(loc.y) parent_z = decimal.Decimal(loc.z) steps = abs((params['z'] - parent_z) / params['resz']).quantize( decimal.Decimal('1'), rounding=decimal.ROUND_FLOOR) if steps == decimal.Decimal(0): steps = decimal.Decimal(1) dx = (params['x'] - parent_x) / steps dy = (params['y'] - parent_y) / steps dz = (params['z'] - parent_z) / steps broken_slices = set( int(bs.index) for bs in BrokenSlice.objects.filter(stack=params['stack_id'])) sign = -1 if dz < 0 else 1 # Loop the creation of treenodes in z resolution steps until target # section is reached parent_id = params['parent_id'] atn_slice_index = ((parent_z - params['stack_translation_z']) / params['resz']).quantize( decimal.Decimal('1'), rounding=decimal.ROUND_FLOOR) for i in range(1, steps + (0 if skip_last else 1)): if (atn_slice_index + i * sign) in broken_slices: continue response_on_error = 'Error while trying to insert treenode.' new_treenode = Treenode() new_treenode.user_id = request.user.id new_treenode.editor_id = request.user.id new_treenode.project_id = project_id new_treenode.location = Double3D(float(parent_x + dx * i), float(parent_y + dy * i), float(parent_z + dz * i)) new_treenode.radius = params['radius'] new_treenode.skeleton_id = parent_skeleton_id new_treenode.confidence = params['confidence'] new_treenode.parent_id = parent_id # This is not a root node. new_treenode.save() parent_id = new_treenode.id # parent_id contains the ID of the last added node return parent_id, parent_skeleton_id except Exception as e: raise Exception(response_on_error + ':' + str(e))
def _export_review_skeleton(project_id=None, skeleton_id=None, format=None): """ Returns a list of segments for the requested skeleton. Each segment contains information about the review status of this part of the skeleton. """ # Get all treenodes of the requested skeleton treenodes = Treenode.objects.filter(skeleton_id=skeleton_id).values_list('id', 'location', 'parent_id') # Get all reviews for the requested skeleton reviews = get_treenodes_to_reviews(skeleton_ids=[skeleton_id]) # Add each treenode to a networkx graph and attach reviewer information to # it. g = nx.DiGraph() reviewed = set() for t in treenodes: loc = Double3D.from_str(t[1]) # While at it, send the reviewer IDs, which is useful to iterate fwd # to the first unreviewed node in the segment. g.add_node(t[0], {'id': t[0], 'x': loc.x, 'y': loc.y, 'z': loc.z, 'rids': reviews[t[0]]}) if reviews[t[0]]: reviewed.add(t[0]) if t[2]: # if parent g.add_edge(t[2], t[0]) # edge from parent to child else: root_id = t[0] # Create all sequences, as long as possible and always from end towards root distances = edge_count_to_root(g, root_node=root_id) # distance in number of edges from root seen = set() sequences = [] # Iterate end nodes sorted from highest to lowest distance to root endNodeIDs = (nID for nID in g.nodes() if 0 == len(g.successors(nID))) for nodeID in sorted(endNodeIDs, key=distances.get, reverse=True): sequence = [g.node[nodeID]] parents = g.predecessors(nodeID) while parents: parentID = parents[0] sequence.append(g.node[parentID]) if parentID in seen: break seen.add(parentID) parents = g.predecessors(parentID) if len(sequence) > 1: sequences.append(sequence) # Calculate status segments = [] for sequence in sorted(sequences, key=len, reverse=True): segments.append({ 'id': len(segments), 'sequence': sequence, 'status': '%.2f' % (100.0 * sum(1 for node in sequence if node['id'] in reviewed) / len(sequence)), 'nr_nodes': len(sequence) }) return segments
def _update(Kind, table, nodes, now, user): if not nodes: return # 0: id # 1: X # 2: Y # 3: Z can_edit_all_or_fail(user, (node[0] for node in nodes.itervalues()), table) for node in nodes.itervalues(): Kind.objects.filter(id=int(node[0])).update( editor=user, edition_time=now, location=Double3D(float(node[1]), float(node[2]), float(node[3])))
def export_review_skeleton(request, project_id=None, skeleton_id=None, format=None): """ Export the skeleton as a list of sequences of entries, each entry containing an id, a sequence of nodes, the percent of reviewed nodes, and the node count. """ treenodes = Treenode.objects.filter(skeleton_id=skeleton_id).values_list('id', 'location', 'parent_id', 'reviewer_id') g = nx.DiGraph() reviewed = set() for t in treenodes: loc = Double3D.from_str(t[1]) # While at it, send the reviewer ID, which is useful to iterate fwd # to the first unreviewed node in the segment. g.add_node(t[0], {'id': t[0], 'x': loc.x, 'y': loc.y, 'z': loc.z, 'rid': t[3]}) if -1 != t[3]: reviewed.add(t[0]) if t[2]: # if parent g.add_edge(t[2], t[0]) # edge from parent to child else: root_id = t[0] # Create all sequences, as long as possible and always from end towards root distances = edge_count_to_root(g, root_node=root_id) # distance in number of edges from root seen = set() sequences = [] # Iterate end nodes sorted from highest to lowest distance to root endNodeIDs = (nID for nID in g.nodes() if 0 == len(g.successors(nID))) for nodeID in sorted(endNodeIDs, key=distances.get, reverse=True): sequence = [g.node[nodeID]] parents = g.predecessors(nodeID) while parents: parentID = parents[0] sequence.append(g.node[parentID]) if parentID in seen: break seen.add(parentID) parents = g.predecessors(parentID) if len(sequence) > 1: sequences.append(sequence) segments = [] for sequence in sorted(sequences, key=len, reverse=True): segments.append({ 'id': len(segments), 'sequence': sequence, 'status': '%.2f' % (100.0 * sum(1 for node in sequence if node['id'] in reviewed) / len(sequence)), 'nr_nodes': len(sequence) }) return HttpResponse(json.dumps(segments))
def insert_stack(self): s = Stack() s.title = "Example Stack" s.image_base = "http://incf.ini.uzh.ch/image-stack-fib/" s.trakem2_project = False s.dimension = Integer3D(x=2048, y=1536, z=460) s.resolution = Double3D(x=5.0001, y=5.0002, z=9.0003) s.num_zoom_levels = -1 s.file_extension = 'jpg' s.tile_width = 256 s.tile_height = 256 s.tile_source_type = 1 s.save() return s
def format_node_data(node): ''' Formats node data for our json output. When we start using Django 1.4, we can use prefetch_related instead of using .values('treenode__xxx'), and will then be able to access a proper location object. ''' location = Double3D.from_str(node['treenode__location']) return { 'id': node['treenode'], 'x': int(location.x), 'y': int(location.y), 'z': int(location.z), 'skid': node['treenode__skeleton']}
def update_textlabel(request: HttpRequest, project_id=None) -> HttpResponse: params = {} parameter_names = [ 'tid', 'pid', 'x', 'y', 'z', 'text', 'type', 'r', 'g', 'b', 'a', 'font_name', 'font_style', 'font_size', 'scaling' ] for p in parameter_names: params[p] = request.POST.get(p, None) # Scaling is given 0 or 1 value by the caller, but our models use bool if params['scaling'] is not None: params['scaling'] = bool(int(params['scaling'])) # Type must be either bubble or text. if params['type'] is not None: if (params['type'] != 'bubble'): params['type'] = 'text' response_on_error = '' try: response_on_error = 'Failed to find Textlabel with id %s.' % params[ 'tid'] label = Textlabel.objects.filter(id=params['tid'])[0] response_on_error = 'Failed to update Textlabel with id %s.' % params[ 'tid'] special_parameters = ['x', 'y', 'z', 'r', 'g', 'b', 'a', 'tid'] # Set new values for label unless they haven't been specified or need # special handling. # for par in [p for p in parameter_names if p not in special_parameters]: for par in set(params.keys()).difference(special_parameters): if params[par] is not None: setattr(label, par, params[par]) label.save() # If all parameters x, y and z have been specified, change the location if all( [val is not None for val in [params[p] for p in ['x', 'y', 'z']]]): response_on_error = 'Failed to update the location of textlabel with id %s' % params[ 'tid'] TextlabelLocation.objects.filter(textlabel=params['tid']).update( location=Double3D(float(params['x']), float(params['y']), float(params['z']))) return HttpResponse(' ') except Exception as e: raise ValueError(response_on_error + ':' + str(e))
def insert_new_treenode(parent_id=None, skeleton=None): """ If the parent_id is not None and the skeleton_id of the parent does not match with the skeleton.id, then the database will throw an error given that the skeleton_id, being defined as foreign key in the treenode table, will not meet the being-foreign requirement. """ new_treenode = Treenode() new_treenode.user = request.user new_treenode.editor = request.user new_treenode.project_id = project_id new_treenode.location = Double3D(float(params['x']), float(params['y']), float(params['z'])) new_treenode.radius = int(params['radius']) new_treenode.skeleton = skeleton new_treenode.confidence = int(params['confidence']) if parent_id: new_treenode.parent_id = parent_id new_treenode.save() return new_treenode
def insert_stack(self): s = Stack() s.title = "Example Stack" s.dimension = Integer3D(x=2048, y=1536, z=460) s.resolution = Double3D(x=5.0001, y=5.0002, z=9.0003) s.save() sm = StackMirror() sm.stack = s sm.image_base = "http://incf.ini.uzh.ch/image-stack-fib/" sm.file_extension = 'jpg' sm.tile_width = 256 sm.tile_height = 256 sm.tile_source_type = 1 sm.save() return s
def create_textlabel(request, project_id=None): print >> sys.stderr, 'creating text label' params = {} param_defaults = { 'x': 0, 'y': 0, 'z': 0, 'text': 'Edit this text...', 'type': 'text', 'r': 1, 'g': 0.5, 'b': 0, 'a': 1, 'fontname': False, 'fontstyle': False, 'fontsize': False, 'scaling': False} for p in param_defaults.keys(): params[p] = request.POST.get(p, param_defaults[p]) if (params['type'] != 'bubble'): params['type'] = 'text' new_label = Textlabel( text=params['text'], type=params['type'], scaling=params['scaling'] ) new_label.project_id = project_id if params['fontname']: new_label.font_name = params['fontname'] if params['fontstyle']: new_label.font_style = params['fontstyle'] if params['fontsize']: new_label.font_size = params['fontsize'] new_label.save() TextlabelLocation( textlabel=new_label, location=Double3D(float(params['x']), float(params['y']), float(params['z']))).save() return HttpResponse(json.dumps({'tid': new_label.id}))
def create_connector(request, project_id=None): query_parameters = {} default_values = {'x': 0, 'y': 0, 'z': 0, 'confidence': 5} for p in default_values.keys(): query_parameters[p] = request.POST.get(p, default_values[p]) parsed_confidence = int(query_parameters['confidence']) if parsed_confidence < 1 or parsed_confidence > 5: return HttpResponse( json.dumps({'error': 'Confidence not in range 1-5 inclusive.'})) location = Double3D(x=float(query_parameters['x']), y=float(query_parameters['y']), z=float(query_parameters['z'])) new_connector = Connector(user=request.user, editor=request.user, project=Project.objects.get(id=project_id), location=location, confidence=parsed_confidence) new_connector.save() return HttpResponse(json.dumps({'connector_id': new_connector.id}))
def _parse_location(loc): return Double3D(*(imap(float, loc[1:-1].split(','))))
def list_connector(request, project_id=None): stack_id = request.POST.get('stack_id', None) skeleton_id = request.POST.get('skeleton_id', None) def empty_result(): return HttpResponse( json.dumps({ 'iTotalRecords': 0, 'iTotalDisplayRecords': 0, 'aaData': [] })) if not skeleton_id: return empty_result() else: skeleton_id = int(skeleton_id) relation_type = int(request.POST.get('relation_type', 0)) # 0: Presyn, 1 Postsyn, 2 Gj display_start = int(request.POST.get('iDisplayStart', 0)) display_length = int(request.POST.get('iDisplayLength', 0)) sorting_column = int(request.POST.get('iSortCol_0', 0)) sort_descending = upper(request.POST.get('sSortDir_0', 'DESC')) != 'ASC' response_on_error = '' try: response_on_error = 'Could not fetch relations.' relation_map = get_relation_to_id_map(project_id) for rel in [ 'presynaptic_to', 'postsynaptic_to', 'gapjunction_with', 'element_of', 'labeled_as' ]: if rel not in relation_map: raise Exception('Failed to find the required relation %s' % rel) if relation_type == 1: relation_type_id = relation_map['presynaptic_to'] inverse_relation_type_id = relation_map['postsynaptic_to'] elif relation_type == 2: relation_type_id = relation_map['gapjunction_with'] inverse_relation_type_id = relation_map['gapjunction_with'] else: relation_type_id = relation_map['postsynaptic_to'] inverse_relation_type_id = relation_map['presynaptic_to'] response_on_error = 'Failed to select connectors.' cursor = connection.cursor() cursor.execute( ''' SELECT connector.id AS connector_id, tn_other.user_id AS connector_user_id, treenode_user.username AS connector_username, connector.location_x AS connector_x, connector.location_y AS connector_y, connector.location_z AS connector_z, tn_other.id AS other_treenode_id, tn_other.location_x AS other_treenode_x, tn_other.location_y AS other_treenode_y, tn_other.location_z AS other_treenode_z, tn_other.skeleton_id AS other_skeleton_id, tn_this.location_x AS this_treenode_x, tn_this.location_y AS this_treenode_y, tn_this.location_z AS this_treenode_z, tn_this.id AS this_treenode_id, tc_this.relation_id AS this_to_connector_relation_id, tc_other.relation_id AS connector_to_other_relation_id, tc_other.confidence AS confidence, to_char(connector.edition_time, 'DD-MM-YYYY HH24:MI') AS last_modified FROM treenode tn_other, treenode_connector tc_other, connector, "auth_user" treenode_user, treenode_connector tc_this, treenode tn_this WHERE treenode_user.id = tn_other.user_id AND tn_other.id = tc_other.treenode_id AND tc_other.connector_id = connector.id AND tc_other.relation_id = %s AND tc_this.connector_id = connector.id AND tn_this.id = tc_this.treenode_id AND tn_this.skeleton_id = %s AND tc_this.relation_id = %s ORDER BY connector_id, other_treenode_id, this_treenode_id ''', [inverse_relation_type_id, skeleton_id, relation_type_id]) connectors = cursor_fetch_dictionary(cursor) connected_skeletons = map(lambda con: con['other_skeleton_id'], connectors) connector_ids = map(lambda con: con['connector_id'], connectors) response_on_error = 'Failed to find counts of treenodes in skeletons.' skel_tn_count = Treenode.objects.filter(skeleton__in=connected_skeletons)\ .values('skeleton').annotate(treenode_count=Count('skeleton')) # .values to group by skeleton_id. See http://tinyurl.com/dj-values-annotate skeleton_to_treenode_count = {} for s in skel_tn_count: skeleton_to_treenode_count[s['skeleton']] = s['treenode_count'] # Rather than do a LEFT OUTER JOIN to also include the connectors # with no partners, just do another query to find the connectors # without the conditions: response_on_error = 'Failed to select all connectors.' cursor.execute( ''' SELECT connector.id AS connector_id, connector.user_id AS connector_user_id, connector_user.username AS connector_username, connector.location_x AS connector_x, connector.location_y AS connector_y, connector.location_z AS connector_z, tn_this.id AS this_treenode_id, tc_this.relation_id AS this_to_connector_relation_id, tc_this.confidence AS confidence, to_char(connector.edition_time, 'DD-MM-YYYY HH24:MI') AS last_modified FROM connector, "auth_user" connector_user, treenode_connector tc_this, treenode tn_this WHERE connector_user.id = connector.user_id AND tc_this.connector_id = connector.id AND tn_this.id = tc_this.treenode_id AND tn_this.skeleton_id = %s AND tc_this.relation_id = %s ORDER BY connector_id, this_treenode_id ''', [skeleton_id, relation_type_id]) for row in cursor_fetch_dictionary(cursor): connector_id = row['connector_id'] if connector_id not in connector_ids: connectors.append(row) connector_ids.append(connector_id) # For each of the connectors, find all of its labels: response_on_error = 'Failed to find the labels for connectors' if (connector_ids > 0): connector_labels = ConnectorClassInstance.objects.filter( project=project_id, connector__in=connector_ids, relation=relation_map['labeled_as']).values( 'connector', 'class_instance__name') labels_by_connector = { } # Key: Connector ID, Value: List of labels. for label in connector_labels: if label['connector'] not in labels_by_connector: labels_by_connector[label['connector']] = [ label['class_instance__name'] ] else: labels_by_connector[label['connector']].append( label['class_instance__name']) # Sort labels by name for labels in labels_by_connector.values(): labels.sort(key=upper) total_result_count = len(connectors) if 0 == total_result_count: return empty_result() # Paging if display_length == 0: connectors = connectors[display_start:] connector_ids = connector_ids[display_start:] else: connectors = connectors[display_start:display_start + display_length] connector_ids = connector_ids[display_start:display_start + display_length] response_on_error = 'Could not retrieve resolution and translation parameters for project.' if stack_id: resolution = get_object_or_404(Stack, id=int(stack_id)).resolution translation = get_object_or_404(ProjectStack, stack=int(stack_id), project=project_id).translation else: resolution = Double3D(1.0, 1.0, 1.0) translation = Double3D(0.0, 0.0, 0.0) # Format output aaData_output = [] for c in connectors: response_on_error = 'Failed to format output for connector with ID %s.' % c[ 'connector_id'] if 'other_skeleton_id' in c: connected_skeleton_treenode_count = skeleton_to_treenode_count[ c['other_skeleton_id']] else: c['other_skeleton_id'] = '' c['other_treenode_id'] = '' c['other_treenode_x'] = c['connector_x'] c['other_treenode_y'] = c['connector_y'] c['other_treenode_z'] = c['connector_z'] connected_skeleton_treenode_count = 0 if c['connector_id'] in labels_by_connector: labels = ', '.join( map(str, labels_by_connector[c['connector_id']])) else: labels = '' row = [] row.append(c['connector_id']) row.append(c['other_skeleton_id']) row.append(c['other_treenode_x']) #('%.2f' % ) row.append(c['other_treenode_y']) z = c['other_treenode_z'] row.append(z) # FIXME: This is the only place we need a stack nad this can be # done in the client as well. So we really want to keep this and # have a more complicated API? row.append(int((z - translation.z) / resolution.z)) row.append(c['confidence']) row.append(labels) row.append(connected_skeleton_treenode_count) row.append(c['connector_username']) row.append(c['other_treenode_id']) row.append(c['last_modified']) aaData_output.append(row) # Sort output def fetch_value_for_sorting(row): value = row[sorting_column] if isinstance(value, str) or isinstance(value, unicode): return upper(value) return value aaData_output.sort(key=fetch_value_for_sorting) # Fix excessive decimal precision in coordinates for row in aaData_output: row[2] = float('%.2f' % row[2]) row[3] = float('%.2f' % row[3]) row[4] = float('%.2f' % row[4]) if sort_descending: aaData_output.reverse() return HttpResponse( json.dumps({ 'iTotalRecords': total_result_count, 'iTotalDisplayRecords': total_result_count, 'aaData': aaData_output })) except Exception as e: raise Exception(response_on_error + ':' + str(e))
def label_update(request, project_id=None, location_id=None, ntype=None): """ location_id is the ID of a treenode or connector. ntype is either 'treenode' or 'connector'. """ labeled_as_relation = Relation.objects.get(project=project_id, relation_name='labeled_as') p = get_object_or_404(Project, pk=project_id) # TODO will FAIL when a tag contains a coma by itself new_tags = request.POST['tags'].split(',') delete_existing_labels = request.POST.get('delete_existing', 'true') == 'true' kwargs = { 'relation': labeled_as_relation, 'class_instance__class_column__class_name': 'label' } table = get_link_model(ntype) if 'treenode' == ntype: kwargs['treenode__id'] = location_id node = Treenode.objects.get(id=location_id) elif 'connector' == ntype: kwargs['connector__id'] = location_id node = Connector.objects.get(id=location_id) if not table: raise Http404('Unknown node type: "%s"' % (ntype, )) # Get the existing list of tags for the tree node/connector and delete any # that are not in the new list. existingLabels = table.objects.filter( **kwargs).select_related('class_instance__name') existing_names = set(ele.class_instance.name for ele in existingLabels) labels_to_delete = table.objects.filter(**kwargs).exclude( class_instance__name__in=new_tags) if delete_existing_labels: # Iterate over all labels that should get deleted to check permission # on each one. Remember each label that couldn't be deleted in the # other_labels array. other_labels = [] deleted_labels = [] for l in labels_to_delete: try: can_edit_or_fail(request.user, l.id, table._meta.db_table) if remove_label(l.id, ntype): deleted_labels.append(l) else: other_labels.append(l) except: other_labels.append(l) # Create change requests for labels associated to the treenode by other users for label in other_labels: change_request_params = { 'type': 'Remove Tag', 'project': p, 'user': request.user, 'recipient': node.user, 'location': Double3D(node.location_x, node.location_y, node.location_z), ntype: node, 'description': "Remove tag '%s'" % label.class_instance.name, 'validate_action': 'from catmaid.control.label import label_exists\n' + 'is_valid = label_exists(%s, "%s")' % (str(label.id), ntype), 'approve_action': 'from catmaid.control.label import remove_label\n' + 'remove_label(%s, "%s")' % (str(label.id), ntype) } ChangeRequest(**change_request_params).save() # Add any new labels. label_class = Class.objects.get(project=project_id, class_name='label') kwargs = { 'user': request.user, 'project': p, 'relation': labeled_as_relation, ntype: node } for tag_name in new_tags: if len(tag_name) > 0 and tag_name not in existing_names: # Make sure the tag instance exists existing_tags = tuple( ClassInstance.objects.filter(project=p, name=tag_name, class_column=label_class)) if len(existing_tags) < 1: tag = ClassInstance(project=p, name=tag_name, user=request.user, class_column=label_class) tag.save() else: tag = existing_tags[0] # Associate the tag with the treenode/connector. kwargs['class_instance'] = tag tci = table( **kwargs ) # creates new TreenodeClassInstance or ConnectorClassInstance tci.save() if node.user != request.user: # Inform the owner of the node that the tag was added and give them the option of removing it. change_request_params = { 'type': 'Add Tag', 'description': 'Added tag \'' + tag_name + '\'', 'project': p, 'user': request.user, 'recipient': node.user, 'location': Double3D(node.location_x, node.location_y, node.location_z), ntype: node, 'validate_action': 'from catmaid.control.label import label_exists\n' + 'is_valid = label_exists(%s, "%s")' % (str(tci.id), ntype), 'reject_action': 'from catmaid.control.label import remove_label\n' + 'remove_label(%s, "%s")' % (str(tci.id), ntype) } ChangeRequest(**change_request_params).save() return HttpResponse(json.dumps({'message': 'success'}), content_type='application/json')
def import_projects(user, pre_projects, make_public, tags, permissions, tile_width, tile_height, tile_source_type, cls_graph_ids_to_link): """ Creates real CATMAID projects out of the PreProject objects and imports them into CATMAID. """ imported = [] not_imported = [] for pp in pre_projects: try: # Create stacks and add them to project stacks = [] for s in pp.stacks: stack = Stack.objects.create(title=s.name, dimension=s.dimension, resolution=s.resolution, image_base=s.image_base, num_zoom_levels=s.num_zoom_levels, file_extension=s.file_extension, tile_width=tile_width, tile_height=tile_height, tile_source_type=tile_source_type, metadata=s.metadata) stacks.append(stack) # Add overlays of this stack for o in s.overlays: Overlay.objects.create(title=o.name, stack=stack, image_base=o.image_base, default_opacity=o.default_opacity, file_extension=o.file_extension, tile_width=tile_width, tile_height=tile_height, tile_source_type=tile_source_type) # Create new project p = Project.objects.create(title=pp.name, public=make_public) # Assign permissions to project assigned_permissions = [] for user_or_group, perm in permissions: assigned_perm = assign(perm.codename, user_or_group, p) assigned_permissions.append(assigned_perm) # Tag the project p.tags.add(*tags) # Add stacks to project for s in stacks: trln = Double3D() ps = ProjectStack.objects.create(project=p, stack=s, translation=trln) # Make project persistent p.save() # Link classification graphs for cg in cls_graph_ids_to_link: workspace = settings.ONTOLOGY_DUMMY_PROJECT_ID cgroot = ClassInstance.objects.get(pk=cg) link_existing_classification(workspace, user, p, cgroot) # Remember created project imported.append(pp) except Exception as e: not_imported.append((pp, e)) return (imported, not_imported)
def label_update(request, project_id=None, location_id=None, ntype=None): """ location_id is the ID of a treenode or connector. ntype is either 'treenode' or 'connector'. """ labeled_as_relation = Relation.objects.get(project=project_id, relation_name='labeled_as') p = get_object_or_404(Project, pk=project_id) # TODO will FAIL when a tag contains a coma by itself new_tags = request.POST['tags'].split(',') delete_existing_labels = request.POST.get('delete_existing', 'true') == 'true' kwargs = { 'relation': labeled_as_relation, 'class_instance__class_column__class_name': 'label' } table = get_link_model(ntype) if 'treenode' == ntype: kwargs['treenode__id'] = location_id node = Treenode.objects.get(id=location_id) elif 'connector' == ntype: kwargs['connector__id'] = location_id node = Connector.objects.get(id=location_id) if not table: raise Http404('Unknown node type: "%s"' % (ntype, )) # Get the existing list of tags for the tree node/connector and delete any # that are not in the new list. existing_labels = table.objects.filter( **kwargs).select_related('class_instance__name') existing_names = set(ele.class_instance.name for ele in existing_labels) duplicate_labels = table.objects.filter(**kwargs).exclude( class_instance__name__in=new_tags).select_related( 'class_instance__name') other_labels = [] deleted_labels = [] if delete_existing_labels: # Iterate over all labels that should get deleted to check permission # on each one. Remember each label that couldn't be deleted in the # other_labels array. for l in duplicate_labels: try: can_edit_or_fail(request.user, l.id, table._meta.db_table) if remove_label(l.id, ntype): deleted_labels.append(l) else: other_labels.append(l) except: other_labels.append(l) # Create change requests for labels associated to the treenode by other users for label in other_labels: change_request_params = { 'type': 'Remove Tag', 'project': p, 'user': request.user, 'recipient': node.user, 'location': Double3D(node.location_x, node.location_y, node.location_z), ntype: node, 'description': "Remove tag '%s'" % label.class_instance.name, 'validate_action': 'from catmaid.control.label import label_exists\n' + 'is_valid = label_exists(%s, "%s")' % (str(label.id), ntype), 'approve_action': 'from catmaid.control.label import remove_label\n' + 'remove_label(%s, "%s")' % (str(label.id), ntype) } ChangeRequest(**change_request_params).save() # Add any new labels. label_class = Class.objects.get(project=project_id, class_name='label') kwargs = { 'user': request.user, 'project': p, 'relation': labeled_as_relation, ntype: node } new_labels = [] for tag_name in new_tags: if len(tag_name) > 0 and tag_name not in existing_names: # Make sure the tag instance exists existing_tags = tuple( ClassInstance.objects.filter(project=p, name=tag_name, class_column=label_class)) if len(existing_tags) < 1: tag = ClassInstance(project=p, name=tag_name, user=request.user, class_column=label_class) tag.save() else: tag = existing_tags[0] # Associate the tag with the treenode/connector. kwargs['class_instance'] = tag tci = table( **kwargs ) # creates new TreenodeClassInstance or ConnectorClassInstance tci.save() new_labels.append(tag_name) if node.user != request.user: # Inform the owner of the node that the tag was added and give them the option of removing it. change_request_params = { 'type': 'Add Tag', 'description': 'Added tag \'' + tag_name + '\'', 'project': p, 'user': request.user, 'recipient': node.user, 'location': Double3D(node.location_x, node.location_y, node.location_z), ntype: node, 'validate_action': 'from catmaid.control.label import label_exists\n' + 'is_valid = label_exists(%s, "%s")' % (str(tci.id), ntype), 'reject_action': 'from catmaid.control.label import remove_label\n' + 'remove_label(%s, "%s")' % (str(tci.id), ntype) } ChangeRequest(**change_request_params).save() response = { 'message': 'success', 'new_labels': new_labels, 'duplicate_labels': [ l.class_instance.name for l in duplicate_labels if l not in deleted_labels ], 'deleted_labels': [l.class_instance.name for l in deleted_labels], } # Check if any labels on this node violate cardinality restrictions on # its skeleton. if 'treenode' == ntype: limited_labels = { l: SKELETON_LABEL_CARDINALITY[l] for l in new_tags if l in SKELETON_LABEL_CARDINALITY } if limited_labels: ll_names, ll_maxes = zip(*limited_labels.items()) cursor = connection.cursor() cursor.execute( """ SELECT ll.name, COUNT(tci.treenode_id), ll.max FROM class_instance ci, treenode_class_instance tci, treenode tn, unnest(%s::text[], %s::integer[]) AS ll (name, max) WHERE ci.name = ll.name AND ci.project_id = %s AND ci.class_id = %s AND tci.class_instance_id = ci.id AND tci.relation_id = %s AND tn.id = tci.treenode_id AND tn.skeleton_id = %s GROUP BY ll.name, ll.max HAVING COUNT(tci.treenode_id) > ll.max """, (list(ll_names), list(ll_maxes), p.id, label_class.id, labeled_as_relation.id, node.skeleton_id)) if cursor.rowcount: response['warning'] = 'The skeleton has too many of the following tags: ' + \ ', '.join('{0} ({1}, max. {2})'.format(*row) for row in cursor.fetchall()) return JsonResponse(response)
def list_treenode_table(request, project_id=None): stack_id = request.POST.get('stack_id', None) specified_skeleton_count = request.POST.get('skeleton_nr', 0) display_start = request.POST.get('iDisplayStart', 0) display_length = request.POST.get('iDisplayLength', -1) should_sort = request.POST.get('iSortCol_0', None) filter_nodetype = request.POST.get('sSearch_1', None) filter_labels = request.POST.get('sSearch_2', None) relation_map = get_relation_to_id_map(project_id) response_on_error = '' try: def search_query_is_empty(): if specified_skeleton_count == 0: return True first_skeleton_id = request.POST.get('skeleton_0', None) if first_skeleton_id is None: return True elif upper(first_skeleton_id) in ['NONE', 'NULL']: return True return False if search_query_is_empty(): return HttpResponse( json.dumps({ 'iTotalRecords': 0, 'iTotalDisplayRecords': 0, 'aaData': [] })) else: response_on_error = 'Could not fetch %s skeleton IDs.' % \ specified_skeleton_count skeleton_ids = [int(request.POST.get('skeleton_%s' % i, 0)) \ for i in range(int(specified_skeleton_count))] if should_sort: column_count = int(request.POST.get('iSortingCols', 0)) sorting_directions = [request.POST.get('sSortDir_%d' % d) \ for d in range(column_count)] sorting_directions = map(lambda d: \ '-' if upper(d) == 'DESC' else '', sorting_directions) fields = [ 'tid', 'type', '"treenode"."labels"', 'confidence', 'x', 'y', 'z', '"treenode"."section"', 'radius', 'username', 'last_modified' ] # TODO type field not supported. sorting_index = [int(request.POST.get('iSortCol_%d' % d)) \ for d in range(column_count)] sorting_cols = map(lambda i: fields[i], sorting_index) response_on_error = 'Could not get the list of treenodes.' t = Treenode.objects \ .filter( project=project_id, skeleton_id__in=skeleton_ids) \ .extra( tables=['auth_user'], where=[ '"treenode"."user_id" = "auth_user"."id"'], select={ 'tid': '"treenode"."id"', 'radius': '"treenode"."radius"', 'confidence': '"treenode"."confidence"', 'parent_id': '"treenode"."parent_id"', 'user_id': '"treenode"."user_id"', 'edition_time': '"treenode"."edition_time"', 'x': '"treenode"."location_x"', 'y': '"treenode"."location_y"', 'z': '"treenode"."location_z"', 'username': '******', 'last_modified': 'to_char("treenode"."edition_time", \'DD-MM-YYYY HH24:MI\')' }) \ .distinct() # Rationale for using .extra(): # Since we don't use .order_by() for ordering, extra fields are not # included in the SELECT statement, and so .distinct() will work as # intended. See http://tinyurl.com/dj-distinct if should_sort: t = t.extra(order_by=[di + col \ for (di, col) in zip(sorting_directions, sorting_cols)]) if int(display_length) == -1: treenodes = list(t[display_start:]) else: treenodes = list(t[display_start:display_start + display_length]) # The number of results to be displayed should include items that are # filtered out. row_count = len(treenodes) # Filter out irrelevant treenodes if a label has been specified if 'labeled_as' in relation_map: response_on_error = 'Could not retrieve labels for project.' project_lables = TreenodeClassInstance.objects.filter( project=project_id, relation=relation_map['labeled_as']).values( 'treenode', 'class_instance__name') labels_by_treenode = {} # Key: Treenode ID, Value: List of labels. for label in project_lables: if label['treenode'] not in labels_by_treenode: labels_by_treenode[label['treenode']] = [ label['class_instance__name'] ] else: labels_by_treenode[label['treenode']].append( label['class_instance__name']) if filter_labels: def label_filter(treenode): if treenode.id not in labels_by_treenode: return False return upper(filter_labels) in upper(' '.join( labels_by_treenode[treenode.tid])) treenodes = filter(label_filter, treenodes) # Filter out irrelevant treenodes if a node type has been specified. # Count treenode's children to derive treenode types. The number of # children a treenode has determines its type. Types: # R : root (parent = null) # S : slab (has one child) # B : branch (has more than one child) # L : leaf (has no children) # X : undefined (uh oh!) if 0 == display_start and -1 == display_length: # All nodes are loaded: determine child_count from loaded nodes child_count = {} for treenode in treenodes: if treenode.parent is None: continue n_children = child_count.get(treenode.parent_id, 0) child_count[treenode.parent_id] = n_children + 1 else: # Query for parents response_on_error = 'Could not retrieve treenode parents.' child_count_query = Treenode.objects.filter( project=project_id, skeleton_id__in=skeleton_ids).annotate( child_count=Count('children')) child_count = {} for treenode in child_count_query: child_count[treenode.id] = treenode.child_count # Determine type for treenode in treenodes: if None == treenode.parent_id: treenode.nodetype = 'R' # Root continue children = child_count.get(treenode.tid, 0) if children == 1: treenode.nodetype = 'S' # Slab elif children == 0: treenode.nodetype = 'L' # Leaf elif children > 1: treenode.nodetype = 'B' # Branch else: treenode.nodetype = 'X' # Unknown, can never happen # Now that we've assigned node types, filter based on them: if filter_nodetype: filter_nodetype = upper(filter_nodetype) treenodes = [t for t in treenodes if t.nodetype in filter_nodetype] users = dict(User.objects.all().values_list('id', 'username')) users[-1] = "None" # Rather than AnonymousUser # Get all reviews for the current treenode set treenode_ids = [t.id for t in treenodes] treenode_to_reviews = get_treenodes_to_reviews(treenode_ids, umap=lambda r: users[r]) if stack_id: response_on_error = 'Could not retrieve resolution and translation ' \ 'parameters for project.' resolution = get_object_or_404(Stack, id=int(stack_id)).resolution translation = get_object_or_404(ProjectStack, stack=int(stack_id), project=project_id).translation else: resolution = Double3D(1.0, 1.0, 1.0) translation = Double3D(0.0, 0.0, 0.0) def formatTreenode(tn): row = [str(tn.tid)] row.append(tn.nodetype) if tn.tid in labels_by_treenode: row.append(', '.join(map(str, labels_by_treenode[tn.tid]))) else: row.append('') row.append(str(tn.confidence)) row.append('%.2f' % tn.x) row.append('%.2f' % tn.y) row.append('%.2f' % tn.z) row.append(int((tn.z - translation.z) / resolution.z)) row.append(str(tn.radius)) row.append(tn.username) row.append(tn.last_modified) row.append(', '.join(treenode_to_reviews.get(tn.id, ["None"]))) return row result = { 'iTotalRecords': row_count, 'iTotalDisplayRecords': row_count } response_on_error = 'Could not format output.' result['aaData'] = map(formatTreenode, treenodes) return HttpResponse(json.dumps(result)) except Exception as e: raise Exception(response_on_error + ':' + str(e))
def handle(self, *args, **options): if not options['user_id']: raise CommandError("You must specify a user ID with --user") user = User.objects.get(pk=options['user_id']) projects: Dict[str, Dict] = { 'Default Project': { 'stacks': [] }, 'Evaluation data set': { 'stacks': [] }, 'Focussed Ion Beam (FIB)': { 'stacks': [] }, } # Define the details of a stack for two of these projects: projects['Default Project']['stacks'].append({ 'title': 'Original data.', 'dimension': Integer3D(4096, 4096, 16), 'resolution': Double3D(3.2614000000000001, 3.2614000000000001, 60), 'comment': '''<p>©2007 by Stephan Saalfeld.</p> <p>Rendered with <a href="http://www.povray.org/">POV-Ray v3.6</a> using this <a href="http://fly.mpi-cbg.de/~saalfeld/download/volume.tar.bz2">scene-file</a>.</p>''', 'mirrors': [{ 'title': 'Public copy', 'image_base': 'http://fly.mpi-cbg.de/map/evaluation/original/', }] }) projects['Focussed Ion Beam (FIB)']['stacks'].append({ 'title': 'Focussed Ion Beam (FIB) stack of Rat Striatum', 'dimension': Integer3D(2048, 1536, 460), 'resolution': Double3D(5, 5, 9), 'comment': ''' <p>©2009 <a href="http://people.epfl.ch/graham.knott">Graham Knott</a>.</p> <p>Public INCF data set available at the <a href="http://www.incf.org/about/nodes/switzerland/data">Swiss INCF Node</a>.</p>''', 'mirrors': [{ 'title': 'FIB Public copy', 'image_base': 'http://incf.ini.uzh.ch/image-stack-fib/', }] }) # Make sure that each project and its stacks exist, and are # linked via ProjectStack: for project_title in projects: project_object, _ = Project.objects.get_or_create( title=project_title) for stack_dict in projects[project_title]['stacks']: stack, _ = Stack.objects.get_or_create( title=stack_dict['title'], defaults={ 'dimension': stack_dict['dimension'], 'resolution': stack_dict['resolution'], }) mirrors = list(StackMirror.objects.filter(stack=stack)) if not mirrors: for m in stack_dict['mirrors']: mirrors.append( StackMirror.objects.create( stack=stack, title=m['title'], image_base=m['image_base'])) ProjectStack.objects.get_or_create(project=project_object, stack=stack) projects[project_title]['project_object'] = project_object # Also set up the FIB project for tracing with treelines: tracing_project = projects['Focussed Ion Beam (FIB)']['project_object'] call_command('catmaid_setup_tracing_for_project', '--project_id', str(tracing_project.id), '--user', str(user.id))