class ProjectFavorite(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def post(self, request: Request, project_id: int) -> JsonResponse: """Mark this project as a favorite. """ project = get_object_or_404(Project, pk=project_id) _, created = FavoriteProject.objects.get_or_create( **{ 'project_id': project.id, 'user_id': request.user.id, }) return JsonResponse({ 'new': created, }) @method_decorator(requires_user_role(UserRole.Browse)) def delete(self, request: Request, project_id: int) -> JsonResponse: """Delete this project as a favorite. """ project = get_object_or_404(Project, pk=project_id) deleted = FavoriteProject.objects.filter(**{ 'project_id': project.id, 'user_id': request.user.id, }).delete() return JsonResponse({ 'deleted': deleted, })
class SuppressedVirtualTreenodeList(APIView): @method_decorator(requires_user_role([UserRole.Browse, UserRole.Annotate])) @never_cache def get(self, request:Request, project_id=None, treenode_id=None, format=None) -> Response: """List suppressed virtual nodes along the edge to this node. --- serializer: SuppressedVirtualTreenodeSerializer """ suppressed = SuppressedVirtualTreenode.objects.filter(child_id=treenode_id) serializer = SuppressedVirtualTreenodeSerializer(suppressed, many=True) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def post(self, request:Request, project_id=None, treenode_id=None, format=None) -> Response: """Suppress a virtual treenode along the edge to this node. Suppress a virtual treenode along the edge between this treenode and its parent from being traversed during normal topology navigation and review. --- parameters: - name: orientation description: | Stack orientation to determine which axis is the coordinate of the plane where virtual nodes are suppressed. 0 for z, 1 for y, 2 for x. required: true type: integer paramType: form - name: location_coordinate description: | Coordinate along the edge from this node to its parent where virtual nodes are suppressed. required: true type: number format: double paramType: form serializer: SuppressedVirtualTreenodeSerializer """ child = get_object_or_404(Treenode, pk=treenode_id) if not child.parent_id: raise ValidationError('Root nodes do not have virtual nodes') orientation = int(request.POST['orientation']) if not 0 <= orientation <= 2: raise ValidationError('Orientation axis must be 0, 1 or 2') location_coordinate = float(request.POST['location_coordinate']) location_field = 'location_' + ['z', 'y', 'x'][orientation] child_c = getattr(child, location_field) parent_c = getattr(child.parent, location_field) if not min(child_c, parent_c) <= location_coordinate <= max(child_c, parent_c): raise ValidationError('Suppressed node must be between child and parent nodes') suppressed = SuppressedVirtualTreenode.objects.create( project_id=project_id, user=request.user, child_id=treenode_id, orientation=orientation, location_coordinate=location_coordinate) serializer = SuppressedVirtualTreenodeSerializer(suppressed) return Response(serializer.data)
class VolumeDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id, volume_id): """Get detailed information on a spatial volume or set its properties. The result will contain the bounding box of the volume's geometry and the actual geometry encoded in X3D format. The response might might therefore be relatively large. """ p = get_object_or_404(Project, pk=project_id) volume = get_volume_details(p.id, volume_id) return Response(volume) @method_decorator(requires_user_role(UserRole.Annotate)) def post(self, request, project_id, volume_id): """Update the properties of a spatial volume. Only the fields that are provided are updated. If no mesh or bounding box parameter is changed, no type has to be provided. --- parameters: - name: type description: Type of volume to edit paramType: form type: string enum: ["box", "trimesh"] required: false - name: title description: Title of volume type: string required: false - name: comment description: A comment on a volume type: string required: false type: 'success': type: boolean required: true 'volume_id': type: integer required: true """ return update_volume(request, project_id=project_id, volume_id=volume_id) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request, project_id, volume_id): """Delete a particular spatial volume. """ return remove_volume(request, project_id=project_id, volume_id=volume_id)
class ProjectTokenList(APIView): @method_decorator(requires_user_role([UserRole.Admin])) @never_cache def get(self, request: Request, project_id) -> Response: """List project tokens available for this project, if the user is an admin. --- serializer: SimpleProjectTokenSerializer """ tokens = ProjectToken.objects.filter(project_id=project_id) serializer = SimpleProjectTokenSerializer(tokens, many=True) return Response(serializer.data) @method_decorator(requires_user_role([UserRole.Admin])) def post(self, request: Request, project_id) -> Response: """Create a new project token. The request requires admin permissions in the project. --- serializer: SimpleProjectTokenSerializer """ project = get_object_or_404(Project, pk=project_id) name = request.POST.get('name', '') needs_approval = get_request_bool(request.POST, 'needs_approval', False) default_permissions = set( get_request_list(request.POST, 'default_permissions', [])) allowed_permissions = set( get_perms_for_model(Project).values_list('codename', flat=True)) unknown_permissions = default_permissions - allowed_permissions if unknown_permissions: raise ValueError( f'Unknown permissions: {", ".join(unknown_permissions)}') token = ProjectToken.objects.create( **{ 'name': name, 'user_id': request.user.id, 'project_id': project.id, 'needs_approval': needs_approval, 'default_permissions': default_permissions, }) if not name: token.name = f'Project token {token.id}' token.save() serializer = SimpleProjectTokenSerializer(token) return Response(serializer.data)
class PointSetDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request:HttpRequest, project_id, pointset_id) -> JsonResponse: """Return a point set. parameters: - name: project_id description: Project of the returned point set type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false """ with_points = get_request_bool(request.query_params, 'with_points', False) simple = get_request_bool(request.query_params, 'simple', False) pointset = PointSet.objects.get(pk=pointset_id, project_id=project_id) pointset_data = serialize_pointset(pointset, simple) if with_points: pointset_data['points'] = [] return JsonResponse(pointset_data)
class ProjectDetail(APIView): def get(self, request:Request, project_id:int) -> JsonResponse: """Get details on a project. """ project = get_object_or_404(Project, pk=project_id) return JsonResponse({ 'id': project.id, 'title': project.title, 'comment': project.comment, }) @method_decorator(requires_user_role('delete_projectt')) @method_decorator(requires_user_role(UserRole.Admin)) def delete(self, request:Request, project_id:int) -> JsonResponse: """Delete a project. This requires <delete_project> permission on a project. """ delete_projects([project_id]) return JsonResponse({ 'deleted_project_id': project_id, }) @method_decorator(requires_user_role(UserRole.Admin)) def post(self, request:Request, project_id:int) -> JsonResponse: """Update properties of a project. This requires <can_administer> permission on a project. """ project = get_object_or_404(Project, pk=project_id) if 'title' in request.data: project.title = request.data.get('title') project.save() return JsonResponse({ 'id': project.id, 'title': project.title, 'comment': project.comment, })
class LandmarkLocationDetail(APIView): @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request, project_id, landmark_id, location_id): """Delete the link between a location and a landmark. If the last link to a location is deleted, the location is removed as well. --- parameters: - name: project_id description: Project of landmark group type: integer paramType: path required: true - name: landmark_id description: The landmark to unlink type: integer paramType: path required: true - name: location_id description: The location to unlink paramType: path type: integer required: true """ can_edit_or_fail(request.user, landmark_id, 'class_instance') landmark = ClassInstance.objects.get(project_id=project_id, pk=int(landmark_id)) pci = PointClassInstance.objects.get( project_id=project_id, class_instance=landmark, point_id=int(location_id), relation=Relation.objects.get(project_id=project_id, relation_name='annotated_with')) can_edit_or_fail(request.user, pci.id, 'point_class_instance') pci_id = pci.id pci.delete() deleted_point = False remaining_pci = PointClassInstance.objects.filter( point_id=int(location_id)) if remaining_pci.count() == 0: try: can_edit_or_fail(request.user, point.id, 'point') Point.objects.get(pk=int(location_id)).delete() deleted_point = True except: pass return Response({ 'link_id': pci_id, 'landmark_id': pci.class_instance_id, 'point_id': pci.point_id, 'deleted_point': deleted_point })
class GroupMemberships(APIView): """ Update the group membership of multiple users at once. """ @method_decorator(requires_user_role(UserRole.Admin)) def post(self, request: Request, project_id) -> Response: """ Update the group membership of multiple users at once. Users and groups as well as their memberships are global, therefore this action requires either superuser status or project tokens need to be in use. If the latter is the case, the requesting user is expected to have a) admin permission in the current project and is b) only allowed to change users and groups visible to them. """ action = request.POST.get('action') if action not in ('add', 'revoke'): raise ValueError('Action needs to be "add" or "revoke"') # Collect user and group information source_users = set( get_request_list(request.POST, 'source_users', [], int)) source_groups = set( get_request_list(request.POST, 'source_groups', [], int)) target_users = set( get_request_list(request.POST, 'target_users', [], int)) target_groups = set( get_request_list(request.POST, 'target_groups', [], int)) # Check permissions if settings.PROJECT_TOKEN_USER_VISIBILITY and not request.user.is_superuser: # Find all visible users and groups visible_user_ids = get_token_visible_users(request.user.id) visible_group_ids = get_token_visible_groups(request.user.id) invisible_user_ids = (set(source_users).union( set(target_users))).difference(set(visible_user_ids)) if invisible_user_ids: raise PermissionError( 'This request includes users beyond the allowed scope') elif not request.user.is_superuser: raise PermissionError('Need superuser permission') updated, warnings = update_group_memberships(action, source_users, source_groups, target_users, target_groups) return JsonResponse({ 'updated_users': updated, 'warnings': warnings, })
class SuppressedVirtualTreenodeDetail(APIView): def get_object(self, project_id, treenode_id, suppressed_id): try: return SuppressedVirtualTreenode.objects.get(pk=suppressed_id, project_id=project_id, child_id=treenode_id) except SuppressedVirtualTreenode.DoesNotExist: raise Http404 @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request:Request, project_id=None, treenode_id=None, suppressed_id=None, format=None) -> Response: """Unsuppress a virtual treenode. """ suppressed = self.get_object(project_id, treenode_id, suppressed_id) suppressed.delete() return Response(status=status.HTTP_204_NO_CONTENT)
class OriginCollection(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request: Request, project_id) -> Response: """List all available data sources / origins. --- parameters: - name: project_id description: Project the data sources are registered in type: integer paramType: path required: true """ datasources = DataSource.objects.filter(project_id=project_id) serializer = DataSourceSerializer(datasources, many=True) return Response(serializer.data)
class AutoproofreaderTaskAPI(APIView): @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def put(self, request, project_id): """Create an autoproofreading job. If a user has permission to queue compute tasks, this api can be used to submit a skeleton allong with sufficient information to access localized segmentations and retrieve a ranking of which sections of the neuron are most likely to contain errors. --- parameters: - name: job_config.json description: Config file containing job initialization information required: true type: file paramType: form - name: sarbor_config.toml description: File detailing sarbor execution configuration required: true type: file paramType: form - name: skeleton.csv description: Csv file containing rows of (node_id, parent_id, x, y, z) required: true type: file paramType: form - name: all_settings.toml description: File containing a full set of settings for this job required: true type: file paramType: form - name: volume.toml description: Contains configuration for Diluvian volume required: false type: file paramType: form - name: diluvian_config.toml description: Contains job specific changes to a trained models config file required: false type: file paramType: form - name: cached_lsd_config.toml description: Contains configuration for a job running on cached lsd segmentations required: false type: file paramType: form """ files = {f.name: f.read().decode("utf-8") for f in request.FILES.values()} all_settings, job_config, job_name, local_temp_dir = self._handle_files(files) settings_config = ConfigFile( user_id=request.user.id, project_id=project_id, config=all_settings ) settings_config.save() # retrieve necessary paths from the chosen server and model server = ComputeServer.objects.get(id=job_config["server_id"]) server_paths = { "address": server.address, "working_dir": server.diluvian_path[2:] if server.diluvian_path.startswith("~/") else server.diluvian_path, "results_dir": server.results_directory, "env_source": server.environment_source_path, } # Get the ssh key for the desired server ssh_key = settings.SSH_KEY_PATH + "/" + server.ssh_key ssh_user = server.ssh_user # store a job in the database now so that information about # ongoing jobs can be retrieved. # gpus = self._get_gpus(job_config) # if self._check_gpu_conflict(gpus): # raise Exception("Not enough compute resources for this job") result = AutoproofreaderResult( user_id=request.user.id, project_id=project_id, config_id=settings_config.id, skeleton_id=job_config["skeleton_id"], skeleton_csv=files["skeleton.csv"], model_id=job_config["model_id"], name=job_name, status="queued", private=True, # gpus=gpus, ) result.save() media_folder = Path(settings.MEDIA_ROOT) segmentations_dir = ( media_folder / "proofreading_segmentations" / str(result.uuid) ) msg_user(request.user.id, "autoproofreader-result-update", {"status": "queued"}) # if self._check_gpu_conflict(): # raise Exception("Not enough compute resources for this job") if job_config.get("segmentation_type", None) == "diluvian": # retrieve the configurations used during the chosen models training. # this is used as the base configuration when running since most # settings should not be changed or are irrelevant to autoproofreading a # skeleton. The settings that do need to be overridden are handled # by the config generated by the widget. model = DiluvianModel.objects.get(id=job_config["model_id"]) if model.config_id is not None: query = ConfigFile.objects.get(id=int(model.config_id)) model_config = query.config file_path = local_temp_dir / "model_config.toml" file_path.write_text(model_config) server_paths["model_file"] = model.model_source_path if job_config.get("segmentation_type", None) is not None: # Retrieve segmentation x = query_segmentation_async.delay( result, project_id, request.user.id, ssh_key, ssh_user, local_temp_dir, segmentations_dir, server_paths, job_name, job_config["segmentation_type"], ) else: raise ValueError("Segmentation type not available: {}".format(job_config)) # Send a response to let the user know the async funcion has started return JsonResponse({"task_id": x.task_id, "status": "queued"}) def _handle_files(self, files): # Check for basic files for x in [ "job_config.json", "sarbor_config.toml", "skeleton.csv", "all_settings.toml", ]: if x not in files.keys(): raise Exception(x + " is missing!") job_config = json.loads(files["job_config.json"]) all_settings = files["all_settings.toml"] # the name of the job, used for storing temporary files # and refering to past runs job_name = self._get_job_name(job_config) # If the temporary directory doesn't exist, create it media_folder = Path(settings.MEDIA_ROOT) if not (media_folder / job_name).exists(): (media_folder / job_name).mkdir() local_temp_dir = media_folder / job_name # Create a copy of the files sent in the request in the # temporary directory so that it can be copied with scp # in the async function for f in files: file_path = local_temp_dir / f file_path.write_text(files[f]) return all_settings, job_config, job_name, local_temp_dir def _get_job_name(self, config): """ Get the name of a job. If the job_name field is not provided generate a default job name based on the date and the skeleton id. """ name = config.get("job_name", "") if len(name) == 0: skid = str(config.get("skeleton_id", None)) date = str(datetime.datetime.now(pytz.utc).date()) if skid is None: raise Exception("missing skeleton id!") name = skid + "_" + date i = len(AutoproofreaderResult.objects.filter(name__startswith=name)) if i > 0: return "{}_{}".format(name, i) else: return name def _get_gpus(self, config): gpus = GPUUtilAPI._query_server(config["server_id"]) config_gpus = config.get("gpus", []) if len(config_gpus) == 0: config_gpus = list(range(len(gpus))) for g in config_gpus: if str(g) not in gpus.keys(): raise Exception( "There is no gpu with id ({}) on the chosen server".format(g) ) usage = [True if (i in config_gpus) else False for i in range(len(gpus))] return usage def _check_gpu_conflict(self, gpus=None): # returns True if there is a conflict ongoing_jobs = AutoproofreaderResult.objects.filter(status="queued") if len(ongoing_jobs) == 0: # jobs will not have taken compute resources if there # are no other jobs. We should probably still check gpu # usage stats to see if the gpus are unavailable for some # reason other than flood filling jobs. return False gpu_utils = [job.gpus for job in ongoing_jobs] if gpus is not None: gpu_utils.append(gpus) # There is a conflict if at least one gpu is claimed by at least 2 jobs return ( len(list(filter(lambda x: x > 1, map(lambda *x: sum(x), *gpu_utils)))) > 0 ) def _get_diluvian_config(self, user_id, project_id, config): """ get a configuration object for this project. It may make sense to reuse configurations accross runs, but that is currently not supported. """ return ConfigFile(user_id=user_id, project_id=project_id, config=config)
class ImageVolumeConfigAPI(APIView): @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def put(self, request, project_id): warnings = [] name = request.POST.get("name", request.data.get("name", None)) config = request.POST.get("config", request.data.get("config", None)) params = [name, config] if any([x is None for x in params]): return JsonResponse({"success": False, "results": request.data}) image_volume_config = ImageVolumeConfig(name=name, config=config, user_id=request.user.id, project_id=project_id) image_volume_config.save() return JsonResponse({ "success": True, "warnings": warnings, "image_volume_config_id": image_volume_config.id, }) @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """ List all available image volume configurations --- parameters: - name: project_id description: Project of the ImageVolumeConfigs type: integer paramType: path required: true - name: image_volume_config_id description: If available, return only the one ImageVolumeConfig type: int paramType: form required: false """ image_volume_config_id = request.query_params.get( "image_volume_config_id", request.data.get("image_volume_config_id", None)) if image_volume_config_id is not None: query_set = ImageVolumeConfig.objects.filter( id=image_volume_config_id, project=project_id) else: query_set = ImageVolumeConfig.objects.filter(project=project_id) return JsonResponse( ImageVolumeConfigSerializer(query_set, many=True).data, safe=False, json_dumps_params={ "sort_keys": True, "indent": 4 }, ) @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def delete(self, request, project_id): """ Delete an image volume configuration --- parameters: - name: project_id description: Project of the ImageVolumeConfigs type: integer paramType: path required: true - name: image_volume_config_id description: ImageVolumeConfig to delete type: int paramType: form required: true """ image_volume_config_id = request.query_params.get( "image_volume_config_id", request.data.get("image_volume_config_id", None)) image_volume = get_object_or_404(ImageVolumeConfig, id=image_volume_config_id) image_volume.delete() return JsonResponse({"success": True})
class SamplerDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request: HttpRequest, project_id, sampler_id) -> JsonResponse: """Get details on a particular sampler. --- parameters: - name: project_id description: The project to operate in. type: integer paramType: path required: false - name: sampler_id description: The sampler to return. type: integer paramType: path required: false - name: with_domains description: Optional flag to include all domains of all result sampler results. type: boolean paramType: form required: false defaultValue: false - name: with_intervals description: Optional flag to include all intervals of all domains. Implies with_domains. type: boolean paramType: form required: false default: false defaultValue: false """ sampler_id = int(sampler_id) with_intervals = get_request_bool(request.GET, 'with_intervals', False) with_domains = get_request_bool(request.GET, 'with_domains', False) or with_intervals if with_domains: sampler = Sampler.objects.prefetch_related( 'samplerdomain_set').get(pk=sampler_id) else: sampler = Sampler.objects.get(pk=sampler_id) sampler_detail = serialize_sampler(sampler) if with_domains: domains = [] domains_and_ends = SamplerDomain.objects.filter(sampler=sampler_id) \ .prefetch_related('samplerdomainend_set') if with_intervals: domains_and_ends = domains_and_ends.prefetch_related( 'samplerinterval_set') for domain in domains_and_ends: domain_data = serialize_domain(domain, with_ends=True, with_intervals=with_intervals) domains.append(domain_data) sampler_detail['domains'] = domains return JsonResponse(sampler_detail) @method_decorator(requires_user_role(UserRole.Browse)) def post(self, request: HttpRequest, project_id, sampler_id) -> JsonResponse: """Set fields of a particular sampler. --- parameters: - name: project_id description: The project to operate in. type: integer paramType: path required: false - name: sampler_id description: The sampler to return. type: integer paramType: path required: false - name: leaf_handling_mode description: Optional flag to include all domains of all result sampler results. type: boolean paramType: form required: false """ sampler_id = int(sampler_id) can_edit_or_fail(request.user, sampler_id, 'catmaid_sampler') sampler = Sampler.objects.get(pk=sampler_id) leaf_handling_mode = request.POST.get('leaf_handling_mode') if leaf_handling_mode and leaf_handling_mode in known_leaf_modes: sampler.leaf_segment_handling = leaf_handling_mode sampler.save() return JsonResponse(serialize_sampler(sampler))
class GPUUtilAPI(APIView): """ API for querying gpu status on the server. Could be used to inform user whether server is currently in use, or which gpus are currently free. """ @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): out = GPUUtilAPI._query_server( request.query_params.get("server_id", None)) return JsonResponse(out, safe=False, json_dumps_params={ "sort_keys": True, "indent": 4 }) def _query_server(server_id): fields = [ ("index", int), ("uuid", str), ("utilization.gpu", float), ("memory.total", int), ("memory.used", int), ("memory.free", int), ("driver_version", str), ("name", str), ("gpu_serial", str), ] server = ComputeServerAPI.get_servers(server_id)[0] bash_script = ( "ssh -i {} {}\n".format(settings.SSH_KEY_PATH, server["address"]) + "nvidia-smi " + "--query-gpu={} ".format(",".join([x[0] for x in fields])) + "--format=csv,noheader,nounits") process = subprocess.Popen("/bin/bash", stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding="utf8") out, err = process.communicate(bash_script) return GPUUtilAPI._parse_query(out, fields) def _parse_query(out, fields): out = out.strip() out = out.split("\n") out = list(map(lambda x: x.split(", "), out)) out = filter(lambda x: len(x) == len(fields), out) def is_valid(x, x_type): try: x_type(x) return True except ValueError: return False out = filter( lambda x: all(list(map(is_valid, x, [f[1] for f in fields]))), out) out = { x[0]: {fields[i + 1]: x[i + 1] for i in range(len(fields) - 1)} for x in out } return out
class ComputeServerAPI(APIView): @method_decorator(requires_user_role(UserRole.Admin)) def put(self, request, project_id): """ Create a new ComputeServer --- parameters: - name: address description: ssh server address. type: string paramType: form required: true - name: name description: Display name for server. type: string paramType: form - name: environment_source_path description: path to activate a virtual environment. type: path paramType: form - name: diluvian_path description: Depricated, not used type: path paramType: form - name: results_directory description: Where to store temporary data on server. type: path paramType: form - name: project_whitelist description: Projects that have access to this server. All if empty type: array paramType: form """ address = request.POST.get("address", request.data.get("address", None)) if "name" in request.POST or "name" in request.data: name = request.POST.get("name", request.data.get("name", None)) else: name = address.split(".")[0] environment_source_path = request.POST.get( "environment_source_path", request.data.get("environment_source_path", None)) diluvian_path = request.POST.get( "diluvian_path", request.data.get("diluvian_path", None)) results_directory = request.POST.get( "results_directory", request.data.get("results_directory", None)) ssh_user = request.POST.get("ssh_user", request.data.get("ssh_user", None)) ssh_key = request.POST.get("ssh_key", request.data.get("ssh_key", name)) project_whitelist = request.POST.get( "project_whitelist", request.data.get("project_whitelist", None)) server = ComputeServer( name=name, address=address, environment_source_path=environment_source_path, diluvian_path=diluvian_path, results_directory=results_directory, ssh_key=ssh_key, ssh_user=ssh_user, project_whitelist=project_whitelist, ) server.save() return JsonResponse({"success": True, "server_id": server.id}) @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """ List all available compute servers --- parameters: - name: project_id description: Project of the returned configurations type: integer paramType: path required: true - name: server_id description: If available, return only the server associated with server_id type: int paramType: form required: false """ server_id = request.query_params.get("server_id", None) if server_id is not None: query_set = ComputeServer.objects.filter( Q(id=server_id) & (Q(project_whitelist__len=0) | Q(project_whitelist__contains=[project_id]))) else: query_set = ComputeServer.objects.filter( Q(project_whitelist__len=0) | Q(project_whitelist__contains=[project_id])) return JsonResponse( ComputeServerSerializer(query_set, many=True).data, safe=False, json_dumps_params={ "sort_keys": True, "indent": 4 }, ) @method_decorator(requires_user_role(UserRole.Admin)) def delete(self, request, project_id): """ delete a compute server --- parameters: - name: project_id description: Project to delete server from. type: integer paramType: path required: true - name: server_id description: server to delete. type: int paramType: form required: false """ # can_edit_or_fail(request.user, point_id, "point") server_id = request.query_params.get( "server_id", request.data.get("server_id", None)) query_set = ComputeServer.objects.filter( Q(id=server_id) & (Q(project_whitelist__len=0) | Q(project_whitelist__contains=[project_id]))) if len(query_set) == 0: return HttpResponseNotFound() elif len(query_set) > 1: return HttpResponseNotFound() else: query_set[0].delete() return JsonResponse({"success": True})
class LandmarkGroupLocationList(APIView): @method_decorator(requires_user_role(UserRole.Annotate)) def put(self, request, project_id, landmarkgroup_id, location_id): """Link a location to a landmark group. --- parameters: - name: project_id description: Project of landmark group type: integer paramType: path required: true - name: landmarkgroup_id description: The landmark group to link type: integer paramType: path required: true - name: location_id description: Existing location ID type: integer paramType: path required: true """ point = Point.objects.get(project_id=project_id, pk=location_id) landmarkgroup = ClassInstance.objects.get( project_id=project_id, pk=landmarkgroup_id, class_column=Class.objects.get(project_id=project_id, class_name="landmarkgroup")) pci = PointClassInstance.objects.create( point=point, user=request.user, class_instance=landmarkgroup, project_id=project_id, relation=Relation.objects.get(project_id=project_id, relation_name="annotated_with")) return Response({ 'link_id': pci.id, 'point_id': point.id, 'landmarkgroup_id': landmarkgroup.id }) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request, project_id, landmarkgroup_id, location_id): """Remove the link between a location and a landmark group. --- parameters: - name: project_id description: Project of landmark group type: integer paramType: path required: true - name: landmarkgroup_id description: The landmark group to link type: integer paramType: path required: true - name: location_id description: Existing location ID type: integer paramType: path required: true """ point = Point.objects.get(project_id=project_id, pk=location_id) landmarkgroup = ClassInstance.objects.get( project_id=project_id, pk=landmarkgroup_id, class_column=Class.objects.get(project_id=project_id, class_name="landmarkgroup")) pci = PointClassInstance.objects.get( point=point, user=request.user, class_instance=landmarkgroup, project_id=project_id, relation=Relation.objects.get(project_id=project_id, relation_name="annotated_with")) can_edit_or_fail(request.user, pci.id, 'point_class_instance') pci_id = pci.id pci.delete() return Response({ 'link_id': pci_id, 'point_id': point.id, 'landmarkgroup_id': landmarkgroup.id })
class PointSetList(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request:HttpRequest, project_id) -> JsonResponse: """List all available point sets or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point sets type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: pointset_ids description: A list of point set IDs to which the query is constrained. type: array paramType: path required: false - name: order_by description: The field to order the response list by (name, id). type: string paramType: path required: false defaultValue: 'id' """ with_points = get_request_bool(request.query_params, 'with_points', False) simple = get_request_bool(request.query_params, 'simple', False) pointset_ids = get_request_list(request.query_params, 'pointset_ids', None, map_fn=int) order_by = request.query_params.get('order_by', 'id') pointsets = list_pointsets(project_id, request.user.id, simple, with_points, pointset_ids, order_by) return JsonResponse(pointsets, safe=False) @method_decorator(requires_user_role(UserRole.Browse)) def post(self, request:HttpRequest, project_id) -> JsonResponse: """List all available point sets or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point sets type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: pointset_ids description: A list of point set IDs to which the query is constrained. type: array paramType: path required: false - name: order_by description: The field to order the response list by (name, id). type: string paramType: path required: false defaultValue: 'id' """ with_points = get_request_bool(request.POST, 'with_points', False) simple = get_request_bool(request.POST, 'simple', False) pointset_ids = get_request_list(request.POST, 'pointset_ids', None, map_fn=int) order_by = request.query_params.get('order_by', 'id') pointsets = list_pointsets(project_id, request.user.id, simple, with_points, pointset_ids, order_by) return JsonResponse(pointsets, safe=False)
class LandmarkGroupImport(APIView): @method_decorator(requires_user_role(UserRole.Annotate)) def post(self, request, project_id): """Import and link landmarks, landmark groups and locations. The passed in <data> parameter is a list of two-element lists, each representing a group along with its linked landmark and locations. The group is represented by its name and the members are a list of four-element lists, containing the landmark name and the location. This results in the following format: [[group_1_name, [[landmark_1_name, x, y, z], [landmark_2_name, x, y, z]]], ...] Note that this parameter has to be transmitted as a JSON encoded string. --- parameters: - name: project_id description: The project the landmark group is part of. type: integer paramType: path required: true - name: data description: The data to import. required: true type: string paramType: form - name: reuse_existing_groups description: Whether existing groups should be reused. type: boolean paramType: form defaultValue: false required: false - name: reuse_existing_landmarks description: Whether existing landmarks should be reused. type: boolean paramType: form defaultValue: false required: false - name: create_non_existing_groups description: Whether non-existing groups should be created. type: boolean paramType: form defaultValue: true required: false - name: create_non_existing_landmarks description: Whether non-existing landmarks should be created. type: boolean paramType: form defaultValue: true required: false """ project_id = int(project_id) if not project_id: raise ValueError("Need project ID") reuse_existing_groups = request.data.get('reuse_existing_groups', 'false') == 'true' reuse_existing_landmarks = request.data.get('reuse_existing_landmarks', 'false') == 'true' create_non_existing_groups = request.data.get( 'create_non_existing_groups', 'true') == 'true' create_non_existing_landmarks = request.data.get( 'create_non_existing_landmarks', 'true') == 'true' # Make sure the data to import matches our expectations data = request.data.get('data') if not data: raise ValueError("Need data to import") data = json.loads(data) for n, (group_name, landmarks) in enumerate(data): if not group_name: raise ValueError("The {}. group doesn't have a name".format(n)) if not landmarks: raise ValueError( "Group {} doesn't contain any landmarks".format( group_name)) for m, link in enumerate(landmarks): if not link or len(link) != 4: raise ValueError("The {}. link of the {}. group ({}) " \ "doesn't conform to the [ID, X, Y, Z] format.".format(m, n, group_name)) for ci in (1, 2, 3): coordinate = link[ci] value = float(coordinate) if math.isnan(value): raise ValueError("The {}. link of the {}. group ({}) " \ "doesn't have a valid {}. coordinate: {}.".format( m, n, group_name, ci, coordinate)) link[ci] = value classes = get_class_to_id_map(project_id) relations = get_relation_to_id_map(project_id) landmark_class = classes['landmark'] landmarkgroup_class = classes['landmarkgroup'] part_of_relation = relations['part_of'] annotated_with_relation = relations['annotated_with'] landmarks = dict( (k.lower(), v) for k, v in ClassInstance.objects.filter( project_id=project_id, class_column=landmark_class).values_list('name', 'id')) landmarkgroups = dict( (k.lower(), v) for k, v in ClassInstance.objects.filter( project_id=project_id, class_column=landmarkgroup_class).values_list('name', 'id')) imported_groups = [] # Keep track of which landmarks have been seen and were accepted. seen_landmarks = set() for n, (group_name, linked_landmarks) in enumerate(data): # Test if group exists already and raise error if they do and are # prevented from being reused (option). existing_group_id = landmarkgroups.get(group_name.lower()) if existing_group_id: if n == 0: if not reuse_existing_groups: raise ValueError("Group \"{}\" exists already ({}). Please" \ "remove it or enable group re-use.".format( group_name, existing_group_id)) can_edit_or_fail(request.user, existing_group_id, 'class_instance') elif create_non_existing_groups: group = ClassInstance.objects.create( project_id=project_id, class_column_id=landmarkgroup_class, user=request.user, name=group_name) existing_group_id = group.id landmarkgroups[group_name.lower()] = group.id else: raise ValueError("Group \"{}\" does not exist. Please create " \ "it or enable automatic creation/".format(group_name)) imported_landmarks = [] imported_group = { 'id': existing_group_id, 'name': group_name, 'members': imported_landmarks } imported_groups.append(imported_group) for m, link in enumerate(linked_landmarks): landmark_name = link[0] x, y, z = link[1], link[2], link[3] existing_landmark_id = landmarks.get(landmark_name.lower()) if existing_landmark_id: # Test only on first look at landmark if existing_landmark_id not in seen_landmarks: if not reuse_existing_landmarks: raise ValueError("Landmark \"{}\" exists already. " \ "Please remove it or enable re-use of " \ "existing landmarks.".format(landmark_name)) can_edit_or_fail(request.user, existing_landmark_id, 'class_instance') elif create_non_existing_landmarks: landmark = ClassInstance.objects.create( project_id=project_id, class_column_id=landmark_class, user=request.user, name=landmark_name) existing_landmark_id = landmark.id landmarks[landmark_name.lower()] = landmark.id else: raise ValueError("Landmark \"{}\" does not exist. Please " \ "create it or enable automatic creation.".format( landmark_name)) seen_landmarks.add(existing_landmark_id) # Make sure the landmark is linked to the group landmark_link = ClassInstanceClassInstance.objects.get_or_create( project_id=project_id, relation_id=part_of_relation, class_instance_a_id=existing_landmark_id, class_instance_b_id=existing_group_id, defaults={'user': request.user}) # With an existing group and landmark in place, the location can # be linked (to both). point = Point.objects.create(project_id=project_id, user=request.user, editor=request.user, location_x=x, location_y=y, location_z=z) point_landmark = PointClassInstance.objects.create( point=point, user=request.user, class_instance_id=existing_landmark_id, project_id=project_id, relation_id=annotated_with_relation) point_landmark_group = PointClassInstance.objects.create( point=point, user=request.user, class_instance_id=existing_group_id, project_id=project_id, relation_id=annotated_with_relation) imported_landmarks.append({ 'id': existing_landmark_id, 'name': landmark_name, 'x': x, 'y': y, 'z': z }) return Response(imported_groups)
class PointCloudList(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request: HttpRequest, project_id) -> JsonResponse: """List all available point clouds or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point clouds type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_images description: Wheter linked images should returned as well. type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: sample_ratio description: Number in [0,1] to optionally sample point cloud type: number paramType: form required: false - name: pointcloud_ids description: A list of point cloud IDs to which the query is constrained. type: array paramType: path required: false - name: order_by description: The field to order the response list by (name, id). type: string paramType: path required: false defaultValue: 'id' """ with_images = get_request_bool(request.query_params, 'with_images', False) with_points = get_request_bool(request.query_params, 'with_points', False) sample_ratio = float(request.query_params.get('sample_ratio', '1.0')) simple = get_request_bool(request.query_params, 'simple', False) pointcloud_ids = get_request_list(request.query_params, 'pointcloud_ids', None, map_fn=int) order_by = request.query_params.get('order_by', 'id') pointclouds = list_pointclouds(project_id, request.user.id, simple, with_images, with_points, sample_ratio, pointcloud_ids, order_by) return JsonResponse(pointclouds, safe=False) @method_decorator(requires_user_role(UserRole.Browse)) def post(self, request: HttpRequest, project_id) -> JsonResponse: """List all available point clouds or optionally a sub set. --- parameters: - name: project_id description: Project of the returned point clouds type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_images description: Wheter linked images should returned as well. type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false - name: sample_ratio description: Number in [0,1] to optionally sample point cloud type: number paramType: form required: false - name: pointcloud_ids description: A list of point cloud IDs to which the query is constrained. type: array paramType: path required: false - name: order_by description: The field to order the response list by (name, id). type: string paramType: path required: false defaultValue: 'id' """ with_images = get_request_bool(request.POST, 'with_images', False) with_points = get_request_bool(request.POST, 'with_points', False) sample_ratio = float(request.POST.get('sample_ratio', '1.0')) simple = get_request_bool(request.POST, 'simple', False) pointcloud_ids = get_request_list(request.POST, 'pointcloud_ids', None, map_fn=int) order_by = request.query_params.get('order_by', 'id') pointclouds = list_pointclouds(project_id, request.user.id, simple, with_images, with_points, sample_ratio, pointcloud_ids, order_by) return JsonResponse(pointclouds, safe=False) @method_decorator(requires_user_role(UserRole.Annotate)) def put(self, request: HttpRequest, project_id) -> JsonResponse: """Create a new pointcloud by providing. --- parameters: - name: project_id description: Project of the new point cloud type: integer paramType: path required: true - name: name description: Name of the new point cloud type: string paramType: form required: true - name: description description: Description of the new point cloud type: string paramType: form required: false - name: points description: Points of point cloud in project space. Can be a stringified JSON array. type: array paramType: form required: true - name: group_id description: A group for which this point cloud will be visible exclusivly. type: integer paramType: form required: false """ name = request.POST.get('name') if not name: raise ValueError("Need name") description = request.POST.get('description', '') source_path = request.POST.get('source_path', '') # Create new Point instances for each import location and link it to the # point cloud. if 'points' in request.POST: points = json.loads(request.POST['points']) else: points = get_request_list(request.POST, 'points') # map_fn=lambda x: [float(x[0]), float(x[1]), float(x[2])]) if not points: raise ValueError("Need points to create point cloud") pc = PointCloud.objects.create(project_id=project_id, name=name, description=description, user=request.user, source_path=source_path) pc.save() image_names = get_request_list(request.POST, 'image_names') image_descriptions = get_request_list(request.POST, 'image_descriptions') n_images = len(request.FILES) if image_names and len(image_names) != n_images: raise ValueError( f"If image names are passed in, there must be exactly as many as passed in files ({n_images})" ) if image_descriptions and len(image_descriptions) != n_images: raise ValueError( f"If image descriptions are passed in, there must be exactly as many as passed in files ({n_images})" ) cursor = connection.cursor() for n, image in enumerate(request.FILES.values()): if image.size > settings.IMPORTED_IMAGE_FILE_MAXIMUM_SIZE: raise ValueError( f"Image {image.name} is bigger than IMPORTED_IMAGE_FILE_MAXIMUM_SIZE ({settings.IMPORTED_IMAGE_FILE_MAXIMUM_SIZE / 1024**2} MB)" ) # Transform into JPEG img = Image.open(image) image_io = BytesIO() img.save(image_io, format='JPEG') cursor.execute( """ INSERT INTO image_data(user_id, project_id, name, description, source_path, content_type, image) VALUES (%(user_id)s, %(project_id)s, %(name)s, %(description)s, %(source_path)s, %(content_type)s, %(image)s) RETURNING id; """, { 'user_id': request.user.id, 'project_id': project_id, 'name': image.name, 'description': image_descriptions[n], 'source_path': image.name, 'content_type': 'image/jpeg', 'image': image_io.getvalue() }) image_data_id = cursor.fetchone()[0] pcid = PointCloudImageData( **{ 'project_id': project_id, 'pointcloud_id': pc.id, 'image_data_id': image_data_id, }) pcid.save() # Find an optional restriction group permission. If a group has no # permission assigned, it is considered readable by all. group_id = request.POST.get('group_id') if group_id is not None: group_id = int(group_id) group = Group.objects.get(pk=group_id) assigned_perm = assign_perm('can_read', group, pc) # Add points cursor = connection.cursor() cursor.execute( """ WITH added_point AS ( INSERT INTO point (project_id, user_id, editor_id, location_x, location_y, location_z) SELECT %(project_id)s, %(user_id)s, %(editor_id)s, p.location[1], p.location[2], p.location[3] FROM reduce_dim(%(points)s) p(location) RETURNING id ) INSERT INTO pointcloud_point (project_id, pointcloud_id, point_id) SELECT %(project_id)s, %(pointcloud_id)s, ap.id FROM added_point ap """, { "project_id": project_id, "user_id": request.user.id, "editor_id": request.user.id, "pointcloud_id": pc.id, "points": points, }) # If images are provided, store them in the database and link them to the # point cloud. images = get_request_list(request.POST, 'images') return JsonResponse(serialize_pointcloud(pc))
class LandmarkList(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """List available landmarks, optionally only the ones in a set of landmark groups. --- parameters: - name: project_id description: Project of landmark type: integer paramType: path required: true - name: with_locations description: Whether to return linked locations required: false defaultValue: false paramType: form """ with_locations = request.query_params.get('with_locations', 'false') == 'true' landmark_class = Class.objects.get(project_id=project_id, class_name="landmark") landmarks = ClassInstance.objects.filter( project_id=project_id, class_column=landmark_class).order_by('id') serializer = BasicClassInstanceSerializer(landmarks, many=True) serialized_landmarks = serializer.data if with_locations and serialized_landmarks: # A landmark class instance's linked locations are points using the # "annotated_with" relation. landmark_ids = [lm['id'] for lm in serialized_landmarks] landmark_template = ",".join("(%s)" for _ in landmark_ids) cursor = connection.cursor() cursor.execute( """ SELECT landmark.id, p.id, p.location_x, p.location_y, p.location_z FROM point_class_instance pci JOIN point p ON pci.point_id = p.id JOIN (VALUES {}) landmark(id) ON pci.class_instance_id = landmark.id WHERE pci.relation_id = ( SELECT id FROM relation WHERE relation_name = 'annotated_with' AND project_id = %s ) AND pci.project_id = %s """.format(landmark_template), landmark_ids + [project_id, project_id]) point_index = defaultdict(list) for point in cursor.fetchall(): point_index[point[0]].append({ 'id': point[1], 'x': point[2], 'y': point[3], 'z': point[4] }) # Append landmark locations to landmarks for lm in serialized_landmarks: lm['locations'] = point_index[lm['id']] return Response(serialized_landmarks) @method_decorator(requires_user_role(UserRole.Annotate)) def put(self, request, project_id): """Add a new landmark. Expect at least the name as parameter. --- parameters: - name: project_id description: Project of landmark type: integer paramType: path required: true - name: name description: Name of new landmark type: string required: true """ name = request.data.get('name') landmark_class = Class.objects.get(project_id=project_id, class_name='landmark') # Prevent creation of duplicate landmarks existing_landmarks = ClassInstance.objects.filter( project_id=project_id, name=name, class_column=landmark_class) if existing_landmarks: raise ValueError( "There is already a landmark with name {}".format(name)) landmark = ClassInstance.objects.create(project_id=project_id, class_column=landmark_class, user=request.user, name=name) landmark.save() serializer = BasicClassInstanceSerializer(landmark) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request, project_id): """Delete a list of landmarks including the linked locations, if they are not used by other landmarks. --- parameters: - name: project_id description: The project the landmark is part of. type: integer paramType: path required: true - name: landmark_ids description: The landmarks to remove. required: true type: integer paramType: form - name: keep_points description: Don't delete points. required: false type: boolean defaultValue: false paramType: form """ keep_points = request.query_params.get('keep_points', 'false') == 'true' landmark_ids = get_request_list(request.query_params, 'landmark_ids', map_fn=int) for l in landmark_ids: can_edit_or_fail(request.user, l, 'class_instance') annotated_with_relation = Relation.objects.get( project_id=project_id, relation_name='annotated_with') point_ids = set() if not keep_points: point_landmark_links = PointClassInstance.objects.filter( project_id=project_id, class_instance_id__in=landmark_ids, relation=annotated_with_relation) # These are the landmark's lined points point_ids = set(pll.point_id for pll in point_landmark_links) landmark_class = Class.objects.get(project_id=project_id, class_name="landmark") landmarks = ClassInstance.objects.filter(pk__in=landmark_ids, project_id=project_id, class_column=landmark_class) if len(landmark_ids) != len(landmarks): raise ValueError("Could not find all landmark IDs") landmarks.delete() if not keep_points: remaining_pll = set( PointClassInstance.objects.filter( project_id=project_id, point_id__in=point_ids, relation=annotated_with_relation).values_list('point_id', flat=True)) points_to_delete = point_ids - remaining_pll Point.objects.filter(project_id=project_id, pk__in=points_to_delete).delete() serializer = BasicClassInstanceSerializer(landmarks, many=True) return Response(serializer.data)
class DiluvianModelAPI(APIView): @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def put(self, request, project_id): """ Add a diluvian model --- parameters: - name: project_id description: Project for which the model will be available. type: integer paramType: path required: true - name: name description: Name of the new model. type: string paramType: form required: true - name: server_id description: | The server on which this model was trained. Holds the model weights. type: integer paramType: form required: true - name: model_source_path description: | File path of trained model on the server it was trained on. Note that if you want to use this model on a different server, the model files must be stored under the same source path. required: true paramType: form - name: config description: This models diluvian config. required: true paramType: form """ warnings = [] name = request.POST.get("name", request.data.get("name", None)) server_id = request.POST.get("server_id", request.data.get("server_id", None)) model_source_path = request.POST.get( "model_source_path", request.data.get("model_source_path", None)) config = request.POST.get("config", request.data.get("config", None)) params = [name, server_id, model_source_path] if any([x is None for x in params]): return JsonResponse({"success": False, "results": request.POST}) if config is not None: model_config = ConfigFile(user_id=request.user.id, project_id=project_id, config=config) model_config.save() config_id = model_config.id else: warnings.append( "Model created with no configuration files. This " + "will make it much harder to reproduce your " + "results later.") config_id = None model = DiluvianModel( name=name, server_id=server_id, model_source_path=model_source_path, config_id=config_id, user_id=request.user.id, project_id=project_id, ) model.save() return JsonResponse({ "success": True, "warnings": warnings, "model_id": model.id }) @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def get(self, request, project_id): """ List all available diluvian models --- parameters: - name: project_id description: Project of the queried models type: integer paramType: path required: true - name: model_id description: If available, return only the model associated with model_id type: int paramType: form required: false """ model_id = request.query_params.get("model_id", request.data.get("model_id", None)) if model_id is not None: query_set = DiluvianModel.objects.filter(id=model_id, project=project_id) else: query_set = DiluvianModel.objects.filter(project=project_id) return JsonResponse( DiluvianModelSerializer(query_set, many=True).data, safe=False, json_dumps_params={ "sort_keys": True, "indent": 4 }, ) @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def delete(self, request, project_id): """ delete a diluvian model --- parameters: - name: project_id description: Project of the queried models type: integer paramType: path required: true - name: model_id description: model to delete type: int paramType: form required: true """ # can_edit_or_fail(request.user, point_id, "point") model_id = request.query_params.get("model_id", request.data.get("model_id", None)) model = get_object_or_404(DiluvianModel, id=model_id) model.delete() return JsonResponse({"success": True})
class PointDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(request, project_id, point_id): """Return details on one particular point. --- parameters: - name: project_id description: Project point is part of type: integer paramType: path required: true - name: point_id description: ID of point type: integer paramType: path required: true """ point = get_object_or_404(Point, pk=point_id, project_id=project_id) serializer = PointSerializer(point) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def post(request, project_id, point_id): """Update one particular point. Requires at least one field to change. --- parameters: - name: project_id description: Project point is part of type: integer paramType: path required: true - name: point_id description: ID of point type: integer paramType: path required: true - name: location_x description: X coordinate type: float paramType: form required: false - name: location_y description: Y coordinate type: float paramType: form required: false - name: location_z description: Z coordinate type: float paramType: form required: false - name: radius description: Optional radius type: float paramType: form required: false - name: confidence description: Optional confidence in [0,5] type: integer paramType: form required: false """ can_edit_or_fail(request.user, point_id, 'point') updated_fields = {} if request.POST.has('x'): updated_fields['location_x'] = float(request.POST.get('x')) if request.POST.has('y'): updated_fields['location_y'] = float(request.POST.get('y')) if request.POST.has('z'): updated_fields['location_z'] = float(request.POST.get('z')) if request.POST.has('radius'): updated_fields['radius'] = float(request.POST.get('radius')) if request.POST.has('confidence'): confidence = max(min(int(request.POST.get('confidence')), 5), 0) updated_fields('confidence', confidence) if not updated_fields: raise ValueError('No field to modify provided') point = get_object_or_404(Point, pk=point_id, project_id=project_id) point.update(**updated_fields) point.save() serializer = PointSerializer(point) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(request, project_id, point_id): """Delete one particular point. --- parameters: - name: project_id description: Project point is part of type: integer paramType: path required: true - name: point_id description: ID of point type: integer paramType: path required: true """ can_edit_or_fail(request.user, point_id, 'point') point = get_object_or_404(Point, pk=point_id, project_id=project_id) point.delete() point.id = None serializer = PointSerializer(point) return Response(serializer.data)
class PointList(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """List points, optionally constrained by various properties. --- parameters: - name: project_id description: Project of points type: integer paramType: path required: true """ points = Point.objects.all() serializer = PointSerializer(points, many=True) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def put(request, project_id): """Add a new point. Expect at least the location as parameters. --- parameters: - name: project_id description: Project of points type: integer paramType: path required: true - name: location_x description: X coordinate type: float paramType: form required: true - name: location_y description: Y coordinate type: float paramType: form required: true - name: location_z description: Z coordinate type: float paramType: form required: true - name: radius description: Optional radius type: float paramType: form required: false - name: confidence description: Optional confidence in [0,5] type: integer paramType: form required: false """ location_x = float(request.POST.get('x')) location_y = float(request.POST.get('y')) location_z = float(request.POST.get('z')) radius = float(request.POST.get('radius'), 0) confidence = min(max(int(request.POST.get('confidence'), 0), 0), 5) point = Point.objects.create(project_id=project_id, user=request.user, editor=request.user, location_x=location_x, location_y=location_y, location_z=location_z, radius=radius, confidence=confidence) point.save() serializer = PointSerializer(point) return Response(serializer.data)
class AutoproofreaderResultAPI(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """Retrieve past job results. Retrieve information on previous jobs. This includes jobs that have not yet completed their computations. --- parameters: - name: result_id description: ID of result to retrieve. If not provided retrieve all results type: integer paramType: path """ result_id = request.query_params.get( "result_id", request.data.get("result_id", None) ) if result_id is not None: query_set = AutoproofreaderResult.objects.filter( Q(project=project_id) & Q(id=result_id) & (Q(user=request.user.id) | Q(private=False)) ) if len(query_set) == 0: return HttpResponseNotFound( "No results found with id {}".format(result_id) ) else: query_set = AutoproofreaderResult.objects.filter( Q(project=project_id) & (Q(user=request.user.id) | Q(private=False)) ) if len(query_set) == 0 and result_id is not None: return JsonResponse([], safe=False) return JsonResponse( AutoproofreaderResultSerializer(query_set, many=True).data, safe=False, json_dumps_params={"sort_keys": True, "indent": 4}, ) @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def patch(self, request, project_id): """Edit an existing result. This api is used to toggle the 'permanent' and 'private' flags of a result. --- parameters: - name: result_id description: ID of result to edit. required: true type: integer paramType: form -name: private description: | Whether to toggle the 'private' flag. If checked only the user who started this job can view its results. type: boolean paramType: form -name: permanent description: | Whether to toggle the 'permanent' flag. If not checked, this result and its data might be deleted to make room for others. type: boolean paramType: form """ result_id = request.query_params.get( "result_id", request.data.get("result_id", None) ) if request.query_params.get("private", request.data.get("private", False)): # toggle privacy setting if result belongs to this user. result = get_object_or_404( AutoproofreaderResult, id=result_id, user=request.user.id, project=project_id, ) result.private = not result.private result.save() if request.query_params.get("permanent", request.data.get("permanent", False)): # toggle permanent setting if result belongs to user or is not private query_set = AutoproofreaderResult.objects.filter( Q(id=result_id) & Q(project=project_id) & (Q(user=request.user.id) | Q(private=False)) ) if len(query_set) == 0: return HttpResponseNotFound() if len(query_set) > 1: raise ValueError("non unique ids found") result = query_set[0] result.permanent = not result.permanent result.save() return JsonResponse({"private": result.private, "permanent": result.permanent}) @method_decorator(requires_user_role(UserRole.QueueComputeTask)) def delete(self, request, project_id): """Delete an existing result. --- parameters: - name: result_id description: ID of result to delete. required: true type: integer paramType: form """ # can_edit_or_fail(request.user, point_id, "point") result_id = request.query_params.get( "result_id", request.data.get("result_id", None) ) result = get_object_or_404( AutoproofreaderResult, id=result_id, user_id=request.user.id, project=project_id, ) result.delete() return JsonResponse({"success": True})
class LandmarkGroupDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id, landmarkgroup_id): """Get details on one particular landmarkgroup group, including its members. --- parameters: - name: project_id description: The project the landmark group is part of. type: integer paramType: path required: true - name: landmarkgroup_id description: The ID of the landmark group. required: true type: integer paramType: path - name: with_members description: Whether to return group members type: boolean paramType: form defaultValue: false required: false - name: with_locations description: Whether to return linked locations required: false defaultValue: false paramType: form """ landmarkgroup_id = int(landmarkgroup_id) with_members = request.query_params.get('with_members', 'false') == 'true' with_locations = request.query_params.get('with_locations', 'false') == 'true' landmarkgroup_class = Class.objects.get(project_id=project_id, class_name='landmarkgroup') landmarkgroup = get_object_or_404(ClassInstance, pk=landmarkgroup_id, project_id=project_id, class_column=landmarkgroup_class) serializer = BasicClassInstanceSerializer(landmarkgroup) data = serializer.data if data: if with_members: # Get member information member_index = get_landmark_group_members( project_id, [landmarkgroup_id]) # Append member information data['members'] = member_index[landmarkgroup_id] if with_locations: # Get linked locations, which represent instances of # landmark in this landmark group. location_index = get_landmark_group_locations( project_id, [landmarkgroup_id]) # Append location information data['locations'] = location_index[landmarkgroup_id] return Response(data) @method_decorator(requires_user_role(UserRole.Annotate)) def post(self, request, project_id, landmarkgroup_id): """Update an existing landmark group. Currently, only the name and group members can be updated. Edit permissions are only needed when removing group members. --- parameters: - name: project_id description: The project the landmark group is part of. type: integer paramType: path required: true - name: landmark_id description: The ID of the landmark group. required: true type: integer paramType: path - name: name description: The new name of the landmark group. required: false type: string paramType: form - name: members description: The new members of the landmark group. required: false type: array items: type: integer paramType: form """ needs_edit_permissions = False project_id = int(project_id) if not project_id: raise ValueError("Need project ID") landmarkgroup_id = int(landmarkgroup_id) if not landmarkgroup_id: raise ValueError("Need landmark group ID") name = request.data.get('name') if request.data.get('members') == 'none': members = [] else: members = get_request_list(request.data, 'members', map_fn=int) if not name and members == None: raise ValueError('Need name or members parameter for update') landmarkgroup_class = Class.objects.get(project_id=project_id, class_name='landmarkgroup') landmarkgroup = get_object_or_404(ClassInstance, pk=landmarkgroup_id, project_id=project_id, class_column=landmarkgroup_class) if name: landmarkgroup.name = name landmarkgroup.save() if members is not None: # Find out which members need to be added and which existing ones # need to be removed. current_members = set( get_landmark_group_members(project_id, [landmarkgroup_id]).get( landmarkgroup_id, [])) new_members = set(members) to_add = new_members - current_members to_remove = current_members - new_members if to_remove: needs_edit_permissions = True part_of = Relation.objects.get(project_id=project_id, relation_name='part_of') ClassInstanceClassInstance.objects.filter( project_id=project_id, class_instance_a__in=to_remove, class_instance_b_id=landmarkgroup_id, relation=part_of).delete() for landmark_id in to_add: ClassInstanceClassInstance.objects.create( project_id=project_id, class_instance_a_id=landmark_id, class_instance_b_id=landmarkgroup_id, relation=part_of, user=request.user) if needs_edit_permissions: can_edit_or_fail(request.user, landmarkgroup_id, 'class_instance') serializer = BasicClassInstanceSerializer(landmarkgroup) return Response(serializer.data) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request, project_id, landmarkgroup_id): """Delete one particular landmark group. --- parameters: - name: project_id description: The project the landmark group is part of. type: integer paramType: path required: true - name: landmarkgroup_id description: The ID of the landmark group to delete. required: true type: integer paramType: path """ can_edit_or_fail(request.user, landmarkgroup_id, 'class_instance') landmarkgroup_class = Class.objects.get(project_id=project_id, class_name='landmarkgroup') landmarkgroup = get_object_or_404(ClassInstance, pk=landmarkgroup_id, project_id=project_id, class_column=landmarkgroup_class) landmarkgroup.delete() landmarkgroup.id = None serializer = BasicClassInstanceSerializer(landmarkgroup) return Response(serializer.data)
class LandmarkLocationList(APIView): @method_decorator(requires_user_role(UserRole.Annotate)) def put(self, request, project_id, landmark_id): """Add a new location or use an existing one and link it to a landmark. Either (x,y,z) or location_id have to be provided. --- parameters: - name: project_id description: Project of landmark group type: integer paramType: path required: true - name: landmark_id description: The landmark to link type: integer paramType: path required: true - name: location_id description: Optional existing location ID type: integer required: false - name: x description: Optional new location X coodinate type: float required: false - name: y description: Optional new location Y coodinate type: float required: false - name: z description: Optional new location Z coodinate type: float required: false """ location_id = request.data.get('location_id') x = float(request.data.get('x')) y = float(request.data.get('y')) z = float(request.data.get('z')) if location_id and (x or y or z): raise ValueError( "Please provide either location ID or coordinates") landmark = ClassInstance.objects.get(project_id=project_id, pk=int(landmark_id)) if location_id: point = Point.objects.get(project_id=project_id, pk=location_id) else: # Create new point point = Point.objects.create(project_id=project_id, user=request.user, editor=request.user, location_x=x, location_y=y, location_z=z) pci = PointClassInstance.objects.create( point=point, user=request.user, class_instance=landmark, project_id=project_id, relation=Relation.objects.get(project_id=project_id, relation_name="annotated_with")) return Response({ 'link_id': pci.id, 'point_id': point.id, 'landmark_id': landmark.id })
class PointCloudDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request: HttpRequest, project_id, pointcloud_id) -> JsonResponse: """Return a point cloud. parameters: - name: project_id description: Project of the returned point cloud type: integer paramType: path required: true - name: simple description: Wheter or not only ID and name should be returned type: bool paramType: form required: false defaultValue: false - name: with_images description: Wheter linked images should returned as well. type: bool paramType: form required: false defaultValue: false - name: with_points description: Wheter linked points should returned as well. type: bool paramType: form required: false defaultValue: false """ with_images = get_request_bool(request.query_params, 'with_images', False) with_points = get_request_bool(request.query_params, 'with_points', False) sample_ratio = float(request.query_params.get('sample_ratio', '1.0')) simple = get_request_bool(request.query_params, 'simple', False) pointcloud = PointCloud.objects.get(pk=pointcloud_id, project_id=project_id) pointcloud_data = serialize_pointcloud(pointcloud, simple) # Check permissions. If there are no read permission assigned at all, # everyone can read. if 'can_read' not in get_perms(request.user, pointcloud) and \ len(get_users_with_perms(pointcloud)) > 0: raise PermissionError( f'User "{request.user.username}" not allowed to read point cloud #{pointcloud.id}' ) if with_images: images = [serialize_image_data(i) for i in pointcloud.images.all()] pointcloud_data['images'] = images if with_points: if sample_ratio == 1.0: points = [ serialize_point(p, compact=True) for p in pointcloud.points.all() ] pointcloud_data['points'] = points else: n_points = PointCloudPoint.objects.filter( pointcloud_id=pointcloud.id).count() n_sample = int(n_points * sample_ratio) cursor = connection.cursor() # Select a random sample of N points in a repeatable fashion. cursor.execute( """ SELECT setseed(0); SELECT id, location_x, location_y, location_z FROM point p JOIN ( SELECT pcp.point_id FROM pointcloud_point pcp WHERE pcp.pointcloud_id = %(pointcloud_id)s ORDER BY random() ) ordered_points(id) USING(id) LIMIT %(n_sample)s """, { 'pointcloud_id': pointcloud.id, 'n_sample': n_sample }) pointcloud_data['points'] = cursor.fetchall() return JsonResponse(pointcloud_data) @method_decorator(requires_user_role(UserRole.Annotate)) def delete(self, request: HttpRequest, project_id, pointcloud_id) -> JsonResponse: """Delete a point cloud. """ can_edit_or_fail(request.user, pointcloud_id, 'pointcloud') pointcloud = PointCloud.objects.get(pk=pointcloud_id, project_id=project_id) cursor = connection.cursor() cursor.execute( """ DELETE FROM pointcloud CASCADE WHERE project_id=%s AND id = %s """, [project_id, pointcloud_id]) return JsonResponse({'deleted': True, 'pointcloud_id': pointcloud.id})
class ServerStats(APIView): @method_decorator(requires_user_role(UserRole.Admin)) def get(self, request:Request, project_id) -> Response: """Return an object that represents the state of various server and database objects. """ return Response({ 'time': self.get_current_timestamp(), 'server': self.get_server_stats(), 'database': self.get_database_stats(), }) def get_current_timestamp(self) -> str: return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") def get_server_stats(self) -> Dict[str, Any]: return { 'load_avg': os.getloadavg(), } def get_database_stats(self) -> Dict[str, Any]: cursor = connection.cursor() cursor.execute("select current_database()") db_name = cursor.fetchone()[0] cursor.execute("SELECT version()") db_version = cursor.fetchone()[0] cursor.execute(""" SELECT (xact_commit * 100) / (xact_commit + xact_rollback), deadlocks, conflicts, temp_files, pg_size_pretty(temp_bytes), blks_read, blks_hit FROM pg_stat_database WHERE datname = %(db_name)s """, { 'db_name': db_name, }) db_stats = cursor.fetchone(); cursor.execute(""" SELECT sum(heap_blks_read) AS heap_read, sum(heap_blks_hit) AS heap_hit, sum(heap_blks_hit)/ (sum(heap_blks_hit) + sum(heap_blks_read)) AS ratio FROM pg_statio_user_tables """) db_cache = cursor.fetchone() cursor.execute(""" SELECT sum(idx_blks_read) AS idx_read, sum(idx_blks_hit) AS idx_hit, (sum(idx_blks_hit) - sum(idx_blks_read)) / sum(idx_blks_hit) AS ratio FROM pg_statio_user_indexes """) db_idx_cache = cursor.fetchone() cursor.execute(""" SELECT checkpoints_timed, checkpoints_req, buffers_clean, maxwritten_clean, buffers_backend_fsync, extract(epoch from now() - pg_last_xact_replay_timestamp()) FROM pg_stat_bgwriter """) bgwriter_stats = cursor.fetchone(); return { 'version': db_version, # Should be above 95% 'c_ratio': db_stats[0], # Should be < 10 'deadlocks': db_stats[1], # Should be < 10 'conflicts': db_stats[2], # Should be < 100 'temp_files': db_stats[3], # Should be < 10 GB 'temp_size': db_stats[4], # blks_hit/blks_read Should be > 90% 'blks_read': db_stats[5], 'blks_hit': db_stats[6], 'cache_hit_ratio': db_stats[6]/(db_stats[5]+db_stats[6]), # user table hit/blks ratio should be > 90% 'user_blks_read': db_cache[0], 'user_blks_hit': db_cache[1], 'user_cache_hit_ratio': db_cache[1]/(db_cache[0]+db_cache[1]), # user table hit/blks ratio should be > 90% 'idx_blks_read': db_idx_cache[0], 'idx_blks_hit': db_idx_cache[1], 'idx_cache_hit_ratio': db_idx_cache[1]/(db_idx_cache[0]+db_idx_cache[1]), # Should be checkpoints_req < checkpoints_timed 'checkpoints_req': bgwriter_stats[0], 'checkpoints_timed': bgwriter_stats[1], # Should be high 'buffers_clean': bgwriter_stats[2], # Should be 0 'maxwritten_clean': bgwriter_stats[3], # Should be 0 'buffers_backend_fsync': bgwriter_stats[4], # Should be close to 0 or 0 'replication_lag': bgwriter_stats[5], }
class PointCloudImageDetail(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request: HttpRequest, project_id, pointcloud_id, image_id) -> HttpResponse: """Return a point cloud. parameters: - name: project_id description: Project of the returned point cloud image type: integer paramType: path required: true - name: pointcloud_id description: Point cloud this image is linked to type: integer paramType: path required: true - name: image_id description: The image to load type: integer paramType: path required: true """ pointcloud = PointCloud.objects.get(pk=pointcloud_id, project_id=project_id) # Check permissions. If there are no read permission assigned at all, # everyone can read. if 'can_read' not in get_perms(request.user, pointcloud) and \ len(get_users_with_perms(pointcloud)) > 0: raise PermissionError( f'User "{request.user.username}" not allowed to read point cloud #{pointcloud.id}' ) cursor = connection.cursor() cursor.execute( """ SELECT image, content_type, name FROM image_data img JOIN pointcloud_image_data pcid ON img.id = pcid.image_data_id WHERE img.project_id = %(project_id)s AND pcid.image_data_id = %(image_id)s AND pcid.pointcloud_id = %(pointcloud_id)s """, { 'project_id': project_id, 'pointcloud_id': pointcloud_id, 'image_id': image_id, }) rows = cursor.fetchall() if len(rows) == 0: raise ValueError("Could not find image") if len(rows) > 1: raise ValueError("Found more than one image") image_data = rows[0][0] content_type = rows[0][1] name = rows[0][2] response = HttpResponse(image_data.tobytes(), content_type=content_type) response['Content-Disposition'] = f'attachment; filename={name}' return response
class LandmarkGroupList(APIView): @method_decorator(requires_user_role(UserRole.Browse)) def get(self, request, project_id): """List available landmark groups. --- parameters: - name: project_id description: Project of landmark groups type: integer paramType: path required: true - name: with_members description: Whether to return group members type: boolean paramType: form defaultValue: false required: false - name: with_locations description: Whether to return linked locations required: false defaultValue: false paramType: form """ with_members = request.query_params.get('with_members', 'false') == 'true' with_locations = request.query_params.get('with_locations', 'false') == 'true' landmarkgroup_class = Class.objects.get(project_id=project_id, class_name="landmarkgroup") landmarkgroups = ClassInstance.objects.filter( project_id=project_id, class_column=landmarkgroup_class).order_by('id') serializer = BasicClassInstanceSerializer(landmarkgroups, many=True) data = serializer.data if data: landmarkgroup_ids = [d['id'] for d in data] if with_members: # Get member information member_index = get_landmark_group_members( project_id, landmarkgroup_ids) # Append member information for group in data: group['members'] = member_index[group['id']] if with_locations: # Get linked locations, which represent instances of # landmark in this landmark group. location_index = get_landmark_group_locations( project_id, landmarkgroup_ids) # Append location information for group in data: group['locations'] = location_index[group['id']] return Response(data) @method_decorator(requires_user_role(UserRole.Annotate)) def put(self, request, project_id): """Add a new landmarkgroup. Expect at least the name as parameter. --- parameters: - name: project_id description: Project of landmark group type: integer paramType: path required: true - name: name description: Name of new landmark group type: string required: true """ name = request.data.get('name') landmarkgroup_class = Class.objects.get(project_id=project_id, class_name='landmarkgroup') # Prevent creation of duplicate landmark group classes existing_landmarkgroups = ClassInstance.objects.filter( project_id=project_id, name=name, class_column=landmarkgroup_class) if existing_landmarkgroups: raise ValueError( "There is already a landmark group with name {}".format(name)) landmarkgroup = ClassInstance.objects.create( project_id=project_id, class_column=landmarkgroup_class, user=request.user, name=name) landmarkgroup.save() serializer = BasicClassInstanceSerializer(landmarkgroup) return Response(serializer.data)