class ProjectListAPI(generics.ListCreateAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
serializer_class = ProjectSerializer
filter_backends = [filters.OrderingFilter]
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
POST=all_permissions.projects_create,
)
ordering = ['-created_at']
def get_queryset(self):
return Project.objects.with_counts().filter(organization=self.request.user.active_organization)
def get_serializer_context(self):
context = super(ProjectListAPI, self).get_serializer_context()
context['created_by'] = self.request.user
return context
def perform_create(self, ser):
try:
project = ser.save(organization=self.request.user.active_organization)
except IntegrityError as e:
if str(e) == 'UNIQUE constraint failed: project.title, project.created_by_id':
raise ProjectExistException('Project with the same name already exists: {}'.
format(ser.validated_data.get('title', '')))
raise LabelStudioDatabaseException('Database error during project creation. Try again.')
class ViewAPI(viewsets.ModelViewSet):
serializer_class = ViewSerializer
filter_backends = [DjangoFilterBackend]
filterset_fields = ["project"]
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
PUT=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
def perform_create(self, serializer):
serializer.save(user=self.request.user)
@swagger_auto_schema(
tags=['Data Manager'],
operation_summary="Reset project views",
operation_description="Reset all views for a specific project.",
request_body=ViewResetSerializer,
)
@action(detail=False, methods=["delete"])
def reset(self, request):
serializer = ViewResetSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
project = generics.get_object_or_404(Project.objects.for_user(request.user), pk=serializer.validated_data['project'].id)
queryset = self.filter_queryset(self.get_queryset()).filter(project=project)
queryset.all().delete()
return Response(status=204)
def get_queryset(self):
return View.objects.filter(project__organization=self.request.user.active_organization)
class AnnotationDraftListAPI(generics.ListCreateAPIView):
parser_classes = (JSONParser, MultiPartParser, FormParser)
serializer_class = AnnotationDraftSerializer
permission_required = ViewClassPermission(
GET=all_permissions.annotations_view,
POST=all_permissions.annotations_create,
)
queryset = AnnotationDraft.objects.all()
swagger_schema = None
def filter_queryset(self, queryset):
task_id = self.kwargs['pk']
return queryset.filter(task_id=task_id)
def perform_create(self, serializer):
task_id = self.kwargs['pk']
annotation_id = self.kwargs.get('annotation_id')
user = self.request.user
logger.debug(
f'User {user} is going to create draft for task={task_id}, annotation={annotation_id}'
)
serializer.save(task_id=self.kwargs['pk'],
annotation_id=annotation_id,
user=self.request.user)
class TaskListAPI(generics.ListCreateAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
queryset = Task.objects.all()
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_create,
)
serializer_class = TaskSerializer
def filter_queryset(self, queryset):
return queryset.filter(
project__organization=self.request.user.active_organization)
@swagger_auto_schema(auto_schema=None)
def get(self, request, *args, **kwargs):
return super(TaskListAPI, self).get(request, *args, **kwargs)
def get_serializer_context(self):
context = super(TaskListAPI, self).get_serializer_context()
project_id = self.request.data.get('project')
if project_id:
context['project'] = generics.get_object_or_404(Project,
pk=project_id)
return context
def post(self, request, *args, **kwargs):
return super(TaskListAPI, self).post(request, *args, **kwargs)
class ProjectActionsAPI(APIView):
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
POST=all_permissions.projects_view,
)
def get(self, request):
pk = int_from_request(request.GET, "project", 1) # replace 1 to None, it's for debug only
project = get_object_with_check_and_log(request, Project, pk=pk)
self.check_object_permissions(request, project)
return Response(get_all_actions(request.user, project))
def post(self, request):
pk = int_from_request(request.GET, "project", None)
project = get_object_with_check_and_log(request, Project, pk=pk)
self.check_object_permissions(request, project)
queryset = get_prepared_queryset(request, project)
# wrong action id
action_id = request.GET.get('id', None)
if action_id is None:
response = {'detail': 'No action id "' + str(action_id) + '", use ?id=<action-id>'}
return Response(response, status=422)
# perform action and return the result dict
kwargs = {'request': request} # pass advanced params to actions
result = perform_action(action_id, project, queryset, request.user, **kwargs)
code = result.pop('response_code', 200)
return Response(result, status=code)
class FileUploadListAPI(generics.mixins.ListModelMixin,
generics.mixins.DestroyModelMixin,
generics.GenericAPIView):
"""
get:
Get files list
Retrieve the list of uploaded files used to create labeling tasks for a specific project.
delete:
Delete files
Delete uploaded files for a specific project.
"""
parser_classes = (JSONParser, MultiPartParser, FormParser)
serializer_class = FileUploadSerializer
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
DELETE=all_permissions.projects_change,
)
queryset = FileUpload.objects.all()
def get_queryset(self):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs.get('pk', 0))
if project.is_draft:
# If project is in draft state, we return all uploaded files, ignoring queried ids
logger.debug(
f'Return all uploaded files for draft project {project}')
return FileUpload.objects.filter(project_id=project.id,
user=self.request.user)
# If requested in regular import, only queried IDs are returned to avoid showing previously imported
ids = json.loads(self.request.query_params.get('ids', '[]'))
logger.debug(f'File Upload IDs found: {ids}')
return FileUpload.objects.filter(project_id=project.id,
id__in=ids,
user=self.request.user)
@swagger_auto_schema(tags=['Import'])
def get(self, request, *args, **kwargs):
return self.list(request, *args, **kwargs)
@swagger_auto_schema(tags=['Import'])
def delete(self, request, *args, **kwargs):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs['pk'])
ids = self.request.data.get('file_upload_ids')
if ids is None:
deleted, _ = FileUpload.objects.filter(project=project).delete()
elif isinstance(ids, list):
deleted, _ = FileUpload.objects.filter(project=project,
id__in=ids).delete()
else:
raise ValueError(
'"file_upload_ids" parameter must be a list of integers')
return Response({'deleted': deleted}, status=status.HTTP_200_OK)
class OrganizationMemberListAPI(generics.ListAPIView):
"""
get:
Get organization members list
Retrieve a list of the organization members.
"""
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.organizations_view,
PUT=all_permissions.organizations_change,
PATCH=all_permissions.organizations_change,
DELETE=all_permissions.organizations_change,
)
serializer_class = OrganizationMemberUserSerializer
def get_queryset(self):
org = generics.get_object_or_404(self.request.user.organizations,
pk=self.kwargs[self.lookup_field])
return org.members
@swagger_auto_schema(tags=['Organizations'])
def get(self, request, *args, **kwargs):
return super(OrganizationMemberListAPI,
self).get(request, *args, **kwargs)
class TaskListAPI(DMTaskListAPI):
serializer_class = TaskSerializer
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_create,
)
filter_backends = [DjangoFilterBackend]
filterset_fields = [
'project',
]
def filter_queryset(self, queryset):
queryset = super().filter_queryset(queryset)
return queryset.filter(
project__organization=self.request.user.active_organization)
def get_serializer_context(self):
context = super().get_serializer_context()
project_id = self.request.data.get('project')
if project_id:
context['project'] = generics.get_object_or_404(Project,
pk=project_id)
return context
def perform_create(self, serializer):
project_id = self.request.data.get('project')
generics.get_object_or_404(Project, pk=project_id)
project = generics.get_object_or_404(Project, pk=project_id)
instance = serializer.save(project=project)
emit_webhooks_for_instance(self.request.user.active_organization,
project, WebhookAction.TASKS_CREATED,
[instance])
class LabelAPI(viewsets.ModelViewSet):
pagination_class = PageNumberPagination
serializer_class = LabelSerializer
permission_required = ViewClassPermission(
GET=all_permissions.labels_view,
POST=all_permissions.labels_create,
PATCH=all_permissions.labels_change,
DELETE=all_permissions.labels_delete,
)
def get_serializer(self, *args, **kwargs):
'''POST request is bulk by default'''
if self.action == 'create':
kwargs['many'] = True
return super().get_serializer(*args, **kwargs)
def perform_create(self, serializer):
serializer.save(created_by=self.request.user, organization=self.request.user.active_organization)
def get_queryset(self):
return Label.objects.filter(organization=self.request.user.active_organization).prefetch_related('links')
def get_serializer_class(self):
if self.request.method == 'POST':
return LabelCreateSerializer
return self.serializer_class
class LabelLinkAPI(viewsets.ModelViewSet):
filter_backends = [DjangoFilterBackend]
filterset_fields = {
'project': ['exact'],
'label__created_at': ['exact', 'gte', 'lte'],
'label__created_by': ['exact'],
}
pagination_class = PageNumberPagination
serializer_class = LabelLinkSerializer
permission_required = ViewClassPermission(
GET=all_permissions.labels_view,
POST=all_permissions.labels_create,
PATCH=all_permissions.labels_change,
DELETE=all_permissions.labels_delete,
)
def get_queryset(self):
return LabelLink.objects.filter(label__organization=self.request.user.active_organization).annotate(
annotations_count=Count(
'project__tasks__annotations',
filter=Q(
project__tasks__annotations__result__icontains=Cast('label__value', output_field=CharField())
),
)
)
@api_webhook('LABEL_LINK_UPDATED')
def update(self, request, *args, **kwargs):
return super().update(request, *args, **kwargs)
@api_webhook_for_delete('LABEL_LINK_DELETED')
def destroy(self, request, *args, **kwargs):
return super().destroy(request, *args, **kwargs)
class AnnotationAPI(RequestDebugLogMixin,
generics.RetrieveUpdateDestroyAPIView):
"""
get:
Get annotation by its ID
Retrieve a specific annotation for a task.
patch:
Update annotation
Update existing attributes on an annotation.
delete:
Delete annotation
Delete an annotation. This action can't be undone!
"""
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.annotations_view,
PUT=all_permissions.annotations_change,
PATCH=all_permissions.annotations_change,
DELETE=all_permissions.annotations_delete,
)
serializer_class = AnnotationSerializer
queryset = Annotation.objects.all()
def perform_destroy(self, annotation):
annotation.delete()
def update(self, request, *args, **kwargs):
# save user history with annotator_id, time & annotation result
annotation_id = self.kwargs['pk']
annotation = get_object_with_check_and_log(request,
Annotation,
pk=annotation_id)
annotation.task.save() # refresh task metrics
return super(AnnotationAPI, self).update(request, *args, **kwargs)
@swagger_auto_schema(tags=['Annotations'])
def get(self, request, *args, **kwargs):
return super(AnnotationAPI, self).get(request, *args, **kwargs)
@swagger_auto_schema(tags=['Annotations'], auto_schema=None)
def put(self, request, *args, **kwargs):
return super(AnnotationAPI, self).put(request, *args, **kwargs)
@swagger_auto_schema(tags=['Annotations'],
request_body=AnnotationSerializer)
def patch(self, request, *args, **kwargs):
return super(AnnotationAPI, self).patch(request, *args, **kwargs)
@swagger_auto_schema(tags=['Annotations'])
def delete(self, request, *args, **kwargs):
return super(AnnotationAPI, self).delete(request, *args, **kwargs)
class ProjectActionsAPI(APIView):
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
POST=all_permissions.projects_view,
)
@swagger_auto_schema(tags=["Data Manager"])
def get(self, request):
"""
get:
Get actions
Retrieve all the registered actions with descriptions that data manager can use.
"""
pk = int_from_request(request.GET, "project", 1) # replace 1 to None, it's for debug only
project = get_object_with_check_and_log(request, Project, pk=pk)
self.check_object_permissions(request, project)
params = {
'can_delete_tasks': True,
'can_manage_annotations': True,
'experimental_feature': False
}
return Response(get_all_actions(params))
@swagger_auto_schema(tags=["Data Manager"])
def post(self, request):
"""
post:
Post actions
Perform an action with the selected items from a specific view.
"""
pk = int_from_request(request.GET, "project", None)
project = get_object_with_check_and_log(request, Project, pk=pk)
self.check_object_permissions(request, project)
queryset = get_prepared_queryset(request, project)
# no selected items on tab
if not queryset.exists():
response = {'detail': 'No selected items for specified view'}
return Response(response, status=404)
# wrong action id
action_id = request.GET.get('id', None)
if action_id is None:
response = {'detail': 'No action id "' + str(action_id) + '", use ?id=<action-id>'}
return Response(response, status=422)
# perform action and return the result dict
kwargs = {'request': request} # pass advanced params to actions
result = perform_action(action_id, project, queryset, **kwargs)
code = result.pop('response_code', 200)
return Response(result, status=code)
class ProjectTaskListAPI(generics.ListCreateAPIView, generics.DestroyAPIView):
parser_classes = (JSONParser, FormParser)
queryset = Task.objects.all()
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
serializer_class = TaskSerializer
redirect_route = 'projects:project-settings'
redirect_kwarg = 'pk'
def get_serializer_class(self):
if self.request.method == 'GET':
return TaskSimpleSerializer
else:
return TaskSerializer
def filter_queryset(self, queryset):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs.get('pk', 0))
tasks = Task.objects.filter(project=project)
return paginator(tasks, self.request)
def delete(self, request, *args, **kwargs):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs['pk'])
task_ids = list(Task.objects.filter(project=project).values('id'))
Task.objects.filter(project=project).delete()
emit_webhooks_for_instance(request.user.active_organization, None,
WebhookAction.TASKS_DELETED, task_ids)
return Response(data={'tasks': task_ids}, status=204)
def get(self, *args, **kwargs):
return super(ProjectTaskListAPI, self).get(*args, **kwargs)
@swagger_auto_schema(auto_schema=None)
def post(self, *args, **kwargs):
return super(ProjectTaskListAPI, self).post(*args, **kwargs)
def get_serializer_context(self):
context = super(ProjectTaskListAPI, self).get_serializer_context()
context['project'] = get_object_with_check_and_log(
self.request, Project, pk=self.kwargs['pk'])
return context
def perform_create(self, serializer):
project = get_object_with_check_and_log(self.request,
Project,
pk=self.kwargs['pk'])
instance = serializer.save(project=project)
emit_webhooks_for_instance(self.request.user.active_organization,
project, WebhookAction.TASKS_CREATED,
[instance])
class TasksListAPI(generics.ListCreateAPIView, generics.DestroyAPIView,
APIViewVirtualMethodMixin, APIViewVirtualRedirectMixin):
"""
get:
List project tasks
Paginated list of tasks for a specific project.
delete:
Delete all tasks
Delete all tasks from a specific project.
"""
parser_classes = (JSONParser, FormParser)
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
serializer_class = TaskSerializer
redirect_route = 'projects:project-settings'
redirect_kwarg = 'pk'
def get_queryset(self):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs.get('pk', 0))
tasks = Task.objects.filter(project=project)
return paginator(tasks, self.request)
@swagger_auto_schema(tags=['Projects'])
def delete(self, request, *args, **kwargs):
project = generics.get_object_or_404(Project.objects.for_user(
self.request.user),
pk=self.kwargs['pk'])
Task.objects.filter(project=project).delete()
return Response(status=204)
@swagger_auto_schema(**paginator_help('tasks', 'Projects'))
def get(self, *args, **kwargs):
return super(TasksListAPI, self).get(*args, **kwargs)
@swagger_auto_schema(auto_schema=None, tags=['Projects'])
def post(self, *args, **kwargs):
return super(TasksListAPI, self).post(*args, **kwargs)
def get_serializer_context(self):
context = super(TasksListAPI, self).get_serializer_context()
context['project'] = get_object_with_check_and_log(
self.request, Project, pk=self.kwargs['pk'])
return context
def perform_create(self, serializer):
project = get_object_with_check_and_log(self.request,
Project,
pk=self.kwargs['pk'])
serializer.save(project=project)
class ProjectAPI(APIViewVirtualRedirectMixin, APIViewVirtualMethodMixin,
generics.RetrieveUpdateDestroyAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
queryset = Project.objects.with_counts()
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
DELETE=all_permissions.projects_delete,
PATCH=all_permissions.projects_change,
PUT=all_permissions.projects_change,
POST=all_permissions.projects_create,
)
serializer_class = ProjectSerializer
redirect_route = 'projects:project-detail'
redirect_kwarg = 'pk'
def get_queryset(self):
return Project.objects.with_counts().filter(
organization=self.request.user.active_organization)
def get(self, request, *args, **kwargs):
return super(ProjectAPI, self).get(request, *args, **kwargs)
def delete(self, request, *args, **kwargs):
return super(ProjectAPI, self).delete(request, *args, **kwargs)
def patch(self, request, *args, **kwargs):
project = self.get_object()
label_config = self.request.data.get('label_config')
# config changes can break view, so we need to reset them
if label_config:
try:
has_changes = config_essential_data_has_changed(
label_config, project.label_config)
except KeyError:
pass
else:
if has_changes:
View.objects.filter(project=project).all().delete()
return super(ProjectAPI, self).patch(request, *args, **kwargs)
def perform_destroy(self, instance):
# we don't need to relaculate counters if we delete whole project
with DisableSignals():
instance.delete()
@swagger_auto_schema(auto_schema=None)
def post(self, request, *args, **kwargs):
return super(ProjectAPI, self).post(request, *args, **kwargs)
@swagger_auto_schema(auto_schema=None)
def put(self, request, *args, **kwargs):
return super(ProjectAPI, self).put(request, *args, **kwargs)
class AnnotationDraftAPI(generics.RetrieveUpdateDestroyAPIView):
parser_classes = (JSONParser, MultiPartParser, FormParser)
serializer_class = AnnotationDraftSerializer
queryset = AnnotationDraft.objects.all()
permission_required = ViewClassPermission(
GET=all_permissions.annotations_view,
PUT=all_permissions.annotations_change,
PATCH=all_permissions.annotations_change,
DELETE=all_permissions.annotations_delete,
)
swagger_schema = None
class TaskAPI(generics.RetrieveUpdateDestroyAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
PUT=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
def get_queryset(self):
return Task.objects.filter(
project__organization=self.request.user.active_organization)
def get_serializer_class(self):
# GET => task + annotations + predictions + drafts
if self.request.method == 'GET':
return TaskWithAnnotationsAndPredictionsAndDraftsSerializer
# POST, PATCH, PUT
else:
return TaskSimpleSerializer
def retrieve(self, request, *args, **kwargs):
task = self.get_object()
# call machine learning api and format response
if task.project.evaluate_predictions_automatically:
for ml_backend in task.project.ml_backends.all():
ml_backend.predict_one_task(task)
result = self.get_serializer(task).data
# use proxy inlining to task data (for credential access)
proxy = bool_from_request(request.GET, 'proxy', True)
result['data'] = task.resolve_uri(result['data'], proxy=proxy)
return Response(result)
def get(self, request, *args, **kwargs):
return super(TaskAPI, self).get(request, *args, **kwargs)
def patch(self, request, *args, **kwargs):
return super(TaskAPI, self).patch(request, *args, **kwargs)
@swagger_auto_schema(tags=['Tasks'])
def delete(self, request, *args, **kwargs):
return super(TaskAPI, self).delete(request, *args, **kwargs)
@swagger_auto_schema(auto_schema=None)
def put(self, request, *args, **kwargs):
return super(TaskAPI, self).put(request, *args, **kwargs)
class UserAPI(viewsets.ModelViewSet):
serializer_class = UserSerializer
permission_required = ViewClassPermission(
GET=all_permissions.organizations_view,
PUT=all_permissions.organizations_change,
POST=all_permissions.organizations_change,
PATCH=all_permissions.organizations_view,
DELETE=all_permissions.organizations_change,
)
http_method_names = ['get', 'post', 'head', 'patch', 'delete']
def get_queryset(self):
return User.objects.filter(
organizations=self.request.user.active_organization)
@swagger_auto_schema(auto_schema=None, methods=['delete', 'post'])
@action(detail=True,
methods=['delete', 'post'],
permission_required=all_permissions.avatar_any)
def avatar(self, request, pk):
if request.method == 'POST':
avatar = check_avatar(request.FILES)
request.user.avatar = avatar
request.user.save()
return Response({'detail': 'avatar saved'}, status=200)
elif request.method == 'DELETE':
request.user.avatar = None
request.user.save()
return Response(status=204)
def update(self, request, *args, **kwargs):
return super(UserAPI, self).update(request, *args, **kwargs)
def list(self, request, *args, **kwargs):
return super(UserAPI, self).list(request, *args, **kwargs)
def create(self, request, *args, **kwargs):
return super(UserAPI, self).create(request, *args, **kwargs)
def retrieve(self, request, *args, **kwargs):
return super(UserAPI, self).retrieve(request, *args, **kwargs)
def partial_update(self, request, *args, **kwargs):
return super(UserAPI, self).partial_update(request, *args, **kwargs)
def destroy(self, request, *args, **kwargs):
return super(UserAPI, self).destroy(request, *args, **kwargs)
class AnnotationAPI(generics.RetrieveUpdateDestroyAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.annotations_view,
PUT=all_permissions.annotations_change,
PATCH=all_permissions.annotations_change,
DELETE=all_permissions.annotations_delete,
)
serializer_class = AnnotationSerializer
queryset = Annotation.objects.all()
def perform_destroy(self, annotation):
annotation.delete()
def update(self, request, *args, **kwargs):
# save user history with annotator_id, time & annotation result
annotation_id = self.kwargs['pk']
annotation = get_object_with_check_and_log(request,
Annotation,
pk=annotation_id)
annotation.task.save() # refresh task metrics
if self.request.data.get('ground_truth'):
annotation.task.ensure_unique_groundtruth(
annotation_id=annotation.id)
return super(AnnotationAPI, self).update(request, *args, **kwargs)
def get(self, request, *args, **kwargs):
return super(AnnotationAPI, self).get(request, *args, **kwargs)
@api_webhook(WebhookAction.ANNOTATION_UPDATED)
@swagger_auto_schema(auto_schema=None)
def put(self, request, *args, **kwargs):
return super(AnnotationAPI, self).put(request, *args, **kwargs)
@api_webhook(WebhookAction.ANNOTATION_UPDATED)
def patch(self, request, *args, **kwargs):
return super(AnnotationAPI, self).patch(request, *args, **kwargs)
@api_webhook_for_delete(WebhookAction.ANNOTATIONS_DELETED)
def delete(self, request, *args, **kwargs):
return super(AnnotationAPI, self).delete(request, *args, **kwargs)
class MLBackendListAPI(generics.ListCreateAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
POST=all_permissions.projects_change,
)
serializer_class = MLBackendSerializer
filter_backends = [DjangoFilterBackend]
filterset_fields = ["is_interactive"]
def get_queryset(self):
project_pk = self.request.query_params.get('project')
project = get_object_with_check_and_log(self.request,
Project,
pk=project_pk)
self.check_object_permissions(self.request, project)
ml_backends = MLBackend.objects.filter(project_id=project.id)
for mlb in ml_backends:
mlb.update_state()
return ml_backends
def perform_create(self, serializer):
ml_backend = serializer.save()
ml_backend.update_state()
class TaskListAPI(generics.ListCreateAPIView):
task_serializer_class = DataManagerTaskSerializer
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
PUT=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
@staticmethod
def get_task_serializer_context(request, project):
all_fields = request.GET.get('fields', None) == 'all' # false by default
return {
'resolve_uri': True,
'request': request,
'project': project,
'drafts': all_fields,
'predictions': all_fields,
'annotations': all_fields
}
def get_task_queryset(self, request, prepare_params):
return Task.prepared.only_filtered(prepare_params=prepare_params)
@staticmethod
def prefetch(queryset):
return queryset.prefetch_related(
'annotations', 'predictions', 'annotations__completed_by', 'project',
'io_storages_azureblobimportstoragelink',
'io_storages_gcsimportstoragelink',
'io_storages_localfilesimportstoragelink',
'io_storages_redisimportstoragelink',
'io_storages_s3importstoragelink',
'file_upload'
)
def get(self, request):
# get project
view_pk = int_from_request(request.GET, 'view', 0) or int_from_request(request.data, 'view', 0)
project_pk = int_from_request(request.GET, 'project', 0) or int_from_request(request.data, 'project', 0)
if project_pk:
project = get_object_with_check_and_log(request, Project, pk=project_pk)
self.check_object_permissions(request, project)
elif view_pk:
view = get_object_with_check_and_log(request, View, pk=view_pk)
project = view.project
self.check_object_permissions(request, project)
else:
return Response({'detail': 'Neither project nor view id specified'}, status=404)
# get prepare params (from view or from payload directly)
prepare_params = get_prepare_params(request, project)
queryset = self.get_task_queryset(request, prepare_params)
context = self.get_task_serializer_context(self.request, project)
# paginated tasks
self.pagination_class = TaskPagination
page = self.paginate_queryset(queryset)
all_fields = 'all' if request.GET.get('fields', None) == 'all' else None
fields_for_evaluation = get_fields_for_evaluation(prepare_params, request.user)
review = bool_from_request(self.request.GET, 'review', False)
if review:
fields_for_evaluation = ['annotators', 'reviewed']
all_fields = None
if page is not None:
ids = [task.id for task in page] # page is a list already
tasks = list(
self.prefetch(
Task.prepared.annotate_queryset(
Task.objects.filter(id__in=ids),
fields_for_evaluation=fields_for_evaluation,
all_fields=all_fields,
)
)
)
tasks_by_ids = {task.id: task for task in tasks}
# keep ids ordering
page = [tasks_by_ids[_id] for _id in ids]
# retrieve ML predictions if tasks don't have them
if not review and project.evaluate_predictions_automatically:
tasks_for_predictions = Task.objects.filter(id__in=ids, predictions__isnull=True)
evaluate_predictions(tasks_for_predictions)
serializer = self.task_serializer_class(page, many=True, context=context)
return self.get_paginated_response(serializer.data)
# all tasks
if project.evaluate_predictions_automatically:
evaluate_predictions(queryset.filter(predictions__isnull=True))
queryset = Task.prepared.annotate_queryset(
queryset, fields_for_evaluation=fields_for_evaluation, all_fields=all_fields
)
serializer = self.task_serializer_class(queryset, many=True, context=context)
return Response(serializer.data)
class TaskListAPI(generics.ListAPIView):
swagger_schema = None
task_serializer_class = DataManagerTaskSerializer
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
PUT=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
@staticmethod
def get_task_serializer_context(request, project):
storage = find_first_many_to_one_related_field_by_prefix(
project, '.*io_storages.*')
resolve_uri = True
if not storage and not project.task_data_login and not project.task_data_password:
resolve_uri = False
all_fields = request.GET.get('fields',
None) == 'all' # false by default
return {
'proxy': bool_from_request(request.GET, 'proxy', True),
'resolve_uri': resolve_uri,
'request': request,
'project': project,
'drafts': all_fields,
'predictions': all_fields,
'annotations': all_fields
}
def get_task_queryset(self, request, prepare_params):
return Task.prepared.only_filtered(prepare_params=prepare_params)
@swagger_auto_schema(tags=['Data Manager'],
responses={200: task_serializer_class(many=True)})
def get(self, request):
"""
get:
Get task list for view
Retrieve a list of tasks with pagination for a specific view using filters and ordering.
"""
# get project
view_pk = int_from_request(request.GET, 'view', 0) or int_from_request(
request.data, 'view', 0)
project_pk = int_from_request(request.GET, 'project',
0) or int_from_request(
request.data, 'project', 0)
if project_pk:
project = get_object_with_check_and_log(request,
Project,
pk=project_pk)
self.check_object_permissions(request, project)
elif view_pk:
view = get_object_with_check_and_log(request, View, pk=view_pk)
project = view.project
self.check_object_permissions(request, project)
else:
return Response(
{'detail': 'Neither project nor view id specified'},
status=404)
# get prepare params (from view or from payload directly)
prepare_params = get_prepare_params(request, project)
queryset = self.get_task_queryset(request, prepare_params)
context = self.get_task_serializer_context(self.request, project)
# paginated tasks
self.pagination_class = TaskPagination
page = self.paginate_queryset(queryset)
all_fields = 'all' if request.GET.get('fields',
None) == 'all' else None
fields_for_evaluation = get_fields_for_evaluation(
prepare_params, request.user)
if page is not None:
ids = [task.id for task in page] # page is a list already
tasks = list(
Task.prepared.annotate_queryset(
Task.objects.filter(id__in=ids),
fields_for_evaluation=fields_for_evaluation,
all_fields=all_fields,
))
tasks_by_ids = {task.id: task for task in tasks}
# keep ids ordering
page = [tasks_by_ids[_id] for _id in ids]
# retrieve ML predictions if tasks don't have them
if project.evaluate_predictions_automatically:
tasks_for_predictions = Task.objects.filter(
id__in=ids, predictions__isnull=True)
evaluate_predictions(tasks_for_predictions)
serializer = self.task_serializer_class(page,
many=True,
context=context)
return self.get_paginated_response(serializer.data)
# all tasks
if project.evaluate_predictions_automatically:
evaluate_predictions(queryset.filter(predictions__isnull=True))
queryset = Task.prepared.annotate_queryset(
queryset,
fields_for_evaluation=fields_for_evaluation,
all_fields=all_fields)
serializer = self.task_serializer_class(queryset,
many=True,
context=context)
return Response(serializer.data)
class TaskAPI(generics.RetrieveUpdateDestroyAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
PUT=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
@staticmethod
def prefetch(queryset):
return queryset.prefetch_related(
'annotations', 'predictions', 'annotations__completed_by',
'project', 'io_storages_azureblobimportstoragelink',
'io_storages_gcsimportstoragelink',
'io_storages_localfilesimportstoragelink',
'io_storages_redisimportstoragelink',
'io_storages_s3importstoragelink', 'file_upload',
'project__ml_backends')
def get_retrieve_serializer_context(self, request):
fields = ['completed_by_full', 'drafts', 'predictions', 'annotations']
return {
'resolve_uri': True,
'completed_by': 'full' if 'completed_by_full' in fields else None,
'predictions': 'predictions' in fields,
'annotations': 'annotations' in fields,
'drafts': 'drafts' in fields,
'request': request
}
def get(self, request, pk):
self.task = self.get_object()
context = self.get_retrieve_serializer_context(request)
context['project'] = project = self.task.project
# get prediction
if (project.evaluate_predictions_automatically or project.show_collab_predictions) \
and not self.task.predictions.exists():
evaluate_predictions([self.task])
serializer = self.get_serializer_class()(self.task,
many=False,
context=context)
data = serializer.data
return Response(data)
def get_queryset(self):
review = bool_from_request(self.request.GET, 'review', False)
selected = {"all": False, "included": [self.kwargs.get("pk")]}
if review:
kwargs = {'fields_for_evaluation': ['annotators', 'reviewed']}
else:
kwargs = {'all_fields': True}
project = self.request.query_params.get(
'project') or self.request.data.get('project')
if not project:
project = Task.objects.get(
id=self.request.parser_context['kwargs'].get('pk')).project.id
return self.prefetch(
Task.prepared.get_queryset(prepare_params=PrepareParams(
project=project, selectedItems=selected),
**kwargs))
def get_serializer_class(self):
# GET => task + annotations + predictions + drafts
if self.request.method == 'GET':
return DataManagerTaskSerializer
# POST, PATCH, PUT
else:
return TaskSimpleSerializer
def retrieve(self, request, *args, **kwargs):
task = self.get_object()
project = task.project
# call machine learning api and format response
if project.evaluate_predictions_automatically:
for ml_backend in task.project.ml_backends.all():
ml_backend.predict_tasks([task])
result = self.get_serializer(task).data
# use proxy inlining to task data (for credential access)
result['data'] = task.resolve_uri(result['data'], project)
return Response(result)
def patch(self, request, *args, **kwargs):
return super(TaskAPI, self).patch(request, *args, **kwargs)
@api_webhook_for_delete(WebhookAction.TASKS_DELETED)
def delete(self, request, *args, **kwargs):
return super(TaskAPI, self).delete(request, *args, **kwargs)
@swagger_auto_schema(auto_schema=None)
def put(self, request, *args, **kwargs):
return super(TaskAPI, self).put(request, *args, **kwargs)
class AnnotationsListAPI(generics.ListCreateAPIView):
parser_classes = (JSONParser, FormParser, MultiPartParser)
permission_required = ViewClassPermission(
GET=all_permissions.annotations_view,
POST=all_permissions.annotations_create,
)
serializer_class = AnnotationSerializer
def get(self, request, *args, **kwargs):
return super(AnnotationsListAPI, self).get(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
return super(AnnotationsListAPI, self).post(request, *args, **kwargs)
def get_queryset(self):
task = generics.get_object_or_404(Task.objects.for_user(
self.request.user),
pk=self.kwargs.get('pk', 0))
return Annotation.objects.filter(
Q(task=task) & Q(was_cancelled=False)).order_by('pk')
def perform_create(self, ser):
task = get_object_with_check_and_log(self.request,
Task,
pk=self.kwargs['pk'])
# annotator has write access only to annotations and it can't be checked it after serializer.save()
user = self.request.user
# updates history
update_id = self.request.user.id
result = ser.validated_data.get('result')
extra_args = {'task_id': self.kwargs['pk']}
# save stats about how well annotator annotations coincide with current prediction
# only for finished task annotations
if result is not None:
prediction = Prediction.objects.filter(
task=task, model_version=task.project.model_version)
if prediction.exists():
prediction = prediction.first()
prediction_ser = PredictionSerializer(prediction).data
else:
logger.debug(
f'User={self.request.user}: there are no predictions for task={task}'
)
prediction_ser = {}
# serialize annotation
extra_args.update({
'prediction': prediction_ser,
})
if 'was_cancelled' in self.request.GET:
extra_args['was_cancelled'] = bool_from_request(
self.request.GET, 'was_cancelled', False)
if 'completed_by' not in ser.validated_data:
extra_args['completed_by'] = self.request.user
# create annotation
logger.debug(f'User={self.request.user}: save annotation')
annotation = ser.save(**extra_args)
logger.debug(f'Save activity for user={self.request.user}')
self.request.user.activity_at = timezone.now()
self.request.user.save()
# Release task if it has been taken at work (it should be taken by the same user, or it makes sentry error
logger.debug(f'User={user} releases task={task}')
task.release_lock(user)
# if annotation created from draft - remove this draft
draft_id = self.request.data.get('draft_id')
if draft_id is not None:
logger.debug(
f'Remove draft {draft_id} after creating annotation {annotation.id}'
)
AnnotationDraft.objects.filter(id=draft_id).delete()
if self.request.data.get('ground_truth'):
annotation.task.ensure_unique_groundtruth(
annotation_id=annotation.id)
return annotation
class ViewAPI(viewsets.ModelViewSet):
queryset = View.objects.all()
serializer_class = ViewSerializer
filter_backends = [DjangoFilterBackend]
filterset_fields = ["project"]
task_serializer_class = DataManagerTaskSerializer
permission_required = ViewClassPermission(
GET=all_permissions.tasks_view,
POST=all_permissions.tasks_change,
PATCH=all_permissions.tasks_change,
PUT=all_permissions.tasks_change,
DELETE=all_permissions.tasks_delete,
)
def perform_create(self, serializer):
serializer.save(user=self.request.user)
@swagger_auto_schema(tags=['Data Manager'])
@action(detail=False, methods=['delete'])
def reset(self, _request):
"""
delete:
Reset project views
Reset all views for a specific project.
"""
queryset = self.filter_queryset(self.get_queryset())
queryset.all().delete()
return Response(status=204)
def get_task_queryset(self, request, view):
return Task.prepared.all(
prepare_params=view.get_prepare_tasks_params())
@swagger_auto_schema(tags=['Data Manager'],
responses={200: task_serializer_class(many=True)})
@action(detail=True, methods=["get"])
def tasks(self, request, pk=None):
"""
get:
Get task list for view
Retrieve a list of tasks with pagination for a specific view using filters and ordering.
"""
view = self.get_object()
queryset = self.get_task_queryset(request, view)
context = {
'proxy': bool_from_request(request.GET, 'proxy', True),
'resolve_uri': True,
'request': request
}
project = view.project
# paginated tasks
self.pagination_class = TaskPagination
page = self.paginate_queryset(queryset)
if page is not None:
# retrieve ML predictions if tasks don't have them
if project.evaluate_predictions_automatically:
ids = [task.id for task in page] # page is a list already
tasks_for_predictions = Task.objects.filter(
id__in=ids, predictions__isnull=True)
evaluate_predictions(tasks_for_predictions)
serializer = self.task_serializer_class(page,
many=True,
context=context)
return self.get_paginated_response(serializer.data)
# all tasks
if project.evaluate_predictions_automatically:
evaluate_predictions(queryset.filter(predictions__isnull=True))
serializer = self.task_serializer_class(queryset,
many=True,
context=context)
return Response(serializer.data)
@swagger_auto_schema(tags=['Data Manager'],
methods=["get", "post", "delete", "patch"])
@action(detail=True,
url_path="selected-items",
methods=["get", "post", "delete", "patch"])
def selected_items(self, request, pk=None):
"""
get:
Get selected items
Retrieve selected tasks for a specified view.
post:
Overwrite selected items
Overwrite the selected items with new data.
patch:
Add selected items
Add selected items to a specific view.
delete:
Delete selected items
Delete selected items from a specific view.
"""
view = self.get_object()
# GET: get selected items from tab
if request.method == "GET":
serializer = SelectedItemsSerializer(view.selected_items)
return Response(serializer.data)
data = request.data
serializer = SelectedItemsSerializer(data=data,
context={
"view": view,
"request": request
})
serializer.is_valid(raise_exception=True)
data = serializer.validated_data
# POST: set whole
if request.method == "POST":
view.selected_items = data
view.save()
return Response(serializer.validated_data, status=201)
selected_items = view.selected_items
if selected_items is None:
selected_items = {"all": False, "included": []}
key = "excluded" if data["all"] else "included"
left = OrderedSet(selected_items.get(key, []))
right = OrderedSet(data.get(key, []))
# PATCH: set particular with union
if request.method == "PATCH":
# make union
result = left | right
view.selected_items = selected_items
view.selected_items[key] = list(result)
view.save(update_fields=["selected_items"])
return Response(view.selected_items, status=201)
# DELETE: delete specified items
if request.method == "DELETE":
result = left - right
view.selected_items[key] = list(result)
view.save(update_fields=["selected_items"])
return Response(view.selected_items, status=204)
class ProjectAPI(APIViewVirtualRedirectMixin, APIViewVirtualMethodMixin,
generics.RetrieveUpdateDestroyAPIView):
"""
get:
Get project by ID
Retrieve information about a project by ID.
patch:
Update project
Update project settings for a specific project.
delete:
Delete project
Delete a project by specified project ID.
"""
parser_classes = (JSONParser, FormParser, MultiPartParser)
queryset = Project.objects.with_counts()
permission_required = ViewClassPermission(
GET=all_permissions.projects_view,
DELETE=all_permissions.projects_delete,
PATCH=all_permissions.projects_change,
PUT=all_permissions.projects_change,
POST=all_permissions.projects_create,
)
serializer_class = ProjectSerializer
redirect_route = 'projects:project-detail'
redirect_kwarg = 'pk'
def get_queryset(self):
return Project.objects.with_counts().filter(
organization=self.request.user.active_organization)
@swagger_auto_schema(tags=['Projects'])
def get(self, request, *args, **kwargs):
return super(ProjectAPI, self).get(request, *args, **kwargs)
@swagger_auto_schema(tags=['Projects'])
def delete(self, request, *args, **kwargs):
return super(ProjectAPI, self).delete(request, *args, **kwargs)
@swagger_auto_schema(tags=['Projects'], request_body=ProjectSerializer)
def patch(self, request, *args, **kwargs):
project = self.get_object()
label_config = self.request.data.get('label_config')
# config changes can break view, so we need to reset them
if label_config:
try:
has_changes = config_essential_data_has_changed(
label_config, project.label_config)
except KeyError:
pass
else:
if has_changes:
View.objects.filter(project=project).all().delete()
return super(ProjectAPI, self).patch(request, *args, **kwargs)
def perform_destroy(self, instance):
"""Performance optimization for whole project deletion
if we catch constraint error fallback to regular .delete() method"""
try:
task_annotation_qs = Annotation.objects.filter(
task__project_id=instance.id)
task_annotation_qs._raw_delete(task_annotation_qs.db)
task_prediction_qs = Prediction.objects.filter(
task__project_id=instance.id)
task_prediction_qs._raw_delete(task_prediction_qs.db)
task_locks_qs = TaskLock.objects.filter(
task__project_id=instance.id)
task_locks_qs._raw_delete(task_locks_qs.db)
task_qs = Task.objects.filter(project_id=instance.id)
task_qs._raw_delete(task_qs.db)
instance.delete()
except IntegrityError as e:
logger.error(
'Fallback to cascade deleting after integrity_error: {}'.
format(str(e)))
instance.delete()
@swagger_auto_schema(auto_schema=None)
def post(self, request, *args, **kwargs):
return super(ProjectAPI, self).post(request, *args, **kwargs)
@swagger_auto_schema(auto_schema=None)
def put(self, request, *args, **kwargs):
return super(ProjectAPI, self).put(request, *args, **kwargs)
