def emit_webhooks(organization, project, action, payload):
"""Run all active webhooks for the action."""
webhooks = get_active_webhooks(organization, project, action)
if project and payload and webhooks.filter(send_payload=True).exists():
payload['project'] = load_func(settings.WEBHOOK_SERIALIZERS['project'])(instance=project).data
for wh in webhooks:
run_webhook(wh, action, payload)
def get(self, request):
pk = int_from_request(request.GET, "project", 1)
project = get_object_with_check_and_log(request, Project, pk=pk)
self.check_object_permissions(request, project)
GET_ALL_COLUMNS = load_func(settings.DATA_MANAGER_GET_ALL_COLUMNS)
data = GET_ALL_COLUMNS(project, request.user)
return Response(data)
def user_login(request):
""" Login page
"""
user = request.user
next_page = request.GET.get('next')
next_page = next_page if next_page else reverse('projects:project-index')
login_form = load_func(settings.USER_LOGIN_FORM)
form = login_form()
if user.is_authenticated:
return redirect(next_page)
if request.method == 'POST':
form = login_form(request.POST)
if form.is_valid():
user = form.cleaned_data['user']
auth.login(request,
user,
backend='django.contrib.auth.backends.ModelBackend')
# user is organization member
org_pk = Organization.find_by_user(user).pk
user.active_organization_id = org_pk
user.save(update_fields=['active_organization'])
return redirect(next_page)
return render(request, 'users/user_login.html', {
'form': form,
'next': next_page
})
def emit_webhooks_for_instance(organization, project, action, instance=None):
"""Run all active webhooks for the action using instances as payload.
Be sure WebhookAction.ACTIONS contains all required fields.
"""
webhooks = get_active_webhooks(organization, project, action)
if not webhooks.exists():
return
payload = {}
# if instances and there is a webhook that sends payload
# get serialized payload
action_meta = WebhookAction.ACTIONS[action]
if instance and webhooks.filter(send_payload=True).exists():
serializer_class = action_meta.get('serializer')
if serializer_class:
payload[action_meta['key']] = serializer_class(instance=instance, many=action_meta['many']).data
if project and payload:
payload['project'] = load_func(settings.WEBHOOK_SERIALIZERS['project'])(instance=project).data
if payload and 'nested-fields' in action_meta:
for key, value in action_meta['nested-fields'].items():
payload[key] = value['serializer'](
instance=get_nested_field(instance, value['field']), many=value['many']
).data
for wh in webhooks:
run_webhook(wh, action, payload)
def _get_serialized_data(self, annotation):
if get_bool_env('FUTURE_SAVE_TASK_TO_STORAGE', default=False):
# export task with annotations
return ExportDataSerializer(annotation.task).data
else:
serializer_class = load_func(settings.STORAGE_ANNOTATION_SERIALIZER)
# deprecated functionality - save only annotation
return serializer_class(annotation).data
def proceed_registration(request, user_form, organization_form, next_page):
""" Register a new user for POST user_signup
"""
# save user to db
save_user = load_func(settings.SAVE_USER)
response = save_user(request, next_page, user_form)
return response
def proceed_registration(request, user_form, organization_form, next_page):
""" Register a new user for POST user_signup
"""
# save user to db
save_user = load_func(settings.SAVE_USER)
user, redirect_url = save_user(request, next_page, user_form,
organization_form)
if Organization.objects.exists():
org = Organization.objects.first()
org.add_user(user)
else:
org = Organization.create_organization(created_by=user,
title='Label Studio')
user.active_organization = org
user.save(update_fields=['active_organization'])
return redirect(redirect_url)
def train(self, project, use_ground_truth=False):
# TODO Replace AnonymousUser with real user from request
user = AnonymousUser()
# Identify if feature flag is turned on
if flag_set(
'ff_back_dev_1417_start_training_mlbackend_webhooks_250122_long',
user):
request = {
'action':
'PROJECT_UPDATED',
'project':
load_func(settings.WEBHOOK_SERIALIZERS['project'])(
instance=project).data
}
return self._request('webhook',
request,
verbose=False,
timeout=TIMEOUT_PREDICT)
else:
# get only tasks with annotations
tasks = project.tasks.annotate(
num_annotations=Count('annotations')).filter(
num_annotations__gt=0)
# create serialized tasks with annotations: {"data": {...}, "annotations": [{...}], "predictions": [{...}]}
tasks_ser = ExportDataSerializer(tasks, many=True).data
logger.debug(
f'{len(tasks_ser)} tasks with annotations are sent to ML backend for training.'
)
request = {
'annotations': tasks_ser,
'project': self._create_project_uid(project),
'label_config': project.label_config,
'params': {
'login': project.task_data_login,
'password': project.task_data_password
}
}
return self._request('train',
request,
verbose=False,
timeout=TIMEOUT_PREDICT)
import ujson as json
from core import version
from core.utils.common import load_func
from core.utils.io import get_all_files_from_dir, get_temp_dir, read_bytes_stream
from django.conf import settings
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.translation import gettext_lazy as _
from label_studio_converter import Converter
from tasks.models import Annotation
logger = logging.getLogger(__name__)
ExportMixin = load_func(settings.EXPORT_MIXIN)
class Export(ExportMixin, models.Model):
class Status(models.TextChoices):
CREATED = 'created', _('Created')
IN_PROGRESS = 'in_progress', _('In progress')
FAILED = 'failed', _('Failed')
COMPLETED = 'completed', _('Completed')
title = models.CharField(
_('title'),
blank=True,
default='',
max_length=2048,
)
self.project.model_version = last_model_version
self.project.save()
self.post_process_annotations(self.db_annotations)
return db_tasks
@staticmethod
def post_process_annotations(db_annotations):
pass
class Meta:
model = Task
fields = "__all__"
TaskSerializer = load_func(settings.TASK_SERIALIZER)
class TaskWithAnnotationsSerializer(TaskSerializer):
"""
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fields['annotations'] = AnnotationSerializer(many=True, default=[], context=self.context)
class Meta:
model = Task
list_serializer_class = load_func(settings.TASK_SERIALIZER_BULK)
exclude = ()
class WebhookAction(models.Model):
PROJECT_CREATED = 'PROJECT_CREATED'
PROJECT_UPDATED = 'PROJECT_UPDATED'
PROJECT_DELETED = 'PROJECT_DELETED'
TASKS_CREATED = 'TASKS_CREATED'
TASKS_DELETED = 'TASKS_DELETED'
ANNOTATION_CREATED = 'ANNOTATION_CREATED'
ANNOTATIONS_CREATED = 'ANNOTATIONS_CREATED'
ANNOTATION_UPDATED = 'ANNOTATION_UPDATED'
ANNOTATIONS_DELETED = 'ANNOTATIONS_DELETED'
LABEL_LINK_CREATED = 'LABEL_LINK_CREATED'
LABEL_LINK_UPDATED = 'LABEL_LINK_UPDATED'
LABEL_LINK_DELETED = 'LABEL_LINK_DELETED'
ACTIONS = {
PROJECT_CREATED: {
'name': _('Project created'),
'description': _(''),
'key': 'project',
'many': False,
'model': Project,
'serializer': load_func(settings.WEBHOOK_SERIALIZERS['project']),
'organization-only': True,
},
PROJECT_UPDATED: {
'name': _('Project updated'),
'description': _(''),
'key': 'project',
'many': False,
'model': Project,
'serializer': load_func(settings.WEBHOOK_SERIALIZERS['project']),
'project-field': '__self__',
},
PROJECT_DELETED: {
'name': _('Project deleted'),
'description': _(''),
'key': 'project',
'many': False,
'model': Project,
'serializer': OnlyIDWebhookSerializer,
'organization-only': True,
},
TASKS_CREATED: {
'name': _('Task created'),
'description': _(''),
'key': 'tasks',
'many': True,
'model': Task,
'serializer': load_func(settings.WEBHOOK_SERIALIZERS['task']),
'project-field': 'project',
},
TASKS_DELETED: {
'name': _('Task deleted'),
'description': _(''),
'key': 'tasks',
'many': True,
'model': Task,
'serializer': OnlyIDWebhookSerializer,
'project-field': 'project',
},
ANNOTATION_CREATED: {
'name': _('Annotation created'),
'description': _(''),
'key': 'annotation',
'many': False,
'model': Annotation,
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['annotation']),
'project-field': 'task__project',
'nested-fields': {
'task': {
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['task']),
'many': False,
'field': 'task',
},
},
},
ANNOTATIONS_CREATED: {
'name': _('Annotations created'),
'description': _(''),
'key': 'annotation',
'many': True,
'model': Annotation,
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['annotation']),
'project-field': 'task__project',
'nested-fields': {
'task': {
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['task']),
'many': True,
'field': 'task',
},
},
},
ANNOTATION_UPDATED: {
'name': _('Annotation updated'),
'description': _(''),
'key': 'annotation',
'many': False,
'model': Annotation,
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['annotation']),
'project-field': 'task__project',
'nested-fields': {
'task': {
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['task']),
'many': False,
'field': 'task',
},
},
},
ANNOTATIONS_DELETED: {
'name': _('Annotation deleted'),
'description': _(''),
'key': 'annotations',
'many': True,
'model': Annotation,
'serializer': OnlyIDWebhookSerializer,
'project-field': 'task__project',
},
LABEL_LINK_CREATED: {
'name': _('Label link created'),
'description': _(''),
'key': 'label_link',
'many': True,
'model': LabelLink,
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['label_link']),
'project-field': 'project',
},
LABEL_LINK_UPDATED: {
'name': _('Label link updated'),
'description': _(''),
'key': 'label_link',
'many': False,
'model': LabelLink,
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['label_link']),
'project-field': 'project',
'nested-fields': {
'label': {
'many': False,
'field': 'label',
'serializer':
load_func(settings.WEBHOOK_SERIALIZERS['label']),
},
}
},
LABEL_LINK_DELETED: {
'name': _('Label link deleted'),
'description': _(''),
'key': 'label_link',
'many': False,
'model': LabelLink,
'serializer': OnlyIDWebhookSerializer,
'project-field': 'project',
},
}
webhook = models.ForeignKey(Webhook,
on_delete=models.CASCADE,
related_name='actions')
action = models.CharField(
_('action of webhook'),
choices=[[key, value['name']] for key, value in ACTIONS.items()],
max_length=128,
db_index=True,
help_text=_('Action value'),
)
class Meta:
db_table = 'webhook_action'
unique_together = [['webhook', 'action']]
def create_organization(cls, created_by=None, title='Your Organization'):
_create_organization = load_func(settings.CREATE_ORGANIZATION)
return _create_organization(title=title, created_by=created_by)
from users.models import User
user_pk = user_or_user_pk.pk if isinstance(user_or_user_pk,
User) else user_or_user_pk
return OrganizationMember.objects.get(user=user_pk,
organization=organization_pk)
@property
def is_owner(self):
return self.user.id == self.organization.created_by.id
class Meta:
ordering = ['pk']
OrganizationMixin = load_func(settings.ORGANIZATION_MIXIN)
class Organization(OrganizationMixin, models.Model):
"""
"""
title = models.CharField(_('organization title'),
max_length=1000,
null=False)
token = models.CharField(_('token'),
max_length=256,
default=create_hash,
unique=True,
null=True,
blank=True)
'tasks__annotations__id',
filter=Q(tasks__annotations__was_cancelled=False) &
Q(tasks__annotations__ground_truth=False) &
Q(tasks__annotations__result__isnull=False)
),
ground_truth_number=Count(
'tasks__annotations__id',
filter=Q(tasks__annotations__ground_truth=True)
),
skipped_annotations_number=Count(
'tasks__annotations__id',
filter=Q(tasks__annotations__was_cancelled=True)
),
)
ProjectMixin = load_func(settings.PROJECT_MIXIN)
class Project(ProjectMixin, models.Model):
"""
"""
objects = ProjectManager()
__original_label_config = None
title = models.CharField(_('title'), null=True, blank=True, default='', max_length=settings.PROJECT_TITLE_MAX_LEN,
help_text=f'Project name. Must be between {settings.PROJECT_TITLE_MIN_LEN} and {settings.PROJECT_TITLE_MAX_LEN} characters long.',
validators=[MinLengthValidator(settings.PROJECT_TITLE_MIN_LEN), MaxLengthValidator(settings.PROJECT_TITLE_MAX_LEN)])
description = models.TextField(_('description'), blank=True, null=True, default='', help_text='Project description')
organization = models.ForeignKey('organizations.Organization', on_delete=models.CASCADE, related_name='projects', null=True)
label_config = models.TextField(_('label config'), blank=True, null=True, default='<View></View>',
def get_avatar(self, user):
return user.avatar_url
def get_initials(self, user):
return user.get_initials()
class Meta:
model = User
fields = (
'id',
'first_name',
'last_name',
'username',
'email',
'last_activity',
'avatar',
'initials',
'phone',
'active_organization',
)
class UserSimpleSerializer(BaseUserSerializer):
class Meta:
model = User
fields = ('id', 'first_name', 'last_name', 'email', 'avatar')
UserSerializer = load_func(settings.USER_SERIALIZER)
class UserLastActivityMixin(models.Model):
last_activity = models.DateTimeField(_('last activity'),
default=timezone.now,
editable=False)
def update_last_activity(self):
self.last_activity = timezone.now()
self.save(update_fields=["last_activity"])
class Meta:
abstract = True
UserMixin = load_func(settings.USER_MIXIN)
class User(UserMixin, AbstractBaseUser, PermissionsMixin,
UserLastActivityMixin):
"""
An abstract base class implementing a fully featured User model with
admin-compliant permissions.
Username and password are required. Other fields are optional.
"""
username = models.CharField(_('username'), max_length=256)
email = models.EmailField(_('email address'), unique=True, blank=True)
first_name = models.CharField(_('first name'), max_length=256, blank=True)
last_name = models.CharField(_('last name'), max_length=256, blank=True)
class Meta:
model = Task
list_serializer_class = load_func(settings.TASK_SERIALIZER_BULK)
exclude = ()
from django.db.models import JSONField
from django.urls import reverse
from django.utils.timesince import timesince
from django.utils.timezone import now
from django.dispatch import receiver, Signal
from model_utils import FieldTracker
from core.utils.common import find_first_one_to_one_related_field_by_prefix, string_is_url, load_func
from core.utils.params import get_env
from data_manager.managers import PreparedTaskManager, TaskManager
from core.bulk_update_utils import bulk_update
logger = logging.getLogger(__name__)
TaskMixin = load_func(settings.TASK_MIXIN)
class Task(TaskMixin, models.Model):
""" Business tasks from project
"""
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID', db_index=True)
data = JSONField('data', null=False, help_text='User imported or uploaded data for a task. Data is formatted according to '
'the project label config. You can find examples of data for your project '
'on the Import page in the Label Studio Data Manager UI.')
meta = JSONField('meta', null=True, default=dict,
help_text='Meta is user imported (uploaded) data and can be useful as input for an ML '
'Backend for embeddings, advanced vectors, and other info. It is passed to '
'ML during training/predicting steps.')
project = models.ForeignKey('projects.Project', related_name='tasks', on_delete=models.CASCADE, null=True,
help_text='Project ID for this task')
fields = ExportSerializer.Meta.fields + [
'task_filter_options',
'annotation_filter_options',
'serialization_options',
]
task_filter_options = TaskFilterOptionsSerializer(required=False, default=None)
annotation_filter_options = AnnotationFilterOptionsSerializer(required=False, default=None)
serialization_options = SerializationOptionsSerializer(required=False, default=None)
class ExportParamSerializer(serializers.Serializer):
interpolate_key_frames = serializers.BooleanField(default=settings.INTERPOLATE_KEY_FRAMES,
help_text='Interpolate video key frames.',
required=False)
download_resources = serializers.BooleanField(default=settings.CONVERTER_DOWNLOAD_RESOURCES,
help_text='Download resources in converter.',
required=False)
# deprecated param to delete
export_type = serializers.CharField(default='JSON',
help_text='Export file format.',
required=False)
exportType = serializers.CharField(help_text='Export file format.',
required=False)
download_all_tasks = serializers.BooleanField(default=False,
help_text='Download all tasks or only finished.',
required=False)
ExportDataSerializer = load_func(settings.EXPORT_DATA_SERIALIZER)
