Exemplo n.º 1
0
def notify_active_upload_sessions(sessions: List[UploadSession]):
    """
    Notify users about active upload sessions to show/track progress
    """
    data = []
    for session in sessions:
        progress = session.document_tasks_progress_total
        document_progress_details = session.document_tasks_progress()
        all_documents = len(document_progress_details)
        processed_documents = sum([
            1 for k, v in document_progress_details.items()
            if v.get('tasks_overall_status') in READY_STATES
        ])
        unprocessed_documents = all_documents - processed_documents

        data.append({
            'session_id':
            session.pk,
            'project_id':
            session.project.pk,
            'user_id':
            session.created_by.pk if session.created_by else None,
            'user_name':
            session.created_by.get_full_name() if session.created_by else None,
            'progress':
            progress,
            'processed_documents':
            processed_documents,
            'unprocessed_documents':
            unprocessed_documents,
            'completed':
            bool(session.completed),
            'created_date':
            session.created_date
        })

    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data)

    admins_and_managers = User.objects.qs_admins_and_managers()
    Websockets().send_to_users(qs_users=admins_and_managers,
                               message_obj=message)

    for reviewer in User.objects.filter(role__is_admin=False,
                                        role__is_manager=False):
        _data = [
            i for i in data if User.objects.filter(pk=reviewer.pk).filter(
                Q(project_owners__pk=i['project_id'])
                | Q(project_reviewers__pk=i['project_id'])
                | Q(project_super_reviewers__pk=i['project_id'])).exists()
        ]
        if _data:
            message = ChannelMessage(
                message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data)
            Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
Exemplo n.º 2
0
def notify_active_upload_sessions(sessions: List[UploadSession]):
    """
    Notify users about active upload sessions to show/track progress
    """
    data = []
    for session in sessions:
        progress = session.document_tasks_progress_total
        document_progress_details = session.document_tasks_progress()
        all_documents = len(document_progress_details)
        processed_documents = sum([1 for k, v in document_progress_details.items() if v.get('tasks_overall_status') in READY_STATES])
        unprocessed_documents = all_documents - processed_documents

        session_data = {'session_id': session.pk,
                        'project_id': session.project.pk,
                        'user_id': session.created_by.pk if session.created_by else None,
                        'user_name': session.created_by.get_full_name() if session.created_by else None,
                        'progress': progress,
                        'processed_documents': processed_documents,
                        'unprocessed_documents': unprocessed_documents,
                        'completed': bool(session.completed),
                        'created_date': session.created_date}
        from apps.project.tasks import LoadArchive
        session_archive_tasks = session.task_set.filter(name=LoadArchive.name,
                                                        progress=100,
                                                        metadata__progress_sent=False)
        if session_archive_tasks.exists():
            archive_tasks_progress_data = list()
            for task in session_archive_tasks.all():
                archive_tasks_progress_data.append(
                    dict(archive_name=task.metadata.get('file_name'),
                         arhive_progress=task.metadata.get('progress')))
            session_data['archive_data'] = archive_tasks_progress_data
            for task in session_archive_tasks:
                task.metadata['progress_sent'] = True
                task.save()
        data.append(session_data)

    message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data)

    admins_and_managers = User.objects.qs_admins_and_managers()
    Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message)

    for reviewer in User.objects.filter(role__is_admin=False, role__is_manager=False):
        _data = [i for i in data
                 if User.objects.filter(pk=reviewer.pk).filter(
                Q(project_owners__pk=i['project_id']) |
                Q(project_reviewers__pk=i['project_id']) |
                Q(project_super_reviewers__pk=i['project_id'])).exists()]
        if _data:
            message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data)
            Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
Exemplo n.º 3
0
    async def _send_to_users(self, qs_users: QuerySet,
                             message_obj: ChannelMessage):
        """
        Send the message to the users returned by the specified Django query set.

        This is an async method made private for calling it from the sync public method.
        :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
        :param message_obj: Message to send.
        :return:
        """
        connected_user_ids = self.get_connected_users()
        if not connected_user_ids:
            return

        # A workaround for "connection already closed" problem.
        # Looks like this code is being executed in a way that
        # the "connection" object it accesses is re-used for a long time and appears broken after some long delay.
        connection.close()

        layer = get_channel_layer()  # type: RedisChannelLayer
        msg = {'type': 'send_to_client', 'message': message_obj.to_dict()}
        coros = list()
        for user_id in qs_users.filter(pk__in=connected_user_ids).values_list(
                'pk', flat=True):
            send_to_user_coro = layer.group_send(
                self.user_id_to_group_name(user_id), msg)
            coros.append(send_to_user_coro)
        await asyncio.gather(*coros)
def document_field_detection_failed_impl(sender, signal, document: Document,
                                         document_field: DocumentField,
                                         message: str,
                                         document_initial_load: bool):
    if document_initial_load:
        # WS notifications for failed detection are disabled when
        # document is being processed for the first time
        return
    project_id = document.project_id
    data = {
        'document': {
            'id': document.pk,
            'name': document.name,
            'project_id': project_id,
            'project_name': document.project.name,
        },
        'field': {
            'id': document_field.uid,
            'code': document_field.code,
            'title': document_field.title
        },
        'message': message
    }
    chan_msg = ChannelMessage(message_types.CHANNEL_MSG_TYPE_DETECTION_FAILED,
                              data)

    users = User.get_users_for_object(object_pk=project_id,
                                      object_model=Project,
                                      perm_name='add_project_document')

    Websockets().send_to_users(qs_users=users, message_obj=chan_msg)
Exemplo n.º 5
0
def notify_failed_load_document(file_name, kwargs):
    """
    Notify users about failed LoadDocument tasks
    """
    from apps.project.models import UploadSession
    project_id = UploadSession.objects.get(pk=kwargs['session_id']).project_id

    data = dict(file_name=file_name,
                project_id=project_id,
                upload_session_id=kwargs['session_id'],
                directory_path=kwargs['directory_path'])
    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data)

    admins_and_managers = User.objects.qs_admins_and_managers()
    Websockets().send_to_users(qs_users=admins_and_managers,
                               message_obj=message)

    non_admins = User.objects \
        .filter(role__is_admin=False, role__is_manager=False) \
        .filter(Q(project_owners__pk=project_id) |
                Q(project_reviewers__pk=project_id) |
                Q(project_super_reviewers__pk=project_id))
    if non_admins.exists():
        Websockets().send_to_users(qs_users=non_admins, message_obj=message)
Exemplo n.º 6
0
def notify_cancelled_upload_session(session, user_id):
    """
    Notify users about cancelled upload session
    """
    cancelled_by_user = User.objects.get(pk=user_id)

    data = {
        'session_id':
        session.pk,
        'project_id':
        session.project_id,
        'cancelled_by_user_id':
        cancelled_by_user.pk if cancelled_by_user else None,
        'cancelled_by_user_name':
        cancelled_by_user.get_full_name() if cancelled_by_user else None
    }

    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data)

    admins_and_managers = User.objects.qs_admins_and_managers()
    Websockets().send_to_users(qs_users=admins_and_managers,
                               message_obj=message)

    non_admins = User.objects \
        .filter(role__is_admin=False, role__is_manager=False) \
        .filter(Q(project_owners__pk=session.project_id) |
                Q(project_reviewers__pk=session.project_id) |
                Q(project_super_reviewers__pk=session.project_id))
    if non_admins.exists():
        Websockets().send_to_users(qs_users=non_admins, message_obj=message)
Exemplo n.º 7
0
def notify_field_annotation_saved(instance: FieldAnnotation):
    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_SAVED, {
            'annotation': _annotation_to_dto(instance),
            'user': _get_user_dto(instance)
        })
    notify_on_document_changes(instance.document.pk, message)
Exemplo n.º 8
0
 def send_to_all_users(self, message_obj: ChannelMessage):
     """
     Send the message to all connected users.
     Each authenticated user is added to a special ALL group and this method sends the message into this group.
     :param message_obj:
     :return:
     """
     layer = get_channel_layer()  # type: RedisChannelLayer
     async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client', 'message': message_obj.to_dict()})
Exemplo n.º 9
0
def notify_field_annotation_deleted(instance: FieldAnnotation):
    if not instance.document.processed:
        return
    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, {
            'annotation': _annotation_to_dto(instance),
            'user': _get_user_dto(instance)
        })
    notify_on_document_changes(instance.document.pk, message)
Exemplo n.º 10
0
def notify_active_pdf2pdfa_tasks(data):
    """
    Notify users about active upload sessions to show/track progress
    """
    user_ids = set([i['user_id'] for i in data])
    for user_id in user_ids:
        user_data = [i for i in data if i['user_id'] == user_id]
        message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_PDF2PDFA_TASKS, user_data)
        Websockets().send_to_user(user_id=user_id, message_obj=message)
Exemplo n.º 11
0
 def send_to_user(self, user_id, message_obj: ChannelMessage):
     """
     Send the message to the specified user.
     :param user_id: ID of the user.
     :param message_obj: Message to send.
     :return:
     """
     layer = get_channel_layer()  # type: RedisChannelLayer
     async_to_sync(layer.group_send)(self.user_id_to_group_name(user_id),
                                     {'type': 'send_to_client', 'message': message_obj.to_dict()})
Exemplo n.º 12
0
def notify_field_annotation_deleted(instance: FieldAnnotation):
    try:
        message = ChannelMessage(
            message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, {
                'annotation': _annotation_to_dto(instance),
                'user': _get_user_dto(instance)
            })
        notify_on_document_changes(instance.document.pk, message)
    except ObjectDoesNotExist:
        logger = get_django_logger()
        logger.warning(f'notify_field_annotation_deleted is called for '
                       f'field {instance.field_id}, that was probably deleted')
        return
Exemplo n.º 13
0
    def send_to_users_by_ids(self, user_ids: List[int], message_obj: ChannelMessage):
        """
        Send the message to the users returned by the specified Django query set.

        :param user_ids: user ids to send messages to
        :param message_obj: Message to send.
        :return:
        """

        connected_user_ids = self.get_connected_users()
        if not connected_user_ids:
            return
        layer = get_channel_layer()  # type: RedisChannelLayer
        async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client',
                                              'user_ids': user_ids,
                                              'message': message_obj.to_dict()})
Exemplo n.º 14
0
def notify_active_upload_sessions(sessions: List[UploadSession]):
    """
    Notify users about active upload sessions to show/track progress
    """
    data = []
    for session in sessions:
        # calculate progress baed on stored entities
        session_data = get_session_data_by_document_query(session)
        data.append(session_data)

    from guardian.shortcuts import get_objects_for_user
    for user in User.objects.filter(is_active=True):
        user_project_ids = list(get_objects_for_user(user, 'project.add_project_document', Project)
                                .values_list('pk', flat=True))
        user_data = [i for i in data if i['project_id'] in user_project_ids]
        message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, user_data)
        Websockets().send_to_user(user_id=user.pk, message_obj=message)
Exemplo n.º 15
0
    def send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage):
        """
        Send the message to the users returned by the specified Django query set.

        :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
        :param message_obj: Message to send.
        :return:
        """

        connected_user_ids = self.get_connected_users()
        if not connected_user_ids:
            return
        user_ids: Set[int] = set(qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True))

        layer = get_channel_layer()  # type: RedisChannelLayer
        async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client',
                                              'user_ids': list(user_ids),
                                              'message': message_obj.to_dict()})
    def notify_user_on_document_values_changed(
            document: Document,
            document_fields_before: Dict[str, Any],
            document_fields_after: Dict[str, Any],
            field_handlers: List[RawdbFieldHandler],
            changed_by_user: User) -> None:
        if not document_fields_after or not document.processed:
            return
        allowed_fields = {h.field_code for h in field_handlers
                          if not h.is_annotation}

        field_changed = {}
        for field in document_fields_after:
            if field not in allowed_fields:
                continue
            if field.endswith(FIELD_CODE_ANNOTATION_SUFFIX):
                continue
            old_val = document_fields_before.get(field) \
                if document_fields_before else None
            new_val = document_fields_after[field]
            if old_val == new_val:
                continue
            field_changed[field] = {'new_val': new_val, 'old_val': old_val}

        if not field_changed:
            return

        # notify clients through WS
        payload = {
            'fields': field_changed,
            'document_id': document.pk,
            'project_id': document.project.pk,
            'project_name': document.project.name,
        }
        if changed_by_user:
            payload['user'] = {
                'id': changed_by_user.pk,
                'first_name': changed_by_user.first_name,
                'last_name': changed_by_user.last_name,
                'login': changed_by_user.username
            }
        message = ChannelMessage(CHANNEL_MSG_TYPE_FIELDS_UPDATED,
                                 payload)
        notify_on_document_changes(document.pk, message)
Exemplo n.º 17
0
def notify_cancelled_upload_session(session, user_id):
    """
    Notify users about cancelled upload session
    """
    cancelled_by_user = User.objects.get(pk=user_id)

    data = {'session_id': session.pk,
            'project_id': session.project_id,
            'cancelled_by_user_id': cancelled_by_user.pk if cancelled_by_user else None,
            'cancelled_by_user_name': cancelled_by_user.name if cancelled_by_user else None}

    message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data)

    users = User.get_users_for_object(
        object_pk=session.project_id,
        object_model=Project,
        perm_name='add_project_document')

    Websockets().send_to_users(qs_users=users, message_obj=message)
Exemplo n.º 18
0
    async def _send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage):
        """
        Send the message to the users returned by the specified Django query set.

        This is an async method made private for calling it from the sync public method.
        :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
        :param message_obj: Message to send.
        :return:
        """
        connected_user_ids = self.get_connected_users()
        if not connected_user_ids:
            return

        layer = get_channel_layer()  # type: RedisChannelLayer
        msg = {'type': 'send_to_client', 'message': message_obj.to_dict()}
        coros = list()
        for user_id in qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True):
            send_to_user_coro = layer.group_send(self.user_id_to_group_name(user_id), msg)
            coros.append(send_to_user_coro)
        await asyncio.gather(*coros)
Exemplo n.º 19
0
def notify_failed_load_document(file_name, session_id, directory_path):
    """
    Notify users about failed LoadDocument tasks
    """
    project_id = UploadSession.objects.get(pk=session_id).project_id

    data = dict(
        file_name=file_name,
        project_id=project_id,
        upload_session_id=session_id,
        directory_path=directory_path
    )
    message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data)

    users = User.get_users_for_object(
        object_pk=project_id,
        object_model=Project,
        perm_name='add_project_document').values_list('pk', flat=True)

    Websockets().send_to_users(qs_users=users, message_obj=message)
Exemplo n.º 20
0
def _notify_field_value_saved(instance: FieldValue, deleted=False):
    if not instance.document.processed:
        return
    field_value = {
        'document': instance.document_id,
        'project_id': instance.document.project_id,
        'field__code': instance.field.code,
        'value': instance.value if not deleted else None
    }

    annotation_stats = DocumentFieldRepository(
    ).get_annotation_stats_by_field_value(instance)

    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, {
            'field_value': field_value,
            'annotation_stats': annotation_stats,
            'user': _get_user_dto(instance)
        })
    notify_on_document_changes(instance.document.pk, message)
Exemplo n.º 21
0
def _notify_field_value_saved(instance: FieldValue, deleted=False):
    try:
        field_value = {
            'document': instance.document_id,
            'project_id': instance.document.project_id,
            'field__code': instance.field.code,
            'value': instance.value if not deleted else None
        }
    except ObjectDoesNotExist:
        logger = get_django_logger()
        logger.warning(f'_notify_field_value_saved is called for '
                       f'field {instance.field_id}, that was probably deleted')
        return

    annotation_stats = DocumentFieldRepository(
    ).get_annotation_stats_by_field_value(instance)

    message = ChannelMessage(
        message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, {
            'field_value': field_value,
            'annotation_stats': annotation_stats,
            'user': _get_user_dto(instance)
        })
    notify_on_document_changes(instance.document.pk, message)