def notify_active_upload_sessions(sessions: List[UploadSession]): """ Notify users about active upload sessions to show/track progress """ data = [] for session in sessions: progress = session.document_tasks_progress_total document_progress_details = session.document_tasks_progress() all_documents = len(document_progress_details) processed_documents = sum([ 1 for k, v in document_progress_details.items() if v.get('tasks_overall_status') in READY_STATES ]) unprocessed_documents = all_documents - processed_documents data.append({ 'session_id': session.pk, 'project_id': session.project.pk, 'user_id': session.created_by.pk if session.created_by else None, 'user_name': session.created_by.get_full_name() if session.created_by else None, 'progress': progress, 'processed_documents': processed_documents, 'unprocessed_documents': unprocessed_documents, 'completed': bool(session.completed), 'created_date': session.created_date }) message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data) admins_and_managers = User.objects.qs_admins_and_managers() Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message) for reviewer in User.objects.filter(role__is_admin=False, role__is_manager=False): _data = [ i for i in data if User.objects.filter(pk=reviewer.pk).filter( Q(project_owners__pk=i['project_id']) | Q(project_reviewers__pk=i['project_id']) | Q(project_super_reviewers__pk=i['project_id'])).exists() ] if _data: message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data) Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
def notify_active_upload_sessions(sessions: List[UploadSession]): """ Notify users about active upload sessions to show/track progress """ data = [] for session in sessions: progress = session.document_tasks_progress_total document_progress_details = session.document_tasks_progress() all_documents = len(document_progress_details) processed_documents = sum([1 for k, v in document_progress_details.items() if v.get('tasks_overall_status') in READY_STATES]) unprocessed_documents = all_documents - processed_documents session_data = {'session_id': session.pk, 'project_id': session.project.pk, 'user_id': session.created_by.pk if session.created_by else None, 'user_name': session.created_by.get_full_name() if session.created_by else None, 'progress': progress, 'processed_documents': processed_documents, 'unprocessed_documents': unprocessed_documents, 'completed': bool(session.completed), 'created_date': session.created_date} from apps.project.tasks import LoadArchive session_archive_tasks = session.task_set.filter(name=LoadArchive.name, progress=100, metadata__progress_sent=False) if session_archive_tasks.exists(): archive_tasks_progress_data = list() for task in session_archive_tasks.all(): archive_tasks_progress_data.append( dict(archive_name=task.metadata.get('file_name'), arhive_progress=task.metadata.get('progress'))) session_data['archive_data'] = archive_tasks_progress_data for task in session_archive_tasks: task.metadata['progress_sent'] = True task.save() data.append(session_data) message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data) admins_and_managers = User.objects.qs_admins_and_managers() Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message) for reviewer in User.objects.filter(role__is_admin=False, role__is_manager=False): _data = [i for i in data if User.objects.filter(pk=reviewer.pk).filter( Q(project_owners__pk=i['project_id']) | Q(project_reviewers__pk=i['project_id']) | Q(project_super_reviewers__pk=i['project_id'])).exists()] if _data: message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data) Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
async def _send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage): """ Send the message to the users returned by the specified Django query set. This is an async method made private for calling it from the sync public method. :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..). :param message_obj: Message to send. :return: """ connected_user_ids = self.get_connected_users() if not connected_user_ids: return # A workaround for "connection already closed" problem. # Looks like this code is being executed in a way that # the "connection" object it accesses is re-used for a long time and appears broken after some long delay. connection.close() layer = get_channel_layer() # type: RedisChannelLayer msg = {'type': 'send_to_client', 'message': message_obj.to_dict()} coros = list() for user_id in qs_users.filter(pk__in=connected_user_ids).values_list( 'pk', flat=True): send_to_user_coro = layer.group_send( self.user_id_to_group_name(user_id), msg) coros.append(send_to_user_coro) await asyncio.gather(*coros)
def document_field_detection_failed_impl(sender, signal, document: Document, document_field: DocumentField, message: str, document_initial_load: bool): if document_initial_load: # WS notifications for failed detection are disabled when # document is being processed for the first time return project_id = document.project_id data = { 'document': { 'id': document.pk, 'name': document.name, 'project_id': project_id, 'project_name': document.project.name, }, 'field': { 'id': document_field.uid, 'code': document_field.code, 'title': document_field.title }, 'message': message } chan_msg = ChannelMessage(message_types.CHANNEL_MSG_TYPE_DETECTION_FAILED, data) users = User.get_users_for_object(object_pk=project_id, object_model=Project, perm_name='add_project_document') Websockets().send_to_users(qs_users=users, message_obj=chan_msg)
def notify_failed_load_document(file_name, kwargs): """ Notify users about failed LoadDocument tasks """ from apps.project.models import UploadSession project_id = UploadSession.objects.get(pk=kwargs['session_id']).project_id data = dict(file_name=file_name, project_id=project_id, upload_session_id=kwargs['session_id'], directory_path=kwargs['directory_path']) message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data) admins_and_managers = User.objects.qs_admins_and_managers() Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message) non_admins = User.objects \ .filter(role__is_admin=False, role__is_manager=False) \ .filter(Q(project_owners__pk=project_id) | Q(project_reviewers__pk=project_id) | Q(project_super_reviewers__pk=project_id)) if non_admins.exists(): Websockets().send_to_users(qs_users=non_admins, message_obj=message)
def notify_cancelled_upload_session(session, user_id): """ Notify users about cancelled upload session """ cancelled_by_user = User.objects.get(pk=user_id) data = { 'session_id': session.pk, 'project_id': session.project_id, 'cancelled_by_user_id': cancelled_by_user.pk if cancelled_by_user else None, 'cancelled_by_user_name': cancelled_by_user.get_full_name() if cancelled_by_user else None } message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data) admins_and_managers = User.objects.qs_admins_and_managers() Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message) non_admins = User.objects \ .filter(role__is_admin=False, role__is_manager=False) \ .filter(Q(project_owners__pk=session.project_id) | Q(project_reviewers__pk=session.project_id) | Q(project_super_reviewers__pk=session.project_id)) if non_admins.exists(): Websockets().send_to_users(qs_users=non_admins, message_obj=message)
def notify_field_annotation_saved(instance: FieldAnnotation): message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_SAVED, { 'annotation': _annotation_to_dto(instance), 'user': _get_user_dto(instance) }) notify_on_document_changes(instance.document.pk, message)
def send_to_all_users(self, message_obj: ChannelMessage): """ Send the message to all connected users. Each authenticated user is added to a special ALL group and this method sends the message into this group. :param message_obj: :return: """ layer = get_channel_layer() # type: RedisChannelLayer async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client', 'message': message_obj.to_dict()})
def notify_field_annotation_deleted(instance: FieldAnnotation): if not instance.document.processed: return message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, { 'annotation': _annotation_to_dto(instance), 'user': _get_user_dto(instance) }) notify_on_document_changes(instance.document.pk, message)
def notify_active_pdf2pdfa_tasks(data): """ Notify users about active upload sessions to show/track progress """ user_ids = set([i['user_id'] for i in data]) for user_id in user_ids: user_data = [i for i in data if i['user_id'] == user_id] message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_PDF2PDFA_TASKS, user_data) Websockets().send_to_user(user_id=user_id, message_obj=message)
def send_to_user(self, user_id, message_obj: ChannelMessage): """ Send the message to the specified user. :param user_id: ID of the user. :param message_obj: Message to send. :return: """ layer = get_channel_layer() # type: RedisChannelLayer async_to_sync(layer.group_send)(self.user_id_to_group_name(user_id), {'type': 'send_to_client', 'message': message_obj.to_dict()})
def notify_field_annotation_deleted(instance: FieldAnnotation): try: message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, { 'annotation': _annotation_to_dto(instance), 'user': _get_user_dto(instance) }) notify_on_document_changes(instance.document.pk, message) except ObjectDoesNotExist: logger = get_django_logger() logger.warning(f'notify_field_annotation_deleted is called for ' f'field {instance.field_id}, that was probably deleted') return
def send_to_users_by_ids(self, user_ids: List[int], message_obj: ChannelMessage): """ Send the message to the users returned by the specified Django query set. :param user_ids: user ids to send messages to :param message_obj: Message to send. :return: """ connected_user_ids = self.get_connected_users() if not connected_user_ids: return layer = get_channel_layer() # type: RedisChannelLayer async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client', 'user_ids': user_ids, 'message': message_obj.to_dict()})
def notify_active_upload_sessions(sessions: List[UploadSession]): """ Notify users about active upload sessions to show/track progress """ data = [] for session in sessions: # calculate progress baed on stored entities session_data = get_session_data_by_document_query(session) data.append(session_data) from guardian.shortcuts import get_objects_for_user for user in User.objects.filter(is_active=True): user_project_ids = list(get_objects_for_user(user, 'project.add_project_document', Project) .values_list('pk', flat=True)) user_data = [i for i in data if i['project_id'] in user_project_ids] message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, user_data) Websockets().send_to_user(user_id=user.pk, message_obj=message)
def send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage): """ Send the message to the users returned by the specified Django query set. :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..). :param message_obj: Message to send. :return: """ connected_user_ids = self.get_connected_users() if not connected_user_ids: return user_ids: Set[int] = set(qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True)) layer = get_channel_layer() # type: RedisChannelLayer async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client', 'user_ids': list(user_ids), 'message': message_obj.to_dict()})
def notify_user_on_document_values_changed( document: Document, document_fields_before: Dict[str, Any], document_fields_after: Dict[str, Any], field_handlers: List[RawdbFieldHandler], changed_by_user: User) -> None: if not document_fields_after or not document.processed: return allowed_fields = {h.field_code for h in field_handlers if not h.is_annotation} field_changed = {} for field in document_fields_after: if field not in allowed_fields: continue if field.endswith(FIELD_CODE_ANNOTATION_SUFFIX): continue old_val = document_fields_before.get(field) \ if document_fields_before else None new_val = document_fields_after[field] if old_val == new_val: continue field_changed[field] = {'new_val': new_val, 'old_val': old_val} if not field_changed: return # notify clients through WS payload = { 'fields': field_changed, 'document_id': document.pk, 'project_id': document.project.pk, 'project_name': document.project.name, } if changed_by_user: payload['user'] = { 'id': changed_by_user.pk, 'first_name': changed_by_user.first_name, 'last_name': changed_by_user.last_name, 'login': changed_by_user.username } message = ChannelMessage(CHANNEL_MSG_TYPE_FIELDS_UPDATED, payload) notify_on_document_changes(document.pk, message)
def notify_cancelled_upload_session(session, user_id): """ Notify users about cancelled upload session """ cancelled_by_user = User.objects.get(pk=user_id) data = {'session_id': session.pk, 'project_id': session.project_id, 'cancelled_by_user_id': cancelled_by_user.pk if cancelled_by_user else None, 'cancelled_by_user_name': cancelled_by_user.name if cancelled_by_user else None} message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data) users = User.get_users_for_object( object_pk=session.project_id, object_model=Project, perm_name='add_project_document') Websockets().send_to_users(qs_users=users, message_obj=message)
async def _send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage): """ Send the message to the users returned by the specified Django query set. This is an async method made private for calling it from the sync public method. :param qs_users: Django query set returning User models. Pk field will be requested via values_list(..). :param message_obj: Message to send. :return: """ connected_user_ids = self.get_connected_users() if not connected_user_ids: return layer = get_channel_layer() # type: RedisChannelLayer msg = {'type': 'send_to_client', 'message': message_obj.to_dict()} coros = list() for user_id in qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True): send_to_user_coro = layer.group_send(self.user_id_to_group_name(user_id), msg) coros.append(send_to_user_coro) await asyncio.gather(*coros)
def notify_failed_load_document(file_name, session_id, directory_path): """ Notify users about failed LoadDocument tasks """ project_id = UploadSession.objects.get(pk=session_id).project_id data = dict( file_name=file_name, project_id=project_id, upload_session_id=session_id, directory_path=directory_path ) message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data) users = User.get_users_for_object( object_pk=project_id, object_model=Project, perm_name='add_project_document').values_list('pk', flat=True) Websockets().send_to_users(qs_users=users, message_obj=message)
def _notify_field_value_saved(instance: FieldValue, deleted=False): if not instance.document.processed: return field_value = { 'document': instance.document_id, 'project_id': instance.document.project_id, 'field__code': instance.field.code, 'value': instance.value if not deleted else None } annotation_stats = DocumentFieldRepository( ).get_annotation_stats_by_field_value(instance) message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, { 'field_value': field_value, 'annotation_stats': annotation_stats, 'user': _get_user_dto(instance) }) notify_on_document_changes(instance.document.pk, message)
def _notify_field_value_saved(instance: FieldValue, deleted=False): try: field_value = { 'document': instance.document_id, 'project_id': instance.document.project_id, 'field__code': instance.field.code, 'value': instance.value if not deleted else None } except ObjectDoesNotExist: logger = get_django_logger() logger.warning(f'_notify_field_value_saved is called for ' f'field {instance.field_id}, that was probably deleted') return annotation_stats = DocumentFieldRepository( ).get_annotation_stats_by_field_value(instance) message = ChannelMessage( message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, { 'field_value': field_value, 'annotation_stats': annotation_stats, 'user': _get_user_dto(instance) }) notify_on_document_changes(instance.document.pk, message)