def notify_active_upload_sessions(sessions: List[UploadSession]):
"""
Notify users about active upload sessions to show/track progress
"""
data = []
for session in sessions:
progress = session.document_tasks_progress_total
document_progress_details = session.document_tasks_progress()
all_documents = len(document_progress_details)
processed_documents = sum([
1 for k, v in document_progress_details.items()
if v.get('tasks_overall_status') in READY_STATES
])
unprocessed_documents = all_documents - processed_documents
data.append({
'session_id':
session.pk,
'project_id':
session.project.pk,
'user_id':
session.created_by.pk if session.created_by else None,
'user_name':
session.created_by.get_full_name() if session.created_by else None,
'progress':
progress,
'processed_documents':
processed_documents,
'unprocessed_documents':
unprocessed_documents,
'completed':
bool(session.completed),
'created_date':
session.created_date
})
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data)
admins_and_managers = User.objects.qs_admins_and_managers()
Websockets().send_to_users(qs_users=admins_and_managers,
message_obj=message)
for reviewer in User.objects.filter(role__is_admin=False,
role__is_manager=False):
_data = [
i for i in data if User.objects.filter(pk=reviewer.pk).filter(
Q(project_owners__pk=i['project_id'])
| Q(project_reviewers__pk=i['project_id'])
| Q(project_super_reviewers__pk=i['project_id'])).exists()
]
if _data:
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data)
Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
def notify_active_upload_sessions(sessions: List[UploadSession]):
"""
Notify users about active upload sessions to show/track progress
"""
data = []
for session in sessions:
progress = session.document_tasks_progress_total
document_progress_details = session.document_tasks_progress()
all_documents = len(document_progress_details)
processed_documents = sum([1 for k, v in document_progress_details.items() if v.get('tasks_overall_status') in READY_STATES])
unprocessed_documents = all_documents - processed_documents
session_data = {'session_id': session.pk,
'project_id': session.project.pk,
'user_id': session.created_by.pk if session.created_by else None,
'user_name': session.created_by.get_full_name() if session.created_by else None,
'progress': progress,
'processed_documents': processed_documents,
'unprocessed_documents': unprocessed_documents,
'completed': bool(session.completed),
'created_date': session.created_date}
from apps.project.tasks import LoadArchive
session_archive_tasks = session.task_set.filter(name=LoadArchive.name,
progress=100,
metadata__progress_sent=False)
if session_archive_tasks.exists():
archive_tasks_progress_data = list()
for task in session_archive_tasks.all():
archive_tasks_progress_data.append(
dict(archive_name=task.metadata.get('file_name'),
arhive_progress=task.metadata.get('progress')))
session_data['archive_data'] = archive_tasks_progress_data
for task in session_archive_tasks:
task.metadata['progress_sent'] = True
task.save()
data.append(session_data)
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, data)
admins_and_managers = User.objects.qs_admins_and_managers()
Websockets().send_to_users(qs_users=admins_and_managers, message_obj=message)
for reviewer in User.objects.filter(role__is_admin=False, role__is_manager=False):
_data = [i for i in data
if User.objects.filter(pk=reviewer.pk).filter(
Q(project_owners__pk=i['project_id']) |
Q(project_reviewers__pk=i['project_id']) |
Q(project_super_reviewers__pk=i['project_id'])).exists()]
if _data:
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, _data)
Websockets().send_to_user(user_id=reviewer.pk, message_obj=message)
async def _send_to_users(self, qs_users: QuerySet,
message_obj: ChannelMessage):
"""
Send the message to the users returned by the specified Django query set.
This is an async method made private for calling it from the sync public method.
:param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
:param message_obj: Message to send.
:return:
"""
connected_user_ids = self.get_connected_users()
if not connected_user_ids:
return
# A workaround for "connection already closed" problem.
# Looks like this code is being executed in a way that
# the "connection" object it accesses is re-used for a long time and appears broken after some long delay.
connection.close()
layer = get_channel_layer() # type: RedisChannelLayer
msg = {'type': 'send_to_client', 'message': message_obj.to_dict()}
coros = list()
for user_id in qs_users.filter(pk__in=connected_user_ids).values_list(
'pk', flat=True):
send_to_user_coro = layer.group_send(
self.user_id_to_group_name(user_id), msg)
coros.append(send_to_user_coro)
await asyncio.gather(*coros)
def document_field_detection_failed_impl(sender, signal, document: Document,
document_field: DocumentField,
message: str,
document_initial_load: bool):
if document_initial_load:
# WS notifications for failed detection are disabled when
# document is being processed for the first time
return
project_id = document.project_id
data = {
'document': {
'id': document.pk,
'name': document.name,
'project_id': project_id,
'project_name': document.project.name,
},
'field': {
'id': document_field.uid,
'code': document_field.code,
'title': document_field.title
},
'message': message
}
chan_msg = ChannelMessage(message_types.CHANNEL_MSG_TYPE_DETECTION_FAILED,
data)
users = User.get_users_for_object(object_pk=project_id,
object_model=Project,
perm_name='add_project_document')
Websockets().send_to_users(qs_users=users, message_obj=chan_msg)
def notify_failed_load_document(file_name, kwargs):
"""
Notify users about failed LoadDocument tasks
"""
from apps.project.models import UploadSession
project_id = UploadSession.objects.get(pk=kwargs['session_id']).project_id
data = dict(file_name=file_name,
project_id=project_id,
upload_session_id=kwargs['session_id'],
directory_path=kwargs['directory_path'])
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data)
admins_and_managers = User.objects.qs_admins_and_managers()
Websockets().send_to_users(qs_users=admins_and_managers,
message_obj=message)
non_admins = User.objects \
.filter(role__is_admin=False, role__is_manager=False) \
.filter(Q(project_owners__pk=project_id) |
Q(project_reviewers__pk=project_id) |
Q(project_super_reviewers__pk=project_id))
if non_admins.exists():
Websockets().send_to_users(qs_users=non_admins, message_obj=message)
def notify_cancelled_upload_session(session, user_id):
"""
Notify users about cancelled upload session
"""
cancelled_by_user = User.objects.get(pk=user_id)
data = {
'session_id':
session.pk,
'project_id':
session.project_id,
'cancelled_by_user_id':
cancelled_by_user.pk if cancelled_by_user else None,
'cancelled_by_user_name':
cancelled_by_user.get_full_name() if cancelled_by_user else None
}
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data)
admins_and_managers = User.objects.qs_admins_and_managers()
Websockets().send_to_users(qs_users=admins_and_managers,
message_obj=message)
non_admins = User.objects \
.filter(role__is_admin=False, role__is_manager=False) \
.filter(Q(project_owners__pk=session.project_id) |
Q(project_reviewers__pk=session.project_id) |
Q(project_super_reviewers__pk=session.project_id))
if non_admins.exists():
Websockets().send_to_users(qs_users=non_admins, message_obj=message)
def notify_field_annotation_saved(instance: FieldAnnotation):
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_SAVED, {
'annotation': _annotation_to_dto(instance),
'user': _get_user_dto(instance)
})
notify_on_document_changes(instance.document.pk, message)
def send_to_all_users(self, message_obj: ChannelMessage):
"""
Send the message to all connected users.
Each authenticated user is added to a special ALL group and this method sends the message into this group.
:param message_obj:
:return:
"""
layer = get_channel_layer() # type: RedisChannelLayer
async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client', 'message': message_obj.to_dict()})
def notify_field_annotation_deleted(instance: FieldAnnotation):
if not instance.document.processed:
return
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, {
'annotation': _annotation_to_dto(instance),
'user': _get_user_dto(instance)
})
notify_on_document_changes(instance.document.pk, message)
def notify_active_pdf2pdfa_tasks(data):
"""
Notify users about active upload sessions to show/track progress
"""
user_ids = set([i['user_id'] for i in data])
for user_id in user_ids:
user_data = [i for i in data if i['user_id'] == user_id]
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_PDF2PDFA_TASKS, user_data)
Websockets().send_to_user(user_id=user_id, message_obj=message)
def send_to_user(self, user_id, message_obj: ChannelMessage):
"""
Send the message to the specified user.
:param user_id: ID of the user.
:param message_obj: Message to send.
:return:
"""
layer = get_channel_layer() # type: RedisChannelLayer
async_to_sync(layer.group_send)(self.user_id_to_group_name(user_id),
{'type': 'send_to_client', 'message': message_obj.to_dict()})
def notify_field_annotation_deleted(instance: FieldAnnotation):
try:
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FIELD_ANNOTATION_DELETED, {
'annotation': _annotation_to_dto(instance),
'user': _get_user_dto(instance)
})
notify_on_document_changes(instance.document.pk, message)
except ObjectDoesNotExist:
logger = get_django_logger()
logger.warning(f'notify_field_annotation_deleted is called for '
f'field {instance.field_id}, that was probably deleted')
return
def send_to_users_by_ids(self, user_ids: List[int], message_obj: ChannelMessage):
"""
Send the message to the users returned by the specified Django query set.
:param user_ids: user ids to send messages to
:param message_obj: Message to send.
:return:
"""
connected_user_ids = self.get_connected_users()
if not connected_user_ids:
return
layer = get_channel_layer() # type: RedisChannelLayer
async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client',
'user_ids': user_ids,
'message': message_obj.to_dict()})
def notify_active_upload_sessions(sessions: List[UploadSession]):
"""
Notify users about active upload sessions to show/track progress
"""
data = []
for session in sessions:
# calculate progress baed on stored entities
session_data = get_session_data_by_document_query(session)
data.append(session_data)
from guardian.shortcuts import get_objects_for_user
for user in User.objects.filter(is_active=True):
user_project_ids = list(get_objects_for_user(user, 'project.add_project_document', Project)
.values_list('pk', flat=True))
user_data = [i for i in data if i['project_id'] in user_project_ids]
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_ACTIVE_UPLOAD_SESSIONS, user_data)
Websockets().send_to_user(user_id=user.pk, message_obj=message)
def send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage):
"""
Send the message to the users returned by the specified Django query set.
:param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
:param message_obj: Message to send.
:return:
"""
connected_user_ids = self.get_connected_users()
if not connected_user_ids:
return
user_ids: Set[int] = set(qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True))
layer = get_channel_layer() # type: RedisChannelLayer
async_to_sync(layer.group_send)(ALL, {'type': 'send_to_client',
'user_ids': list(user_ids),
'message': message_obj.to_dict()})
def notify_user_on_document_values_changed(
document: Document,
document_fields_before: Dict[str, Any],
document_fields_after: Dict[str, Any],
field_handlers: List[RawdbFieldHandler],
changed_by_user: User) -> None:
if not document_fields_after or not document.processed:
return
allowed_fields = {h.field_code for h in field_handlers
if not h.is_annotation}
field_changed = {}
for field in document_fields_after:
if field not in allowed_fields:
continue
if field.endswith(FIELD_CODE_ANNOTATION_SUFFIX):
continue
old_val = document_fields_before.get(field) \
if document_fields_before else None
new_val = document_fields_after[field]
if old_val == new_val:
continue
field_changed[field] = {'new_val': new_val, 'old_val': old_val}
if not field_changed:
return
# notify clients through WS
payload = {
'fields': field_changed,
'document_id': document.pk,
'project_id': document.project.pk,
'project_name': document.project.name,
}
if changed_by_user:
payload['user'] = {
'id': changed_by_user.pk,
'first_name': changed_by_user.first_name,
'last_name': changed_by_user.last_name,
'login': changed_by_user.username
}
message = ChannelMessage(CHANNEL_MSG_TYPE_FIELDS_UPDATED,
payload)
notify_on_document_changes(document.pk, message)
def notify_cancelled_upload_session(session, user_id):
"""
Notify users about cancelled upload session
"""
cancelled_by_user = User.objects.get(pk=user_id)
data = {'session_id': session.pk,
'project_id': session.project_id,
'cancelled_by_user_id': cancelled_by_user.pk if cancelled_by_user else None,
'cancelled_by_user_name': cancelled_by_user.name if cancelled_by_user else None}
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_CANCELLED_UPLOAD_SESSION, data)
users = User.get_users_for_object(
object_pk=session.project_id,
object_model=Project,
perm_name='add_project_document')
Websockets().send_to_users(qs_users=users, message_obj=message)
async def _send_to_users(self, qs_users: QuerySet, message_obj: ChannelMessage):
"""
Send the message to the users returned by the specified Django query set.
This is an async method made private for calling it from the sync public method.
:param qs_users: Django query set returning User models. Pk field will be requested via values_list(..).
:param message_obj: Message to send.
:return:
"""
connected_user_ids = self.get_connected_users()
if not connected_user_ids:
return
layer = get_channel_layer() # type: RedisChannelLayer
msg = {'type': 'send_to_client', 'message': message_obj.to_dict()}
coros = list()
for user_id in qs_users.filter(pk__in=connected_user_ids).values_list('pk', flat=True):
send_to_user_coro = layer.group_send(self.user_id_to_group_name(user_id), msg)
coros.append(send_to_user_coro)
await asyncio.gather(*coros)
def notify_failed_load_document(file_name, session_id, directory_path):
"""
Notify users about failed LoadDocument tasks
"""
project_id = UploadSession.objects.get(pk=session_id).project_id
data = dict(
file_name=file_name,
project_id=project_id,
upload_session_id=session_id,
directory_path=directory_path
)
message = ChannelMessage(message_types.CHANNEL_MSG_TYPE_FAILED_LOAD_DOCUMENT, data)
users = User.get_users_for_object(
object_pk=project_id,
object_model=Project,
perm_name='add_project_document').values_list('pk', flat=True)
Websockets().send_to_users(qs_users=users, message_obj=message)
def _notify_field_value_saved(instance: FieldValue, deleted=False):
if not instance.document.processed:
return
field_value = {
'document': instance.document_id,
'project_id': instance.document.project_id,
'field__code': instance.field.code,
'value': instance.value if not deleted else None
}
annotation_stats = DocumentFieldRepository(
).get_annotation_stats_by_field_value(instance)
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, {
'field_value': field_value,
'annotation_stats': annotation_stats,
'user': _get_user_dto(instance)
})
notify_on_document_changes(instance.document.pk, message)
def _notify_field_value_saved(instance: FieldValue, deleted=False):
try:
field_value = {
'document': instance.document_id,
'project_id': instance.document.project_id,
'field__code': instance.field.code,
'value': instance.value if not deleted else None
}
except ObjectDoesNotExist:
logger = get_django_logger()
logger.warning(f'_notify_field_value_saved is called for '
f'field {instance.field_id}, that was probably deleted')
return
annotation_stats = DocumentFieldRepository(
).get_annotation_stats_by_field_value(instance)
message = ChannelMessage(
message_types.CHANNEL_MSG_TYPE_FIELD_VALUE_SAVED, {
'field_value': field_value,
'annotation_stats': annotation_stats,
'user': _get_user_dto(instance)
})
notify_on_document_changes(instance.document.pk, message)