def post(self, request, *args, **kwargs):
_call_task_func(clean_tasks, (),
request.user.pk,
queue=settings.CELERY_QUEUE_SERIAL)
return self.json_response('Cleaning task started.')
def process(self, **kwargs):
session_id = kwargs['session_id']
session = UploadSession.objects.get(pk=session_id)
# 1. Purge Tasks
self.track_timelog('')
session_tasks = Task.objects.main_tasks().filter(metadata__session_id=session_id)
self.log_info(f'Purge {session_tasks.count()} session tasks.')
for a_task in session_tasks:
try:
purge_task(a_task.id, log_func=lambda m: self.log_info(m))
except:
# case when task is already deleted as subtask
pass
self.track_timelog('1 - purge tasks')
# 2. Remove Documents+
document_ids = \
list(Document.objects.filter(upload_session_id=session_id).values_list('pk', flat=True))
self.log_info(f'Remove {len(document_ids)} documents')
from apps.document.repository.document_bulk_delete import get_document_bulk_delete
# TODO: WHY it fails with
# psycopg2.errors.ForeignKeyViolation: update or delete
# on table "document_textunit" violates foreign key constraint
attempts = 3
delay = 60
attempts_made = 0
delete_manager = get_document_bulk_delete()
error_logged = False
for attempt in range(1, attempts + 1):
attempts_made += 1
try:
delete_manager.delete_documents(document_ids)
break
except Exception as e:
if not error_logged:
self.log_error('Error while deleting documents', exc_info=e)
error_logged = True
self.log_info(f'Attempt #{attempt} of {attempts} to delete documents failed, retry')
time.sleep(delay)
self.track_timelog(f'2 - bulk delete for {len(document_ids)} documents')
if attempts_made > 1:
self.log_error(f'{attempts_made} of {attempts} tried to delete documents')
# 3. Remove files
file_storage_exists = file_storage.document_exists(session_id)
self.log_info(f'File Storage exists: {file_storage_exists}')
files_removed, failed_removing = (0, 0)
if file_storage_exists:
files = file_storage.list_documents(session_id)
self.log_info(f'Remove {len(files)} files from File Storage.')
for file_path in files:
file_storage.delete_document(file_path)
try:
file_storage.delete_document(session_id)
files_removed += 1
except:
# TODO: removing folders through LocalStorage is not implemented
failed_removing += 1
self.track_timelog(f'3 - remove files ({files_removed} removed, {failed_removing} failed)')
# 4. Remove Upload Session
if not session:
raise Exception(f"Couldn't find session by id ({session_id})")
self.log_info(f'Send WS notification that session uid="{session_id}" is cancelled.')
notify_cancelled_upload_session(session, kwargs.get('user_id'))
self.log_info(f'Remove session uid="{session_id}".')
project = session.project
session.delete()
self.track_timelog('4 - delete session')
# 5. Reindex Project
self.log_info(f'Reindex project id="{project.id}" documents.')
from apps.rawdb.tasks import reindex_all_project_documents
_call_task_func(reindex_all_project_documents, (project.pk,), None)
self.track_timelog('5 - reindex project')