def post(self, request, *args, **kwargs): _call_task_func(clean_tasks, (), request.user.pk, queue=settings.CELERY_QUEUE_SERIAL) return self.json_response('Cleaning task started.')
def process(self, **kwargs): session_id = kwargs['session_id'] session = UploadSession.objects.get(pk=session_id) # 1. Purge Tasks self.track_timelog('') session_tasks = Task.objects.main_tasks().filter(metadata__session_id=session_id) self.log_info(f'Purge {session_tasks.count()} session tasks.') for a_task in session_tasks: try: purge_task(a_task.id, log_func=lambda m: self.log_info(m)) except: # case when task is already deleted as subtask pass self.track_timelog('1 - purge tasks') # 2. Remove Documents+ document_ids = \ list(Document.objects.filter(upload_session_id=session_id).values_list('pk', flat=True)) self.log_info(f'Remove {len(document_ids)} documents') from apps.document.repository.document_bulk_delete import get_document_bulk_delete # TODO: WHY it fails with # psycopg2.errors.ForeignKeyViolation: update or delete # on table "document_textunit" violates foreign key constraint attempts = 3 delay = 60 attempts_made = 0 delete_manager = get_document_bulk_delete() error_logged = False for attempt in range(1, attempts + 1): attempts_made += 1 try: delete_manager.delete_documents(document_ids) break except Exception as e: if not error_logged: self.log_error('Error while deleting documents', exc_info=e) error_logged = True self.log_info(f'Attempt #{attempt} of {attempts} to delete documents failed, retry') time.sleep(delay) self.track_timelog(f'2 - bulk delete for {len(document_ids)} documents') if attempts_made > 1: self.log_error(f'{attempts_made} of {attempts} tried to delete documents') # 3. Remove files file_storage_exists = file_storage.document_exists(session_id) self.log_info(f'File Storage exists: {file_storage_exists}') files_removed, failed_removing = (0, 0) if file_storage_exists: files = file_storage.list_documents(session_id) self.log_info(f'Remove {len(files)} files from File Storage.') for file_path in files: file_storage.delete_document(file_path) try: file_storage.delete_document(session_id) files_removed += 1 except: # TODO: removing folders through LocalStorage is not implemented failed_removing += 1 self.track_timelog(f'3 - remove files ({files_removed} removed, {failed_removing} failed)') # 4. Remove Upload Session if not session: raise Exception(f"Couldn't find session by id ({session_id})") self.log_info(f'Send WS notification that session uid="{session_id}" is cancelled.') notify_cancelled_upload_session(session, kwargs.get('user_id')) self.log_info(f'Remove session uid="{session_id}".') project = session.project session.delete() self.track_timelog('4 - delete session') # 5. Reindex Project self.log_info(f'Reindex project id="{project.id}" documents.') from apps.rawdb.tasks import reindex_all_project_documents _call_task_func(reindex_all_project_documents, (project.pk,), None) self.track_timelog('5 - reindex project')