def get_document_loading_progress(self) -> Dict[str, int]: from apps.task.tasks import LoadDocuments # from apps.project.tasks import LoadArchive load_tasks_total = Task.objects.filter( name__in=[LoadDocuments.name], upload_session_id=self.pk).count() docs = self.document_set.all() doc_type = self.project.type docs_count = docs.count() docs_count = max(load_tasks_total, docs_count) doc_ids = list(docs.values_list('pk', flat=True)) docs_cached = 0 if doc_ids: doc_ids_str = ','.join([str(id) for id in doc_ids]) cache_table_name = doc_fields_table_name(doc_type.code) with connection.cursor() as cursor: cursor.execute(f'''SELECT COUNT(*) FROM "{cache_table_name}" WHERE document_id IN ({doc_ids_str});''') row = cursor.fetchone() docs_cached = row[0] # total progress is a weighted sum result = {'task_count': load_tasks_total, 'docs_count': docs_count, 'docs_cached': docs_cached} progress = 0 if docs_count > 0: numerator = result['docs_cached'] progress = round(100 * numerator / docs_count, 1) result['progress'] = progress return result
def get_non_indexed_doc_ids_not_planned_to_index_by_predicate( doc_type_code: str, predicate: str, pack_size: int = 100) -> Generator[List[int], None, None]: table_name = doc_fields_table_name(doc_type_code) with connection.cursor() as cursor: # return documents of the specified type which # - do not exist in the corresponding fields cache # - have no planned but not-started reindex tasks on them cursor.execute( 'select dd.id \n' 'from document_document dd \n' f'left outer join "{table_name}" df on dd.id = df.document_id \n' 'left outer join lateral (select jsonb_array_elements(args->0) doc_id \n' ' from task_task \n' f" where name = '{_get_reindex_task_name()}' \n" " and own_status = 'PENDING'\n" ' and date_work_start is null) tt on tt.doc_id = to_jsonb(dd.id) \n' f'where {predicate} and df.document_id is null and tt.doc_id is null \n' 'and dd.processed is true') rows = cursor.fetchmany(pack_size) while rows: yield [row[0] for row in rows] rows = cursor.fetchmany(pack_size)
def do_rename(apps, schema_editor): DocumentField = apps.get_model('document', 'DocumentField') to_rename = list() # type: List[Tuple[str, str, str]] for field in DocumentField.objects.filter(type='ratio'): table_name = doc_fields_table_name(field.document_type.code) handler = RatioRawdbFieldHandler(field.code, field.type, field.title, table_name) to_rename.append( (table_name, escape_column_name(handler.field_column_name_base + '_con'), escape_column_name(handler.field_column_name_base + '_den'))) for table_name, column_from, column_to in to_rename: with connection.cursor() as cursor: cursor.execute(f''' DO $$ BEGIN IF EXISTS(SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' and column_name='{column_from}') AND NOT EXISTS(SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' and column_name='{column_to}') THEN ALTER TABLE {table_name} RENAME COLUMN {column_from} TO {column_to}; END IF; END $$; ''')
def do_migrate(apps, schema_editor): DocumentType = apps.get_model('document', 'DocumentType') with connection.cursor() as cursor: for document_type in DocumentType.objects.all(): table_name = doc_fields_table_name( document_type_code=document_type.code) cursor.execute('DROP TABLE IF EXISTS "{table_name}"'.format( table_name=table_name))
def there_are_non_indexed_docs_not_planned_to_index( document_type: DocumentType, log: ProcessLogger) -> bool: for doc_id in non_indexed_doc_ids_not_planned_to_index_by_doc_type(document_type, 1): if doc_id: task_name = _get_reindex_task_name() fields_table = doc_fields_table_name(document_type.code) log.info(f'there_are_non_indexed_docs_not_planned_to_index: ' f'found document id={doc_id} of type {document_type.code}, ' f'task {task_name}. Fields table: {fields_table}') return True return False