async def prepare_tables(self):
"""
Prepare tables structure for transferring data process
"""
logger.info('prepare tables structure for transferring process')
self.tables = {
f'{table_name}': DBTable(name=table_name, )
for table_name in self.table_names
}
chunks_table_names = make_chunks(
iterable=self.table_names,
size=settings.TABLES_LIMIT_PER_TRANSACTION,
is_list=True,
)
coroutines = [
self._prepare_chunk_tables(chunk_table_names=chunk_table_names, )
for chunk_table_names in chunks_table_names
]
if coroutines:
await asyncio.gather(*coroutines)
logger.info(f'prepare tables progress - {len(self.tables.keys())}/'
f'{len(self.table_names)}')
async def _direct_recursively_preparing_foreign_table(
self,
table: DBTable,
column: DBColumn,
need_transfer_pks: Iterable[int],
stack_tables: Set[DBTable],
):
"""
Recursively preparing foreign table
"""
need_transfer_pks_chunks = make_chunks(
iterable=need_transfer_pks,
size=self.CHUNK_SIZE,
is_list=True,
)
coroutines = [
self._direct_recursively_preparing_foreign_table_chunk(
table=table,
column=column,
need_transfer_pks_chunk=need_transfer_pks_chunk,
stack_tables=stack_tables,
)
for need_transfer_pks_chunk in need_transfer_pks_chunks
]
if coroutines:
await asyncio.wait(coroutines)
async def _revert_recursively_preparing_revert_table_column(
self,
revert_table: DBTable,
revert_column: DBColumn,
need_transfer_pks: Set[Union[int, str]],
):
"""
Recursively preparing revert table column
"""
need_transfer_pks_chunks = make_chunks(
iterable=need_transfer_pks,
size=self.CHUNK_SIZE,
is_list=True,
)
coroutines = [
self._revert_recursively_preparing_revert_table_column_chunk(
revert_table=revert_table,
revert_column=revert_column,
need_transfer_pks_chunk=need_transfer_pks_chunk,
)
for need_transfer_pks_chunk in need_transfer_pks_chunks
]
if coroutines:
await asyncio.wait(coroutines)
async def _transfer_table_data(self, table):
"""
Перенос данных таблицы
"""
logger.info(f"start transferring table \"{table.name}\", "
f"need to import - {len(table.need_transfer_pks)}")
need_import_ids_chunks = make_chunks(
iterable=table.need_transfer_pks,
size=self.CHUNK_SIZE,
)
for need_import_ids_chunk in need_import_ids_chunks:
await self._transfer_chunk_table_data(
table=table,
need_import_ids_chunk=need_import_ids_chunk,
)
logger.info(f"finished transferring table \"{table.name}\"")