async def _get_table_column_values_part(
self,
table_column_values_sql: str,
table_column_values: List[Union[str, int]],
):
if table_column_values_sql:
logger.debug(table_column_values_sql)
async with self._src_database.connection_pool.acquire() as connection: # noqa
try:
table_column_values_part = await connection.fetch(table_column_values_sql) # noqa
except asyncpg.PostgresSyntaxError as e:
logger.warning(
f"{str(e)} --- {table_column_values_sql} --- "
f"_get_table_column_values_part"
)
table_column_values_part = []
filtered_table_column_values_part = [
record[0]
for record in table_column_values_part if
record[0] is not None
]
table_column_values.extend(filtered_table_column_values_part)
del table_column_values_part
del table_column_values_sql
async def _direct_recursively_preparing_table_chunk(
self,
table: DBTable,
need_transfer_pks_chunk: List[int],
stack_tables: Optional[Set[DBTable]] = None,
):
"""
Recursively preparing table
"""
logger.debug(make_str_from_iterable([t.name for t in stack_tables]))
coroutines = [
self._direct_recursively_preparing_foreign_table(
table=table,
column=column,
need_transfer_pks=need_transfer_pks_chunk,
stack_tables=stack_tables,
)
for column in table.not_self_fk_columns if
not (
column.constraint_table.with_key_column or
column.constraint_table in stack_tables
)
]
if coroutines:
await asyncio.wait(coroutines)
del need_transfer_pks_chunk
async def _set_table_counters(self, table_name: str):
"""
Filling table max pk and count of records
"""
async with self._src_database.connection_pool.acquire() as connection:
table = self._dst_database.tables[table_name]
try:
count_table_records_sql = (
SQLRepository.get_count_table_records(
primary_key=table.primary_key, ))
except AttributeError as e:
logger.warning(f'{str(e)} --- _set_table_counters {"-"*10} - '
f"{table.name}")
raise AttributeError
except UndefinedFunctionError:
raise UndefinedFunctionError
res = await connection.fetchrow(count_table_records_sql)
if res and res[0] and res[1]:
logger.debug(f"table {table_name} with full count {res[0]}, "
f"max pk - {res[1]}")
table.full_count = int(res[0])
table.max_pk = (int(res[1]) if isinstance(res[1], int) else
table.full_count + 100000)
del count_table_records_sql
async def _direct_recursively_preparing_foreign_table_chunk(
self,
table: DBTable,
column: DBColumn,
need_transfer_pks_chunk: Iterable[int],
stack_tables: Set[DBTable],
):
"""
Direct recursively preparing foreign table chunk
"""
foreign_table = column.constraint_table
foreign_table.is_checked = True
# Если таблица с key_column, то нет необходимости пробрасывать
# идентификаторы записей
if table.with_key_column:
foreign_table_pks = await self._get_table_column_values(
table=table,
column=column,
)
else:
need_transfer_pks = (need_transfer_pks_chunk
if not table.is_full_prepared else ())
foreign_table_pks = await self._get_table_column_values(
table=table,
column=column,
primary_key_values=need_transfer_pks,
)
# если найдены значения внешних ключей отличающиеся от null, то
# записи из внешней талицы с этими идентификаторами должны быть
# импортированы
if foreign_table_pks:
logger.debug(
f"table - {table.name}, column - {column.name} - reversed "
f"collecting of fk_ids ----- {foreign_table.name}")
foreign_table_pks_difference = foreign_table_pks.difference(
foreign_table.need_transfer_pks)
# если есть разница между предполагаемыми записями для импорта
# и уже выбранными ранее, то разницу нужно импортировать
if foreign_table_pks_difference:
foreign_table.update_need_transfer_pks(
need_transfer_pks=foreign_table_pks_difference, )
await asyncio.wait([
asyncio.create_task(
self._direct_recursively_preparing_table(
table=foreign_table,
need_transfer_pks=foreign_table_pks_difference,
stack_tables=stack_tables,
)),
])
del foreign_table_pks_difference
del foreign_table_pks
del need_transfer_pks_chunk
def is_full_prepared(self):
logger.debug(
f'table - {self.name} -- count table records {self.full_count} and '
f'need transfer pks {len(self.need_transfer_pks)}')
if len(self.need_transfer_pks
) >= self.full_count - self.inaccuracy_count: # noqa
logger.info(f'table {self.name} full transferred')
return True
async def collect(self):
logger.info('start preparing tables sorted by dependency..')
not_transferred_tables = list(
filter(
lambda t: (
not t.is_ready_for_transferring
and t.name not in settings.TABLES_WITH_GENERIC_FOREIGN_KEY
),
self._dst_database.tables.values(),
)
)
logger.debug(
f'tables not transferring {str(len(not_transferred_tables))}'
)
dependencies_between_models = []
for table in self._dst_database.tables_without_generics:
for fk_column in table.not_self_fk_columns:
dependencies_between_models.append(
(table.name, fk_column.constraint_table.name)
)
sorted_dependencies_result = topological_sort(
dependency_pairs=dependencies_between_models,
)
sorted_dependencies_result.cyclic.reverse()
sorted_dependencies_result.sorted.reverse()
sorted_tables_by_dependency = (
sorted_dependencies_result.cyclic + sorted_dependencies_result.sorted
)
without_relatives = list(
{
table.name
for table in self._dst_database.tables_without_generics
}.difference(
sorted_tables_by_dependency
)
)
sorted_tables_by_dependency = without_relatives + sorted_tables_by_dependency
# явно ломаю асинхронность, т.к. порядок импорта таблиц важен
for table_name in sorted_tables_by_dependency:
table = self._dst_database.tables[table_name]
if not table.is_ready_for_transferring:
await self._prepare_unready_table(
table=table,
)
logger.info('preparing tables sorted by dependency finished.')
async def _prepare_content_type_generic_data(
self,
target_table: DBTable,
rel_table_name: str,
):
if not rel_table_name:
logger.debug('not send rel_table_name')
return
rel_table = self._dst_database.tables.get(rel_table_name)
if not rel_table:
logger.debug(f'table {rel_table_name} not found')
return
object_id_column = await target_table.get_column_by_name('object_id')
if rel_table.primary_key.data_type != object_id_column.data_type:
logger.debug(
f'pk of table {rel_table_name} has an incompatible data type'
)
return
logger.info('prepare content type generic data')
where_conditions = {
'object_id': rel_table.need_transfer_pks,
'content_type_id': [self.content_type_table[rel_table.name]],
}
need_transfer_pks = await self._get_table_column_values(
table=target_table,
column=target_table.primary_key,
where_conditions_columns=where_conditions,
)
logger.info(
f'{target_table.name} need transfer pks {len(need_transfer_pks)}'
)
target_table.update_need_transfer_pks(
need_transfer_pks=need_transfer_pks,
)
del where_conditions
del need_transfer_pks
async def _prepare_unready_table(
self,
table: DBTable,
):
"""
Preparing table records for transferring
"""
logger.info(
f'start preparing table "{table.name}"'
)
# обход таблиц связанных через внешние ключи
where_conditions_columns = {}
fk_columns = table.highest_priority_fk_columns
with_full_transferred_table = False
for fk_column in fk_columns:
logger.debug(f'prepare column {fk_column.name}')
fk_table = self._dst_database.tables[
fk_column.constraint_table.name
]
if fk_table.need_transfer_pks:
if not fk_table.is_full_prepared:
where_conditions_columns[fk_column.name] = (
fk_table.need_transfer_pks
)
else:
with_full_transferred_table = True
if (
fk_columns and
not where_conditions_columns and
not with_full_transferred_table
):
return
table_pks = await self._get_table_column_values(
table=table,
column=table.primary_key,
where_conditions_columns=where_conditions_columns,
)
if (
fk_columns and
where_conditions_columns and
not table_pks
):
return
table.update_need_transfer_pks(
need_transfer_pks=table_pks,
)
logger.debug(
f'table "{table.name}" need transfer pks - '
f'{len(table.need_transfer_pks)}'
)
del table_pks
# обход таблиц ссылающихся на текущую таблицу
logger.debug('prepare revert tables')
coroutines = [
self._prepare_revert_table(
table=table,
revert_table=revert_table,
revert_columns=revert_columns,
)
for revert_table, revert_columns in table.revert_foreign_tables.items() # noqa
]
if coroutines:
await asyncio.wait(coroutines)
if not table.need_transfer_pks:
all_records = await self._get_table_column_values(
table=table,
column=table.primary_key,
)
table.update_need_transfer_pks(
need_transfer_pks=all_records,
)
del all_records
table.is_ready_for_transferring = True
logger.info(
f'finished collecting records ids of table "{table.name}"'
)
