async def set_max_sequence(self, dst_pool: Pool):
async with dst_pool.acquire() as connection:
try:
get_serial_sequence_sql = SQLRepository.get_serial_sequence_sql(
table_name=self.name,
pk_column_name=self.primary_key.name,
)
except AttributeError:
logger.warning(
f'AttributeError --- {self.name} --- set_max_sequence')
return
serial_seq_name = await connection.fetchrow(get_serial_sequence_sql
)
if serial_seq_name and serial_seq_name[0]:
serial_seq_name = serial_seq_name[0]
max_val = self.max_pk + 100000
set_sequence_val_sql = (
SQLRepository.get_set_sequence_value_sql(
seq_name=serial_seq_name,
seq_val=max_val,
))
await connection.execute(set_sequence_val_sql)
async def _get_table_column_values_part(
self,
table_column_values_sql: str,
table_column_values: List[Union[str, int]],
):
if table_column_values_sql:
logger.debug(table_column_values_sql)
async with self._src_database.connection_pool.acquire() as connection: # noqa
try:
table_column_values_part = await connection.fetch(table_column_values_sql) # noqa
except asyncpg.PostgresSyntaxError as e:
logger.warning(
f"{str(e)} --- {table_column_values_sql} --- "
f"_get_table_column_values_part"
)
table_column_values_part = []
filtered_table_column_values_part = [
record[0]
for record in table_column_values_part if
record[0] is not None
]
table_column_values.extend(filtered_table_column_values_part)
del table_column_values_part
del table_column_values_sql
async def _set_table_counters(self, table_name: str):
"""
Filling table max pk and count of records
"""
async with self._src_database.connection_pool.acquire() as connection:
table = self._dst_database.tables[table_name]
try:
count_table_records_sql = (
SQLRepository.get_count_table_records(
primary_key=table.primary_key, ))
except AttributeError as e:
logger.warning(f'{str(e)} --- _set_table_counters {"-"*10} - '
f"{table.name}")
raise AttributeError
except UndefinedFunctionError:
raise UndefinedFunctionError
res = await connection.fetchrow(count_table_records_sql)
if res and res[0] and res[1]:
logger.debug(f"table {table_name} with full count {res[0]}, "
f"max pk - {res[1]}")
table.full_count = int(res[0])
table.max_pk = (int(res[1]) if isinstance(res[1], int) else
table.full_count + 100000)
del count_table_records_sql
async def _get_table_column_values(
self,
table: DBTable,
column: DBColumn,
primary_key_values: Iterable[Union[int, str]] = (),
where_conditions_columns: Optional[Dict[str, Iterable[Union[
int, str]]]] = None, # noqa
is_revert=False,
) -> Set[Union[str, int]]:
# если таблица находится в исключенных, то ее записи не нужно
# импортировать
try:
if column.constraint_table.name in settings.EXCLUDED_TABLES:
return set()
except AttributeError as e:
logger.warning(f"{str(e)} --- _get_table_column_values")
return set()
# формирование запроса на получения идентификаторов записей
# внешней таблицы
table_column_values_sql_list = await SQLRepository.get_table_column_values_sql(
table=table,
column=column,
key_column_values=self._key_column_values,
primary_key_values=primary_key_values,
where_conditions_columns=where_conditions_columns,
is_revert=is_revert,
)
table_column_values = []
for table_column_values_sql in table_column_values_sql_list:
await self._get_table_column_values_part(
table_column_values_sql=table_column_values_sql,
table_column_values=table_column_values,
)
del table_column_values_sql_list[:]
unique_table_column_values = set(table_column_values)
del table_column_values[:]
return unique_table_column_values
# Dst database connection params
DST_DB_HOST = get_str_environ_parameter(name='DATABASER_DST_DB_HOST', )
DST_DB_PORT = get_str_environ_parameter(name='DATABASER_DST_DB_PORT', )
DST_DB_SCHEMA = get_str_environ_parameter(
name='DATABASER_DST_DB_SCHEMA',
default='public',
)
DST_DB_NAME = get_str_environ_parameter(name='DATABASER_DST_DB_NAME', )
DST_DB_USER = get_str_environ_parameter(name='DATABASER_DST_DB_USER', )
DST_DB_PASSWORD = get_str_environ_parameter(name='DATABASER_DST_DB_PASSWORD', )
# Test mode parameters
TEST_MODE = get_bool_environ_parameter(name='DATABASER_TEST_MODE', )
if TEST_MODE:
logger.warning('TEST MODE ACTIVATED!!!')
KEY_TABLE_NAME = get_str_environ_parameter(name='DATABASER_KEY_TABLE_NAME', )
KEY_COLUMN_NAMES = get_iterable_environ_parameter(
name='DATABASER_KEY_COLUMN_NAMES', )
KEY_COLUMN_VALUES = get_iterable_environ_parameter(
name='DATABASER_KEY_COLUMN_VALUES',
type_=int,
)
KEY_TABLE_HIERARCHY_COLUMN_NAME = get_str_environ_parameter(
name='DATABASER_KEY_TABLE_HIERARCHY_COLUMN_NAME', )
EXCLUDED_TABLES = get_iterable_environ_parameter(
name='DATABASER_EXCLUDED_TABLES', )
TABLES_WITH_GENERIC_FOREIGN_KEY = get_iterable_environ_parameter(
name='DATABASER_TABLES_WITH_GENERIC_FOREIGN_KEY', )