async def set_max_sequence(self, dst_pool: Pool): async with dst_pool.acquire() as connection: try: get_serial_sequence_sql = SQLRepository.get_serial_sequence_sql( table_name=self.name, pk_column_name=self.primary_key.name, ) except AttributeError: logger.warning( f'AttributeError --- {self.name} --- set_max_sequence') return serial_seq_name = await connection.fetchrow(get_serial_sequence_sql ) if serial_seq_name and serial_seq_name[0]: serial_seq_name = serial_seq_name[0] max_val = self.max_pk + 100000 set_sequence_val_sql = ( SQLRepository.get_set_sequence_value_sql( seq_name=serial_seq_name, seq_val=max_val, )) await connection.execute(set_sequence_val_sql)
async def _get_table_column_values_part( self, table_column_values_sql: str, table_column_values: List[Union[str, int]], ): if table_column_values_sql: logger.debug(table_column_values_sql) async with self._src_database.connection_pool.acquire() as connection: # noqa try: table_column_values_part = await connection.fetch(table_column_values_sql) # noqa except asyncpg.PostgresSyntaxError as e: logger.warning( f"{str(e)} --- {table_column_values_sql} --- " f"_get_table_column_values_part" ) table_column_values_part = [] filtered_table_column_values_part = [ record[0] for record in table_column_values_part if record[0] is not None ] table_column_values.extend(filtered_table_column_values_part) del table_column_values_part del table_column_values_sql
async def _set_table_counters(self, table_name: str): """ Filling table max pk and count of records """ async with self._src_database.connection_pool.acquire() as connection: table = self._dst_database.tables[table_name] try: count_table_records_sql = ( SQLRepository.get_count_table_records( primary_key=table.primary_key, )) except AttributeError as e: logger.warning(f'{str(e)} --- _set_table_counters {"-"*10} - ' f"{table.name}") raise AttributeError except UndefinedFunctionError: raise UndefinedFunctionError res = await connection.fetchrow(count_table_records_sql) if res and res[0] and res[1]: logger.debug(f"table {table_name} with full count {res[0]}, " f"max pk - {res[1]}") table.full_count = int(res[0]) table.max_pk = (int(res[1]) if isinstance(res[1], int) else table.full_count + 100000) del count_table_records_sql
async def _get_table_column_values( self, table: DBTable, column: DBColumn, primary_key_values: Iterable[Union[int, str]] = (), where_conditions_columns: Optional[Dict[str, Iterable[Union[ int, str]]]] = None, # noqa is_revert=False, ) -> Set[Union[str, int]]: # если таблица находится в исключенных, то ее записи не нужно # импортировать try: if column.constraint_table.name in settings.EXCLUDED_TABLES: return set() except AttributeError as e: logger.warning(f"{str(e)} --- _get_table_column_values") return set() # формирование запроса на получения идентификаторов записей # внешней таблицы table_column_values_sql_list = await SQLRepository.get_table_column_values_sql( table=table, column=column, key_column_values=self._key_column_values, primary_key_values=primary_key_values, where_conditions_columns=where_conditions_columns, is_revert=is_revert, ) table_column_values = [] for table_column_values_sql in table_column_values_sql_list: await self._get_table_column_values_part( table_column_values_sql=table_column_values_sql, table_column_values=table_column_values, ) del table_column_values_sql_list[:] unique_table_column_values = set(table_column_values) del table_column_values[:] return unique_table_column_values
# Dst database connection params DST_DB_HOST = get_str_environ_parameter(name='DATABASER_DST_DB_HOST', ) DST_DB_PORT = get_str_environ_parameter(name='DATABASER_DST_DB_PORT', ) DST_DB_SCHEMA = get_str_environ_parameter( name='DATABASER_DST_DB_SCHEMA', default='public', ) DST_DB_NAME = get_str_environ_parameter(name='DATABASER_DST_DB_NAME', ) DST_DB_USER = get_str_environ_parameter(name='DATABASER_DST_DB_USER', ) DST_DB_PASSWORD = get_str_environ_parameter(name='DATABASER_DST_DB_PASSWORD', ) # Test mode parameters TEST_MODE = get_bool_environ_parameter(name='DATABASER_TEST_MODE', ) if TEST_MODE: logger.warning('TEST MODE ACTIVATED!!!') KEY_TABLE_NAME = get_str_environ_parameter(name='DATABASER_KEY_TABLE_NAME', ) KEY_COLUMN_NAMES = get_iterable_environ_parameter( name='DATABASER_KEY_COLUMN_NAMES', ) KEY_COLUMN_VALUES = get_iterable_environ_parameter( name='DATABASER_KEY_COLUMN_VALUES', type_=int, ) KEY_TABLE_HIERARCHY_COLUMN_NAME = get_str_environ_parameter( name='DATABASER_KEY_TABLE_HIERARCHY_COLUMN_NAME', ) EXCLUDED_TABLES = get_iterable_environ_parameter( name='DATABASER_EXCLUDED_TABLES', ) TABLES_WITH_GENERIC_FOREIGN_KEY = get_iterable_environ_parameter( name='DATABASER_TABLES_WITH_GENERIC_FOREIGN_KEY', )