def __init__(self, db_settings, max_queue_len=None): # There is no upper limit on the queue length by default if not max_queue_len: max_queue_len = -1 db_settings = self._filter_pool_settings(db_settings) # Create delivery queue for database statements to be executed self._queue = DeliveryQueue( self._process_statement, drop_callback=self._on_queue_drop, max_queue_len=max_queue_len, # Max parallel statements - should correspond with connection count max_active_queue_len=db_settings['pool_size'], # How often to retry deliver some retry_after_s=5) self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS) self._drop_count = 0 # Create db-pool with the provided arguments self._db_pool = self._create_db_pool(**db_settings) self._second_db_pool = None self._suspended = False
def __init__(self, db_settings, max_queue_len=None): # There is no upper limit on the queue length by default if not max_queue_len: max_queue_len = -1 db_settings = self._filter_pool_settings(db_settings) # Create delivery queue for database statements to be executed self._queue = DeliveryQueue(self._process_statement, drop_callback=self._on_queue_drop, max_queue_len=max_queue_len, # Max parallel statements - should correspond with connection count max_active_queue_len=db_settings['pool_size'], # How often to retry deliver some retry_after_s=5) self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS) self._drop_count = 0 # Create db-pool with the provided arguments self._db_pool = self._create_db_pool(**db_settings) self._second_db_pool = None self._suspended = False
class DbSink(object): def __init__(self, db_settings, max_queue_len=None): # There is no upper limit on the queue length by default if not max_queue_len: max_queue_len = -1 db_settings = self._filter_pool_settings(db_settings) # Create delivery queue for database statements to be executed self._queue = DeliveryQueue( self._process_statement, drop_callback=self._on_queue_drop, max_queue_len=max_queue_len, # Max parallel statements - should correspond with connection count max_active_queue_len=db_settings['pool_size'], # How often to retry deliver some retry_after_s=5) self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS) self._drop_count = 0 # Create db-pool with the provided arguments self._db_pool = self._create_db_pool(**db_settings) self._second_db_pool = None self._suspended = False @staticmethod def _on_connect(conn): log.info("New DB connection created") conn.cursor().execute( "set session transaction isolation level read uncommitted") conn.autocommit(True) def _create_db_pool(self, driver, host, port, user, password, dbname, pool_size): return adbapi.ConnectionPool(driver, host=host, port=port, user=user, passwd=password, db=dbname, cp_openfun=self._on_connect, cp_reconnect=True, cp_min=1, cp_max=pool_size) @staticmethod def _filter_pool_settings(settings): settings = { 'driver': settings.get('driver'), 'host': settings.get('host'), 'port': settings.get('port'), 'user': settings.get('user'), 'password': settings.get('password'), 'dbname': settings.get('dbname'), 'pool_size': settings.get('pool_size'), } if sum(val > 1 for val in settings.values()) != 7: raise Exception('Some database parameter not specified (%s)' % (str(settings))) return settings def get_queue_length(self): return self._queue.get_length() def get_stats(self): return self._stats.get_data() def reset_stats(self): return self._stats.reset() def execute(self, statement_name, statement): return self._queue.deliver((statement_name, statement)) def direct_query(self, name, query): return self._inspect(name, self._db_pool.runQuery(query), True) def _process_statement(self, obj): """ Deliver queue's processor """ # Try to deliver later, db_pool is not ready or the operation is suspended if self._suspended or not self._db_pool: return False try: (name, statement) = obj return self._inspect(name, self._db_pool.runOperation(statement), False) except BaseException, e: return e
class DbSink(object): def __init__(self, db_settings, max_queue_len=None): # There is no upper limit on the queue length by default if not max_queue_len: max_queue_len = -1 db_settings = self._filter_pool_settings(db_settings) # Create delivery queue for database statements to be executed self._queue = DeliveryQueue(self._process_statement, drop_callback=self._on_queue_drop, max_queue_len=max_queue_len, # Max parallel statements - should correspond with connection count max_active_queue_len=db_settings['pool_size'], # How often to retry deliver some retry_after_s=5) self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS) self._drop_count = 0 # Create db-pool with the provided arguments self._db_pool = self._create_db_pool(**db_settings) self._second_db_pool = None self._suspended = False @staticmethod def _on_connect(conn): log.info("New DB connection created") conn.cursor().execute("set session transaction isolation level read uncommitted") conn.autocommit(True) def _create_db_pool(self, driver, host, port, user, password, dbname, pool_size): return adbapi.ConnectionPool(driver, host=host, port=port, user=user, passwd=password, db=dbname, cp_openfun=self._on_connect, cp_reconnect=True, cp_min=1, cp_max=pool_size) @staticmethod def _filter_pool_settings(settings): settings = { 'driver': settings.get('driver'), 'host': settings.get('host'), 'port': settings.get('port'), 'user': settings.get('user'), 'password': settings.get('password'), 'dbname': settings.get('dbname'), 'pool_size': settings.get('pool_size'), } if sum(val > 1 for val in settings.values()) != 7: raise Exception('Some database parameter not specified (%s)' % (str(settings))) return settings def get_queue_length(self): return self._queue.get_length() def get_stats(self): return self._stats.get_data() def reset_stats(self): return self._stats.reset() def execute(self, statement_name, statement): return self._queue.deliver((statement_name, statement)) def direct_query(self, name, query): return self._inspect(name, self._db_pool.runQuery(query), True) def _process_statement(self, obj): """ Deliver queue's processor """ # Try to deliver later, db_pool is not ready or the operation is suspended if self._suspended or not self._db_pool: return False try: (name, statement) = obj return self._inspect(name, self._db_pool.runOperation(statement), False) except BaseException, e: return e