def do_get(self): try: wait = self._max_overflow > -1 and self._overflow >= self._max_overflow return self._pool.get(wait, self._timeout) except Queue.Empty: if self._max_overflow > -1 and self._overflow >= self._max_overflow: if not wait: return self.do_get() else: raise exc.TimeoutError( "QueuePool limit of size %d overflow %d reached, connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) if self._overflow_lock is not None: self._overflow_lock.acquire() if self._max_overflow > -1 and self._overflow >= self._max_overflow: if self._overflow_lock is not None: self._overflow_lock.release() return self.do_get() try: con = self.create_connection() self._overflow += 1 finally: if self._overflow_lock is not None: self._overflow_lock.release() return con
def _do_get(self): try: wait = self._max_overflow > -1 and \ self._overflow >= self._max_overflow return self._pool.get(wait, self._timeout) except sqla_queue.SAAbort as aborted: return aborted.context._do_get() except sqla_queue.Empty: if self._max_overflow > -1 and \ self._overflow >= self._max_overflow: if not wait: return self._do_get() else: raise exc.TimeoutError( "QueuePool limit of size %d overflow %d reached, " "connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) self._overflow_lock.acquire() try: if self._max_overflow > -1 and \ self._overflow >= self._max_overflow: return self._do_get() else: con = self._create_connection() self._overflow += 1 return con finally: self._overflow_lock.release()
def _do_get(self): # print 'get'*30 # print self._pool.qsize() use_overflow = self._max_overflow > -1 try: wait = use_overflow and self._overflow >= self._max_overflow return self._pool.get(wait, self._timeout) except Empty: if use_overflow and self._overflow >= self._max_overflow: if not wait: return self._do_get() else: raise exc.TimeoutError( "GreenletQueuePool limit of size %d overflow %d reached, " "connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) if self._inc_overflow(): try: return self._create_connection() except: self._dec_overflow() else: return self._do_get()
def test_sql_or_recoverable_error_sqlalchemy(self, mock_rollback): e = exc.TimeoutError() assert retry.sql_or_recoverable_error(e) mock_rollback.assert_called()
class QueuePool(Pool): """A :class:`.Pool` that imposes a limit on the number of open connections. :class:`.QueuePool` is the default pooling implementation used for all :class:`.Engine` objects, unless the SQLite dialect is in use. """ def __init__(self, creator, pool_size=5, max_overflow=10, timeout=30, **kw): """ Construct a QueuePool. :param creator: a callable function that returns a DB-API connection object. The function will be called with parameters. :param pool_size: The size of the pool to be maintained, defaults to 5. This is the largest number of connections that will be kept persistently in the pool. Note that the pool begins with no connections; once this number of connections is requested, that number of connections will remain. ``pool_size`` can be set to 0 to indicate no size limit; to disable pooling, use a :class:`~sqlalchemy.pool.NullPool` instead. :param max_overflow: The maximum overflow size of the pool. When the number of checked-out connections reaches the size set in pool_size, additional connections will be returned up to this limit. When those additional connections are returned to the pool, they are disconnected and discarded. It follows then that the total number of simultaneous connections the pool will allow is pool_size + `max_overflow`, and the total number of "sleeping" connections the pool will allow is pool_size. `max_overflow` can be set to -1 to indicate no overflow limit; no limit will be placed on the total number of concurrent connections. Defaults to 10. :param timeout: The number of seconds to wait before giving up on returning a connection. Defaults to 30. :param recycle: If set to non -1, number of seconds between connection recycling, which means upon checkout, if this timeout is surpassed the connection will be closed and replaced with a newly opened connection. Defaults to -1. :param echo: If True, connections being pulled and retrieved from the pool will be logged to the standard output, as well as pool sizing information. Echoing can also be achieved by enabling logging for the "sqlalchemy.pool" namespace. Defaults to False. :param use_threadlocal: If set to True, repeated calls to :meth:`connect` within the same application thread will be guaranteed to return the same connection object, if one has already been retrieved from the pool and has not been returned yet. Offers a slight performance advantage at the cost of individual transactions by default. The :meth:`unique_connection` method is provided to bypass the threadlocal behavior installed into :meth:`connect`. :param reset_on_return: Determine steps to take on connections as they are returned to the pool. reset_on_return can have any of these values: * 'rollback' - call rollback() on the connection, to release locks and transaction resources. This is the default value. The vast majority of use cases should leave this value set. * True - same as 'rollback', this is here for backwards compatibility. * 'commit' - call commit() on the connection, to release locks and transaction resources. A commit here may be desirable for databases that cache query plans if a commit is emitted, such as Microsoft SQL Server. However, this value is more dangerous than 'rollback' because any data changes present on the transaction are committed unconditionally. * None - don't do anything on the connection. This setting should only be made on a database that has no transaction support at all, namely MySQL MyISAM. By not doing anything, performance can be improved. This setting should **never be selected** for a database that supports transactions, as it will lead to deadlocks and stale state. * False - same as None, this is here for backwards compatibility. .. versionchanged:: 0.7.6 ``reset_on_return`` accepts values. :param listeners: A list of :class:`~sqlalchemy.interfaces.PoolListener`-like objects or dictionaries of callables that receive events when DB-API connections are created, checked out and checked in to the pool. """ Pool.__init__(self, creator, **kw) self._pool = sqla_queue.Queue(pool_size) self._overflow = 0 - pool_size self._max_overflow = max_overflow self._timeout = timeout self._overflow_lock = self._max_overflow > -1 and \ threading.Lock() or DummyLock() def _do_return_conn(self, conn): try: self._pool.put(conn, False) except sqla_queue.Full: conn.close() self._overflow_lock.acquire() try: self._overflow -= 1 finally: self._overflow_lock.release() def _do_get(self): try: wait = self._max_overflow > -1 and \ self._overflow >= self._max_overflow return self._pool.get(wait, self._timeout) except sqla_queue.SAAbort, aborted: return aborted.context._do_get() except sqla_queue.Empty: if self._max_overflow > -1 and \ self._overflow >= self._max_overflow: if not wait: return self._do_get() else: raise exc.TimeoutError( "QueuePool limit of size %d overflow %d reached, " "connection timed out, timeout %d" % (self.size(), self.overflow(), self._timeout)) self._overflow_lock.acquire() try: if self._max_overflow > -1 and \ self._overflow >= self._max_overflow: return self._do_get() else: con = self._create_connection() self._overflow += 1 return con finally: self._overflow_lock.release()