def _put_conn(self, conn, key=None, close=False, fail_silently=False):
"""Stow away a connection."""
if self._disable_pooling:
self._release(conn)
return
self._log('Putting away %s%s' % (conn, ' key=' + key if key else ''))
if self._disposed:
if fail_silently:
return
raise PoolError('Connection pool is disposed')
if key is None:
key = self._rused.get(id(conn))
if not key:
raise PoolError('Trying to put un-keyed connection')
if len(self._pool) < self.max_conn and not close:
# Return the connection into a consistent state before putting
# it back into the pool
if not conn.closed:
status = conn.get_transaction_status()
if status == _ext.TRANSACTION_STATUS_UNKNOWN:
# server connection lost
self._log('Connection lost. Closing %s' % conn)
self._release(conn.close)
elif status != _ext.TRANSACTION_STATUS_IDLE:
# connection in error or in transaction
self._log(
'Connection is in transaction. Rolling back %s' % conn)
conn.rollback()
self._pool.append(conn)
else:
# regular idle connection
self._pool.append(conn)
# If the connection is closed, we just discard it.
else:
self._log(
'''Closing (pool exhausted or
explicit close requested) %s''' % conn)
self._release(conn)
self._purge_expired_connections()
# here we check for the presence of key because it can happen that a
# thread tries to put back a connection after a call to close
if not self._disposed or key in self._used:
del self._used[key]
del self._rused[id(conn)]
def _release_all(self):
"""Release all connections.
Note that this can lead to some code fail badly when trying to use
an already closed connection. If you call .release_all() make sure
your code can deal with it.
"""
# Make sure that all connections lying about are collected before we go on.
try:
gc.collect()
except (TypeError, AttributeError):
# We've detected that we're being called in an incomplete
# finalization state, we just bail out, leaving the connections
# to take care of themselves.
return
if self._disposed:
raise PoolError('Connection pool is disposed')
if not self._disable_pooling:
close_list = self._pool + list(self._used.values())
self._log('Closing %d connection(s)' % len(close_list))
for conn in close_list:
try:
conn.close()
except:
pass
self._disposed = True
self._pool = []
self._used = {}
def borrow(self, connection_info):
"""
:param dict connection_info: dict of psql connection keywords
:rtype: PsycoConnection
"""
# free dead and leaked connections
for i, (cnx, _) in tools.reverse_enumerate(self._connections):
if cnx.closed:
self._connections.pop(i)
self._debug('Removing closed connection at index %d: %r', i,
cnx.dsn)
continue
if getattr(cnx, 'leaked', False):
delattr(cnx, 'leaked')
self._connections.pop(i)
self._connections.append((cnx, False))
_logger.info('%r: Free leaked connection to %r', self, cnx.dsn)
for i, (cnx, used) in enumerate(self._connections):
if not used and cnx._original_dsn == connection_info:
try:
cnx.reset()
except psycopg2.OperationalError:
self._debug('Cannot reset connection at index %d: %r', i,
cnx.dsn)
# psycopg2 2.4.4 and earlier do not allow closing a closed connection
if not cnx.closed:
cnx.close()
continue
self._connections.pop(i)
self._connections.append((cnx, True))
self._debug('Borrow existing connection to %r at index %d',
cnx.dsn, i)
return cnx
if len(self._connections) >= self._maxconn:
# try to remove the oldest connection not used
for i, (cnx, used) in enumerate(self._connections):
if not used:
self._connections.pop(i)
if not cnx.closed:
cnx.close()
self._debug('Removing old connection at index %d: %r', i,
cnx.dsn)
break
else:
# note: this code is called only if the for loop has completed (no break)
raise PoolError('The Connection Pool Is Full')
try:
result = psycopg2.connect(connection_factory=PsycoConnection,
**connection_info)
except psycopg2.Error:
_logger.info('Connection to the database failed')
raise
result._original_dsn = connection_info
self._connections.append((result, True))
self._debug('Create new connection')
return result
def borrow(self, dsn):
self._debug('Borrow connection to %r', dsn)
# free leaked connections
for i, (cnx, _) in tools.reverse_enumerate(self._connections):
if getattr(cnx, 'leaked', False):
delattr(cnx, 'leaked')
self._connections.pop(i)
self._connections.append((cnx, False))
self._debug('Free leaked connection to %r', cnx.dsn)
for i, (cnx, used) in enumerate(self._connections):
if not used and dsn_are_equals(cnx.dsn, dsn):
self._connections.pop(i)
self._connections.append((cnx, True))
self._debug('Existing connection found at index %d', i)
return cnx
if len(self._connections) >= self._maxconn:
# try to remove the oldest connection not used
for i, (cnx, used) in enumerate(self._connections):
if not used:
self._connections.pop(i)
self._debug('Removing old connection at index %d: %r', i,
cnx.dsn)
break
else:
# note: this code is called only if the for loop has completed (no break)
raise PoolError('The Connection Pool Is Full')
result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection)
self._connections.append((result, True))
self._debug('Create new connection')
return result
def _getconn(self, key=None):
"""Get a free connection and assign it to 'key' if not None."""
if self.closed: raise PoolError("connection pool is closed")
if key is None: key = self._getkey()
if key in self._used:
return self._used[key]
if self._pool:
self._used[key] = conn = self._pool.pop()
self._rused[id(conn)] = key
return conn
else:
if len(self._used) == self.maxconn:
raise PoolError("connection pool exausted")
return self._connect(key)
def borrow(self, dsn):
self._debug('Borrow connection to %r', dsn)
# free dead and leaked connections
for i, (cnx, _) in tools.reverse_enumerate(self._connections):
if cnx.closed:
self._connections.pop(i)
self._debug('Removing closed connection at index %d: %r', i,
cnx.dsn)
continue
if getattr(cnx, 'leaked', False):
delattr(cnx, 'leaked')
self._connections.pop(i)
self._connections.append((cnx, False))
_logger.warning('%r: Free leaked connection to %r', self,
cnx.dsn)
for i, (cnx, used) in enumerate(self._connections):
if not used and dsn_are_equals(cnx.dsn, dsn):
try:
cnx.reset()
except psycopg2.OperationalError:
self._debug('Cannot reset connection at index %d: %r', i,
cnx.dsn)
# psycopg2 2.4.4 and earlier do not allow closing a closed connection
if not cnx.closed:
cnx.close()
continue
self._connections.pop(i)
self._connections.append((cnx, True))
self._debug('Existing connection found at index %d', i)
return cnx
if len(self._connections) >= self._maxconn:
# try to remove the oldest connection not used
for i, (cnx, used) in enumerate(self._connections):
if not used:
self._connections.pop(i)
if not cnx.closed:
cnx.close()
self._debug('Removing old connection at index %d: %r', i,
cnx.dsn)
break
else:
# note: this code is called only if the for loop has completed (no break)
raise PoolError('The Connection Pool Is Full')
try:
result = psycopg2.connect(dsn=dsn,
connection_factory=PsycoConnection)
except psycopg2.Error:
_logger.exception('Connection to the database failed')
raise
self._connections.append((result, True))
self._debug('Create new connection')
return result
def _putconn(self, conn, key=None, close=False):
"""Put away a connection."""
if self.closed: raise PoolError("connection pool is closed")
if key is None: key = self._rused[id(conn)]
if not key:
raise PoolError("trying to put unkeyed connection")
if len(self._pool) < self.minconn and not close:
self._pool.append(conn)
else:
conn.close()
# here we check for the presence of key because it can happen that a
# thread tries to put back a connection after a call to close
if not self.closed or key in self._used:
del self._used[key]
del self._rused[id(conn)]
def closeall(self):
if self.closed: raise PoolError("connection pool is closed")
# TODO -- might be better to do this decrementing self.size?
while not self._pool.empty():
conn = self._pool.get_nowait()
try:
conn.close()
except Exception:
pass
self.closed = True
def __enter__(self):
for _ in range(TRAILS):
try:
self.conn = Connection.connection_pool.getconn()
self.conn.autocommit = False
self.cursor = self.conn.cursor()
return self
except PoolError:
logger.info('Pool doesn\'t have available connection. Please wait')
time.sleep(int(POOL_DELAY))
raise PoolError('Can\'t get a connection.')
def _get_conn(self, key=None):
"""Get a free connection and assign it to 'key' if not None."""
if self._disposed:
raise PoolError('Connection pool is disposed')
if self._disable_pooling:
return self._connect(key)
if key is None:
key = self._get_key()
if key in self._used:
return self._used[key]
if self._pool:
self._used[key] = conn = self._pool.pop()
self._rused[id(conn)] = key
self._tused[id(conn)] = time.time()
return conn
else:
if len(self._used) == self.max_conn:
raise PoolError('Connection pool exhausted')
return self._connect(key)
def give_back(self, connection, keep_in_pool=True):
self._debug('Give back connection to %r', connection.dsn)
for i, (cnx, used) in enumerate(self._connections):
if cnx is connection:
self._connections.pop(i)
if keep_in_pool:
self._connections.append((cnx, False))
self._debug('Put connection to %r in pool', cnx.dsn)
else:
self._debug('Forgot connection to %r', cnx.dsn)
break
else:
raise PoolError('This connection does not below to the pool')
def give_back(self, connection, keep_in_pool=True):
self._debug_dsn('Give back connection to %r', connection.dsn)
for i, (cnx, used) in enumerate(self._connections):
if cnx is connection:
self._connections.pop(i)
if keep_in_pool and not (cnx.closed or not cnx.status):
self._connections.insert(i,(cnx, False))
self._debug_dsn('Put connection to %r back in pool', cnx.dsn)
else:
self._debug_dsn('Forgot connection to %r', cnx.dsn)
break
else:
raise PoolError('This connection does not below to the pool')
def _getconn(self, key=None, exactly=False):
"""Get a free connection and assign it to 'key' if not None."""
internal_key = False
if self.closed:
raise PoolError("connection pool is closed")
if key is None:
key = self._getkey()
internal_key = True
if key in self._used:
return self._used[key]
if not internal_key and exactly:
return None
if self._pool:
self._used[key] = conn = self._pool.pop()
self._rused[id(conn)] = key
self._tused[id(conn)] = datetime.datetime.now()
return conn
else:
if len(self._used) == self.maxconn:
raise PoolError("connection pool exausted")
return self._connect(key)
def getconn(self, key=None):
if self.closed: raise PoolError("connection pool is closed")
pool = self._pool
if self.size >= self.maxconn or pool.qsize():
conn = pool.get()
else:
try:
conn = self._connect()
except:
raise
return conn
def _closeall(self):
"""Close all connections.
Note that this can lead to some code fail badly when trying to use
an already closed connection. If you call .closeall() make sure
your code can deal with it.
"""
if self.closed: raise PoolError("connection pool is closed")
for conn in self._pool + list(self._used.values()):
try:
conn.close()
except:
pass
self.closed = True
def _putconn(self, conn, key=None, close=False):
"""Put away a connection."""
if self.closed:
raise PoolError("connection pool is closed")
if key is None:
key = self._rused.get(id(conn))
if not key:
raise PoolError(
"trying to put unkeyed [{key}] connection".format(key=key))
if len(self._pool) < self.maxconn and not close:
# Return the connection into a consistent state before putting
# it back into the pool
if not conn.closed:
status = conn.get_transaction_status()
if status == _ext.TRANSACTION_STATUS_UNKNOWN:
# server connection lost
self._disconnect(conn.close)
elif status != _ext.TRANSACTION_STATUS_IDLE:
# connection in error or in transaction
conn.rollback()
self._pool.append(conn)
else:
# regular idle connection
self._pool.append(conn)
# If the connection is closed, we just discard it.
else:
self._disconnect(conn)
self.clear_expired_connections()
# here we check for the presence of key because it can happen that a
# thread tries to put back a connection after a call to close
if not self.closed or key in self._used:
del self._used[key]
del self._rused[id(conn)]
def give_back(self, connection, keep_in_pool=True):
self._debug('Give back connection to %r', connection.dsn)
for i, (cnx, used, time_used) in enumerate(self._connections):
if cnx is connection:
self._connections.pop(i)
if keep_in_pool:
self._connections.append((cnx, False, time.time()))
self._debug('Put connection to %r in pool', cnx.dsn)
else:
self._debug('Forgot connection to %r', cnx.dsn)
cnx.close()
break
# decodio: close connections
elif self._idle_timeout > 0.0 and not used:
if time_used > (time.time() - self._idle_timeout):
self._debug('Idle connection timeout. Closing %r.',
cnx.dsn)
cnx.close()
else:
raise PoolError('This connection does not belong to the pool')
def borrow(self, dsn):
self._debug('Borrow connection to %r', dsn)
# free dead and leaked connections
time_now = time.time()
for i, (cnx, _) in tools.reverse_enumerate(self._connections):
if (cnx.closed
or cnx.status == psycopg2.extensions.STATUS_READY and
(CONNECTION_LIFETIME
and time_now - cnx.init_time > CONNECTION_LIFETIME) or
(CURSOR_TIMEOUT
and time_now - cnx.last_execution_time > CURSOR_TIMEOUT)):
self._connections.pop(i)
self._debug('Removing closed connection at index %d: %r', i,
cnx.dsn)
if not cnx.closed:
cnx.close() # Close unclosed connection.
continue
if getattr(cnx, 'leaked', False):
delattr(cnx, 'leaked')
self._connections.pop(i)
self._connections.append((cnx, False))
_logger.warning('%r: Free leaked connection to %r', self,
cnx.dsn)
for i, (cnx, used) in enumerate(self._connections):
if not used and dsn_are_equals(cnx.dsn, dsn):
self._connections.pop(i)
try:
cnx.reset()
except psycopg2.OperationalError:
self._debug(
'Cannot reset connection at index %d: %r, removing it',
i, cnx.dsn)
# psycopg2 2.4.4 and earlier do not allow closing a closed connection
if not cnx.closed:
cnx.close()
continue
self._connections.append((cnx, True))
self._debug('Existing connection found at index %d', i)
return cnx
if len(self._connections) >= self._maxconn:
# try to remove the oldest connection not used
for i, (cnx, used) in enumerate(self._connections):
if not used:
self._connections.pop(i)
self._debug('Removing old connection at index %d: %r', i,
cnx.dsn)
break
else:
# note: this code is called only if the for loop has completed (no break)
raise PoolError('The Connection Pool Is Full')
try:
result = psycopg2.connect(dsn=dsn,
connection_factory=PsycoConnection)
result.init_time = time.time()
result.executing = False
except psycopg2.Error:
_logger.exception('Connection to the database failed')
raise
self._connections.append((result, True))
self._debug('Create new connection')
return result
class ConnectionPool(object):
""" The pool of connections to database(s)
Keep a set of connections to pg databases open, and reuse them
to open cursors for all transactions.
The connections are *not* automatically closed. Only a close_db()
can trigger that.
"""
__logger = logging.getLogger('db.connection_pool')
def locked(fun):
@wraps(fun)
def _locked(self, *args, **kwargs):
self._lock.acquire()
try:
return fun(self, *args, **kwargs)
finally:
self._lock.release()
return _locked
def __init__(self, maxconn=64, pgmode=None):
self._connections = []
self._maxconn = max(maxconn, 1)
self._lock = threading.Lock()
self._debug_pool = tools.config.get_misc('debug', 'db_pool', False)
self.sql_stats = {}
if pgmode: # not None or False
Cursor.set_pgmode(pgmode)
def __del__(self):
# explicitly free them
del self._connections
if self.sql_stats:
self.print_all_stats()
def __repr__(self):
used = len([1 for c, u in self._connections[:] if u])
count = len(self._connections)
return "ConnectionPool(used=%d/count=%d/max=%d)" % (used, count, self._maxconn)
def _debug(self, msg, *args):
if self._debug_pool:
msg = '%r ' + msg
self.__logger.debug(msg, self, *args)
def _debug_dsn(self, msg, *args, **kwargs):
"""Debug function, that will decode the dsn_pos'th argument as dsn
@param kwargs may only contain 'dsn_pos'
"""
if not self._debug_pool:
return
def cleanup(dsn):
cl = [x for x in dsn.strip().split() if x.split('=', 1)[0] != 'password']
return ' '.join(cl)
largs = list(args)
dsn_pos = kwargs.pop('dsn_pos', 0)
if kwargs:
raise TypeError("Unknown keyword argument(s): %r" % kwargs)
assert dsn_pos < len(largs)
largs[dsn_pos] = cleanup(largs[dsn_pos])
msg = '%r ' + msg
self.__logger.debug(msg, self, *largs)
def set_pool_debug(self, do_debug = True):
self._debug_pool = do_debug
self.__logger.info("Debugging set to %s" % str(do_debug))
@locked
def borrow(self, dsn, do_cursor=False):
self._debug_dsn('Borrow connection to %r', dsn)
# free leaked connections
for i, (cnx, _) in tools.reverse_enumerate(self._connections):
if getattr(cnx, 'leaked', False):
delattr(cnx, 'leaked')
self._connections.pop(i)
self._connections.append((cnx, False))
self._debug_dsn('Free leaked connection to %r', cnx.dsn)
result = None
for i, (cnx, used) in enumerate(self._connections):
if not used and dsn_are_equals(cnx.dsn, dsn):
self._connections.pop(i)
try:
if psycopg2.__version__ >= '2.2' :
pr = cnx.poll()
self._debug("Poll: %d", pr)
except OperationalError, e:
self._debug("Error in poll: %s" % e)
continue
if cnx.closed or not cnx.status:
# something is wrong with that connection, let it out
self._debug("Troubled connection ")
continue
self._debug('Existing connection found at index %d', i)
if do_cursor:
try:
cur = cnx.cursor(cursor_factory=psycopg1cursor)
if psycopg2.__version__ < '2.2' and not cur.isready():
continue
if cur.closed:
continue
self._connections.insert(i,(cnx, True))
result = (cnx, cur)
except OperationalError:
continue
else:
self._connections.insert(i,(cnx, True))
result = cnx
break
if result:
return result
if len(self._connections) >= self._maxconn:
# try to remove the oldest connection not used
for i, (cnx, used) in enumerate(self._connections):
if not used:
self._connections.pop(i)
self._debug_dsn('Removing old connection at index %d: %r', i, cnx.dsn, dsn_pos=1)
break
else:
# note: this code is called only if the for loop has completed (no break)
raise PoolError('The Connection Pool Is Full')
try:
result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection)
except psycopg2.Error, e:
self.__logger.exception('Connection to the database failed')
raise
def putconn(self, conn=None, key=None, close=False):
if self.closed: raise PoolError("connection pool is closed")
if close:
conn.close()
else:
self._pool.put(conn)
