def _put_conn(self, conn, key=None, close=False, fail_silently=False): """Stow away a connection.""" if self._disable_pooling: self._release(conn) return self._log('Putting away %s%s' % (conn, ' key=' + key if key else '')) if self._disposed: if fail_silently: return raise PoolError('Connection pool is disposed') if key is None: key = self._rused.get(id(conn)) if not key: raise PoolError('Trying to put un-keyed connection') if len(self._pool) < self.max_conn and not close: # Return the connection into a consistent state before putting # it back into the pool if not conn.closed: status = conn.get_transaction_status() if status == _ext.TRANSACTION_STATUS_UNKNOWN: # server connection lost self._log('Connection lost. Closing %s' % conn) self._release(conn.close) elif status != _ext.TRANSACTION_STATUS_IDLE: # connection in error or in transaction self._log( 'Connection is in transaction. Rolling back %s' % conn) conn.rollback() self._pool.append(conn) else: # regular idle connection self._pool.append(conn) # If the connection is closed, we just discard it. else: self._log( '''Closing (pool exhausted or explicit close requested) %s''' % conn) self._release(conn) self._purge_expired_connections() # here we check for the presence of key because it can happen that a # thread tries to put back a connection after a call to close if not self._disposed or key in self._used: del self._used[key] del self._rused[id(conn)]
def _release_all(self): """Release all connections. Note that this can lead to some code fail badly when trying to use an already closed connection. If you call .release_all() make sure your code can deal with it. """ # Make sure that all connections lying about are collected before we go on. try: gc.collect() except (TypeError, AttributeError): # We've detected that we're being called in an incomplete # finalization state, we just bail out, leaving the connections # to take care of themselves. return if self._disposed: raise PoolError('Connection pool is disposed') if not self._disable_pooling: close_list = self._pool + list(self._used.values()) self._log('Closing %d connection(s)' % len(close_list)) for conn in close_list: try: conn.close() except: pass self._disposed = True self._pool = [] self._used = {}
def borrow(self, connection_info): """ :param dict connection_info: dict of psql connection keywords :rtype: PsycoConnection """ # free dead and leaked connections for i, (cnx, _) in tools.reverse_enumerate(self._connections): if cnx.closed: self._connections.pop(i) self._debug('Removing closed connection at index %d: %r', i, cnx.dsn) continue if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) _logger.info('%r: Free leaked connection to %r', self, cnx.dsn) for i, (cnx, used) in enumerate(self._connections): if not used and cnx._original_dsn == connection_info: try: cnx.reset() except psycopg2.OperationalError: self._debug('Cannot reset connection at index %d: %r', i, cnx.dsn) # psycopg2 2.4.4 and earlier do not allow closing a closed connection if not cnx.closed: cnx.close() continue self._connections.pop(i) self._connections.append((cnx, True)) self._debug('Borrow existing connection to %r at index %d', cnx.dsn, i) return cnx if len(self._connections) >= self._maxconn: # try to remove the oldest connection not used for i, (cnx, used) in enumerate(self._connections): if not used: self._connections.pop(i) if not cnx.closed: cnx.close() self._debug('Removing old connection at index %d: %r', i, cnx.dsn) break else: # note: this code is called only if the for loop has completed (no break) raise PoolError('The Connection Pool Is Full') try: result = psycopg2.connect(connection_factory=PsycoConnection, **connection_info) except psycopg2.Error: _logger.info('Connection to the database failed') raise result._original_dsn = connection_info self._connections.append((result, True)) self._debug('Create new connection') return result
def borrow(self, dsn): self._debug('Borrow connection to %r', dsn) # free leaked connections for i, (cnx, _) in tools.reverse_enumerate(self._connections): if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) self._debug('Free leaked connection to %r', cnx.dsn) for i, (cnx, used) in enumerate(self._connections): if not used and dsn_are_equals(cnx.dsn, dsn): self._connections.pop(i) self._connections.append((cnx, True)) self._debug('Existing connection found at index %d', i) return cnx if len(self._connections) >= self._maxconn: # try to remove the oldest connection not used for i, (cnx, used) in enumerate(self._connections): if not used: self._connections.pop(i) self._debug('Removing old connection at index %d: %r', i, cnx.dsn) break else: # note: this code is called only if the for loop has completed (no break) raise PoolError('The Connection Pool Is Full') result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection) self._connections.append((result, True)) self._debug('Create new connection') return result
def _getconn(self, key=None): """Get a free connection and assign it to 'key' if not None.""" if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._getkey() if key in self._used: return self._used[key] if self._pool: self._used[key] = conn = self._pool.pop() self._rused[id(conn)] = key return conn else: if len(self._used) == self.maxconn: raise PoolError("connection pool exausted") return self._connect(key)
def borrow(self, dsn): self._debug('Borrow connection to %r', dsn) # free dead and leaked connections for i, (cnx, _) in tools.reverse_enumerate(self._connections): if cnx.closed: self._connections.pop(i) self._debug('Removing closed connection at index %d: %r', i, cnx.dsn) continue if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) _logger.warning('%r: Free leaked connection to %r', self, cnx.dsn) for i, (cnx, used) in enumerate(self._connections): if not used and dsn_are_equals(cnx.dsn, dsn): try: cnx.reset() except psycopg2.OperationalError: self._debug('Cannot reset connection at index %d: %r', i, cnx.dsn) # psycopg2 2.4.4 and earlier do not allow closing a closed connection if not cnx.closed: cnx.close() continue self._connections.pop(i) self._connections.append((cnx, True)) self._debug('Existing connection found at index %d', i) return cnx if len(self._connections) >= self._maxconn: # try to remove the oldest connection not used for i, (cnx, used) in enumerate(self._connections): if not used: self._connections.pop(i) if not cnx.closed: cnx.close() self._debug('Removing old connection at index %d: %r', i, cnx.dsn) break else: # note: this code is called only if the for loop has completed (no break) raise PoolError('The Connection Pool Is Full') try: result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection) except psycopg2.Error: _logger.exception('Connection to the database failed') raise self._connections.append((result, True)) self._debug('Create new connection') return result
def _putconn(self, conn, key=None, close=False): """Put away a connection.""" if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._rused[id(conn)] if not key: raise PoolError("trying to put unkeyed connection") if len(self._pool) < self.minconn and not close: self._pool.append(conn) else: conn.close() # here we check for the presence of key because it can happen that a # thread tries to put back a connection after a call to close if not self.closed or key in self._used: del self._used[key] del self._rused[id(conn)]
def closeall(self): if self.closed: raise PoolError("connection pool is closed") # TODO -- might be better to do this decrementing self.size? while not self._pool.empty(): conn = self._pool.get_nowait() try: conn.close() except Exception: pass self.closed = True
def __enter__(self): for _ in range(TRAILS): try: self.conn = Connection.connection_pool.getconn() self.conn.autocommit = False self.cursor = self.conn.cursor() return self except PoolError: logger.info('Pool doesn\'t have available connection. Please wait') time.sleep(int(POOL_DELAY)) raise PoolError('Can\'t get a connection.')
def _get_conn(self, key=None): """Get a free connection and assign it to 'key' if not None.""" if self._disposed: raise PoolError('Connection pool is disposed') if self._disable_pooling: return self._connect(key) if key is None: key = self._get_key() if key in self._used: return self._used[key] if self._pool: self._used[key] = conn = self._pool.pop() self._rused[id(conn)] = key self._tused[id(conn)] = time.time() return conn else: if len(self._used) == self.max_conn: raise PoolError('Connection pool exhausted') return self._connect(key)
def give_back(self, connection, keep_in_pool=True): self._debug('Give back connection to %r', connection.dsn) for i, (cnx, used) in enumerate(self._connections): if cnx is connection: self._connections.pop(i) if keep_in_pool: self._connections.append((cnx, False)) self._debug('Put connection to %r in pool', cnx.dsn) else: self._debug('Forgot connection to %r', cnx.dsn) break else: raise PoolError('This connection does not below to the pool')
def give_back(self, connection, keep_in_pool=True): self._debug_dsn('Give back connection to %r', connection.dsn) for i, (cnx, used) in enumerate(self._connections): if cnx is connection: self._connections.pop(i) if keep_in_pool and not (cnx.closed or not cnx.status): self._connections.insert(i,(cnx, False)) self._debug_dsn('Put connection to %r back in pool', cnx.dsn) else: self._debug_dsn('Forgot connection to %r', cnx.dsn) break else: raise PoolError('This connection does not below to the pool')
def _getconn(self, key=None, exactly=False): """Get a free connection and assign it to 'key' if not None.""" internal_key = False if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._getkey() internal_key = True if key in self._used: return self._used[key] if not internal_key and exactly: return None if self._pool: self._used[key] = conn = self._pool.pop() self._rused[id(conn)] = key self._tused[id(conn)] = datetime.datetime.now() return conn else: if len(self._used) == self.maxconn: raise PoolError("connection pool exausted") return self._connect(key)
def getconn(self, key=None): if self.closed: raise PoolError("connection pool is closed") pool = self._pool if self.size >= self.maxconn or pool.qsize(): conn = pool.get() else: try: conn = self._connect() except: raise return conn
def _closeall(self): """Close all connections. Note that this can lead to some code fail badly when trying to use an already closed connection. If you call .closeall() make sure your code can deal with it. """ if self.closed: raise PoolError("connection pool is closed") for conn in self._pool + list(self._used.values()): try: conn.close() except: pass self.closed = True
def _putconn(self, conn, key=None, close=False): """Put away a connection.""" if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._rused.get(id(conn)) if not key: raise PoolError( "trying to put unkeyed [{key}] connection".format(key=key)) if len(self._pool) < self.maxconn and not close: # Return the connection into a consistent state before putting # it back into the pool if not conn.closed: status = conn.get_transaction_status() if status == _ext.TRANSACTION_STATUS_UNKNOWN: # server connection lost self._disconnect(conn.close) elif status != _ext.TRANSACTION_STATUS_IDLE: # connection in error or in transaction conn.rollback() self._pool.append(conn) else: # regular idle connection self._pool.append(conn) # If the connection is closed, we just discard it. else: self._disconnect(conn) self.clear_expired_connections() # here we check for the presence of key because it can happen that a # thread tries to put back a connection after a call to close if not self.closed or key in self._used: del self._used[key] del self._rused[id(conn)]
def give_back(self, connection, keep_in_pool=True): self._debug('Give back connection to %r', connection.dsn) for i, (cnx, used, time_used) in enumerate(self._connections): if cnx is connection: self._connections.pop(i) if keep_in_pool: self._connections.append((cnx, False, time.time())) self._debug('Put connection to %r in pool', cnx.dsn) else: self._debug('Forgot connection to %r', cnx.dsn) cnx.close() break # decodio: close connections elif self._idle_timeout > 0.0 and not used: if time_used > (time.time() - self._idle_timeout): self._debug('Idle connection timeout. Closing %r.', cnx.dsn) cnx.close() else: raise PoolError('This connection does not belong to the pool')
def borrow(self, dsn): self._debug('Borrow connection to %r', dsn) # free dead and leaked connections time_now = time.time() for i, (cnx, _) in tools.reverse_enumerate(self._connections): if (cnx.closed or cnx.status == psycopg2.extensions.STATUS_READY and (CONNECTION_LIFETIME and time_now - cnx.init_time > CONNECTION_LIFETIME) or (CURSOR_TIMEOUT and time_now - cnx.last_execution_time > CURSOR_TIMEOUT)): self._connections.pop(i) self._debug('Removing closed connection at index %d: %r', i, cnx.dsn) if not cnx.closed: cnx.close() # Close unclosed connection. continue if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) _logger.warning('%r: Free leaked connection to %r', self, cnx.dsn) for i, (cnx, used) in enumerate(self._connections): if not used and dsn_are_equals(cnx.dsn, dsn): self._connections.pop(i) try: cnx.reset() except psycopg2.OperationalError: self._debug( 'Cannot reset connection at index %d: %r, removing it', i, cnx.dsn) # psycopg2 2.4.4 and earlier do not allow closing a closed connection if not cnx.closed: cnx.close() continue self._connections.append((cnx, True)) self._debug('Existing connection found at index %d', i) return cnx if len(self._connections) >= self._maxconn: # try to remove the oldest connection not used for i, (cnx, used) in enumerate(self._connections): if not used: self._connections.pop(i) self._debug('Removing old connection at index %d: %r', i, cnx.dsn) break else: # note: this code is called only if the for loop has completed (no break) raise PoolError('The Connection Pool Is Full') try: result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection) result.init_time = time.time() result.executing = False except psycopg2.Error: _logger.exception('Connection to the database failed') raise self._connections.append((result, True)) self._debug('Create new connection') return result
class ConnectionPool(object): """ The pool of connections to database(s) Keep a set of connections to pg databases open, and reuse them to open cursors for all transactions. The connections are *not* automatically closed. Only a close_db() can trigger that. """ __logger = logging.getLogger('db.connection_pool') def locked(fun): @wraps(fun) def _locked(self, *args, **kwargs): self._lock.acquire() try: return fun(self, *args, **kwargs) finally: self._lock.release() return _locked def __init__(self, maxconn=64, pgmode=None): self._connections = [] self._maxconn = max(maxconn, 1) self._lock = threading.Lock() self._debug_pool = tools.config.get_misc('debug', 'db_pool', False) self.sql_stats = {} if pgmode: # not None or False Cursor.set_pgmode(pgmode) def __del__(self): # explicitly free them del self._connections if self.sql_stats: self.print_all_stats() def __repr__(self): used = len([1 for c, u in self._connections[:] if u]) count = len(self._connections) return "ConnectionPool(used=%d/count=%d/max=%d)" % (used, count, self._maxconn) def _debug(self, msg, *args): if self._debug_pool: msg = '%r ' + msg self.__logger.debug(msg, self, *args) def _debug_dsn(self, msg, *args, **kwargs): """Debug function, that will decode the dsn_pos'th argument as dsn @param kwargs may only contain 'dsn_pos' """ if not self._debug_pool: return def cleanup(dsn): cl = [x for x in dsn.strip().split() if x.split('=', 1)[0] != 'password'] return ' '.join(cl) largs = list(args) dsn_pos = kwargs.pop('dsn_pos', 0) if kwargs: raise TypeError("Unknown keyword argument(s): %r" % kwargs) assert dsn_pos < len(largs) largs[dsn_pos] = cleanup(largs[dsn_pos]) msg = '%r ' + msg self.__logger.debug(msg, self, *largs) def set_pool_debug(self, do_debug = True): self._debug_pool = do_debug self.__logger.info("Debugging set to %s" % str(do_debug)) @locked def borrow(self, dsn, do_cursor=False): self._debug_dsn('Borrow connection to %r', dsn) # free leaked connections for i, (cnx, _) in tools.reverse_enumerate(self._connections): if getattr(cnx, 'leaked', False): delattr(cnx, 'leaked') self._connections.pop(i) self._connections.append((cnx, False)) self._debug_dsn('Free leaked connection to %r', cnx.dsn) result = None for i, (cnx, used) in enumerate(self._connections): if not used and dsn_are_equals(cnx.dsn, dsn): self._connections.pop(i) try: if psycopg2.__version__ >= '2.2' : pr = cnx.poll() self._debug("Poll: %d", pr) except OperationalError, e: self._debug("Error in poll: %s" % e) continue if cnx.closed or not cnx.status: # something is wrong with that connection, let it out self._debug("Troubled connection ") continue self._debug('Existing connection found at index %d', i) if do_cursor: try: cur = cnx.cursor(cursor_factory=psycopg1cursor) if psycopg2.__version__ < '2.2' and not cur.isready(): continue if cur.closed: continue self._connections.insert(i,(cnx, True)) result = (cnx, cur) except OperationalError: continue else: self._connections.insert(i,(cnx, True)) result = cnx break if result: return result if len(self._connections) >= self._maxconn: # try to remove the oldest connection not used for i, (cnx, used) in enumerate(self._connections): if not used: self._connections.pop(i) self._debug_dsn('Removing old connection at index %d: %r', i, cnx.dsn, dsn_pos=1) break else: # note: this code is called only if the for loop has completed (no break) raise PoolError('The Connection Pool Is Full') try: result = psycopg2.connect(dsn=dsn, connection_factory=PsycoConnection) except psycopg2.Error, e: self.__logger.exception('Connection to the database failed') raise
def putconn(self, conn=None, key=None, close=False): if self.closed: raise PoolError("connection pool is closed") if close: conn.close() else: self._pool.put(conn)