def _gevent_did_patch(_event): try: from psycopg2.extensions import set_wait_callback except ImportError: pass else: set_wait_callback(_GeventPsycopg2WaitCallback())
def make_psycopg_green(): """Configure Psycopg to be used with gevent in non-blocking way.""" if not hasattr(extensions, 'set_wait_callback'): raise ImportError( "support for coroutines not available in this Psycopg version (%s)" % psycopg2.__version__) extensions.set_wait_callback(gevent_wait_callback)
def make_asynchronous(): try: extensions.POLL_OK except AttributeError: # pragma nocover raise ImproperlyConfigured( 'Psycopg2 does not have support for asynchronous connections. ' 'You need at least version 2.2.0 of Psycopg2.') extensions.set_wait_callback(psycopg2_wait_callback)
def unmake_psycopg_green(): """Undo make_psycopg_green().""" if not hasattr(extensions, 'set_wait_callback'): raise ImportError( "support for coroutines not available in this Psycopg version (%s)" % psycopg2.__version__) extensions.set_wait_callback(None)
def patch_psycopg(): """Configure Psycopg to be used with gevent in non-blocking way.""" if not hasattr(extensions, 'set_wait_callback'): raise ImportError( "support for coroutines not available in this Psycopg version (%s)" % psycopg2.__version__) extensions.set_wait_callback(gevent_wait_callback)
def make_psycopg_green(): """Configure psycopg2 to call our wait function """ if not hasattr(extensions, 'set_wait_callback'): raise ImportError('support for coroutines not available in this psycopg version ({})' .format(psycopg2.__version__)) extensions.set_wait_callback(psycopg2_wait_cb)
def make_asynchronous(): try: extensions.POLL_OK except AttributeError: raise ImproperlyConfigured( 'Psycopg2 does not have support for asynchronous connections. ' 'You need at least version 2.2.0 of Psycopg2.') extensions.set_wait_callback(psycopg2_wait_callback)
def make_psycopg_green(): """Configure psycopg2 to call our wait function """ if not hasattr(extensions, 'set_wait_callback'): raise ImportError( 'support for coroutines not available in this psycopg version ({})' .format(psycopg2.__version__)) extensions.set_wait_callback(psycopg2_wait_cb)
def _set_wait_callback(is_virtual_database): global _wait_callback_is_set if _wait_callback_is_set: return _wait_callback_is_set = True if is_virtual_database: return # When running a query, make pressing CTRL+C raise a KeyboardInterrupt # See http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ # See also https://github.com/psycopg/psycopg2/issues/468 ext.set_wait_callback(_wait_select)
def make_green(engine): """ Set up psycopg2 & SQLAlchemy to be greenlet-friendly. Note: psycogreen does not really monkey patch psycopg2 in the manner that gevent monkey patches socket. """ log.warn('Making the system green...') extensions.set_wait_callback(gevent_wait_callback) # Assuming that gevent monkey patched the builtin # threading library, we're likely good to use # SQLAlchemy's QueuePool, which is the default # pool class. However, we need to make it use # threadlocal connections # # engine.pool._use_threadlocal = True
def main(): global running, channel_mngr, workers # Make green psycopg: extensions.set_wait_callback(gevent_wait_callback) # Decide how many processes to use try: proc_count = int(sys.argv[1]) except: proc_count = multiprocessing.cpu_count() workers = [None] * proc_count channel.init(proc_count) gevent.signal(signal.SIGINT, sig_quit) gevent.signal(signal.SIGTERM, sig_quit) # Remove the sockets directory shutil.rmtree(sock_dir, True) os.mkdir(sock_dir) # Spawn the reloaders for the workers reloaders = [gevent.spawn(reloader, i) for i in xrange(proc_count)] setproctitle.setproctitle('diggems master') # Channel manager process reloader: while running: print 'Starting channel manager process.' proc = gipc.start_process(channel.rpc_dispatcher, daemon=True, name='channel_mngr') proc.join() print 'Channel manager process has quit.' print 'Done with channel manager' gevent.joinall(reloaders) print 'All done, quiting'
def db_config(name, user, password, host='127.0.0.1', port=5432, pool_size=10, pool_block=1, patch_psycopg2_with_gevent=True, log=None, **kwargs): ### _db_config _db_config.clear() _db_config.update(database=name, user=user, password=password, host=host, port=port, **kwargs) ### pool global _db_pool connections_to_create = pool_size - _db_pool.qsize() if connections_to_create > 0: # Create connections. if pool_block > connections_to_create: pool_block = connections_to_create _connect(pool_block) gevent.spawn(_connect, connections_to_create - pool_block) else: # Delete connections. for _ in xrange(-connections_to_create): try: db_conn = _db_pool.get(block=False) except Empty: break try: db_conn.close() except Exception: pass ### patch_psycopg2_with_gevent if patch_psycopg2_with_gevent: extensions.set_wait_callback(_gevent_wait_callback) ### log global _log _log = log
def gevent_wait_callback(conn, timeout=None): """A wait callback useful to allow gevent to work with Psycopg.""" while 1: state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == extensions.POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise OperationalError( "Bad result from poll: %r" % state) extensions.set_wait_callback(gevent_wait_callback) class DatabaseConnectionPool(object): def __init__(self, maxsize=100): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize self.pool = Queue() self.size = 0 def get(self): pool = self.pool if self.size >= self.maxsize or pool.qsize(): return pool.get()
from sakura.common.access import GRANT_LEVELS # psycopg should let gevent switch to other greenlets def wait_callback(conn, timeout=None): while True: state = conn.poll() if state == POLL_OK: break elif state == POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise psycopg2.OperationalError("Bad result from poll: %r" % state) set_wait_callback(wait_callback) DEBUG_CURSORS=False #DEBUG_CURSORS=True TYPES_SAKURA_TO_PG = { 'int8': 'smallint', 'int16': 'smallint', 'int32': 'integer', 'int64': 'bigint', 'float32': 'real', 'float64': 'double precision', 'string': 'text', 'bool': 'boolean', 'date': 'timestamp with time zone' }
elif state == POLL_READ: select.select([conn.fileno()], [], [], _WAIT_SELECT_TIMEOUT) elif state == POLL_WRITE: select.select([], [conn.fileno()], [], _WAIT_SELECT_TIMEOUT) else: raise conn.OperationalError("bad state from poll: %s" % state) except KeyboardInterrupt: conn.cancel() # the loop will be broken by a server error continue # When running a query, make pressing CTRL+C raise a KeyboardInterrupt # See http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ # See also https://github.com/psycopg/psycopg2/issues/468 ext.set_wait_callback(_wait_select) def register_date_typecasters(connection): """ Casts date and timestamp values to string, resolves issues with out of range dates (e.g. BC) which psycopg2 can't handle """ def cast_date(value, cursor): return value cursor = connection.cursor() cursor.execute('SELECT NULL::date') date_oid = cursor.description[0][1] cursor.execute('SELECT NULL::timestamp') timestamp_oid = cursor.description[0][1] cursor.execute('SELECT NULL::timestamp with time zone')
def json_contains(self, column, json): return JSONContains(Cast(column, 'jsonb'), Cast(json, 'jsonb')) register_type(UNICODE) if PYDATE: register_type(PYDATE) if PYDATETIME: register_type(PYDATETIME) if PYTIME: register_type(PYTIME) if PYINTERVAL: register_type(PYINTERVAL) register_adapter(float, lambda value: AsIs(repr(value))) register_adapter(Decimal, lambda value: AsIs(str(value))) def convert_json(value): from trytond.protocols.jsonrpc import JSONDecoder return json.loads(value, object_hook=JSONDecoder()) register_default_json(loads=convert_json) register_default_jsonb(loads=convert_json) if is_gevent_monkey_patched(): from psycopg2.extensions import set_wait_callback from psycopg2.extras import wait_select set_wait_callback(wait_select)
while 1: if panic: raise Exception('whatever') state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: trampoline(conn.fileno(), read=True) elif state == extensions.POLL_WRITE: trampoline(conn.fileno(), write=True) else: raise psycopg2.OperationalError( "Bad result from poll: %r" % state) extensions.set_wait_callback(wait_cb) # SIGHUP handler to inject a fail in the callback def handler(signum, frame): panic.append(True) signal.signal(signal.SIGHUP, handler) # Simulate another green thread working def worker(): while 1: print "I'm working"
# Gevent Monkey patching def gevent_wait_callback(conn, timeout=None): """A wait callback useful to allow gevent to work with Psycopg.""" while 1: state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == extensions.POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise OperationalError("Bad result from poll: %r" % state) extensions.set_wait_callback(gevent_wait_callback) # End Gevent Monkey patching # Set JSON to Pyon default simplejson to get str instead of unicode in deserialization register_default_json(None, globally=True, loads=json.loads) # THREAD (GEVENT) LOCAL - Holds current transaction and per request stats db_context = threading.local() class DatabaseConnectionPool(object): """ Gevent compliant database connection pool """ def __init__(self, maxsize=100): if not isinstance(maxsize, (int, long)): raise TypeError('Expected integer, got %r' % (maxsize, )) self.maxsize = maxsize # Maximum connections (pool + checkout out)
def pulsar_wait_callback(conn, timeout=None): """A wait callback useful to allow gevent to work with Psycopg.""" while 1: state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == extensions.POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise psycopg2.OperationalError("Bad result from poll: %r" % state) extensions.set_wait_callback(pulsar_wait_callback) class Async(object): def wait(self, callback, errback=None, registered=False): exc = None loop = self._loop try: state = self.connection.poll() except Exception: exc = sys.exc_info() if registered: loop.remove_connector(self._sock_fd) else: if state == POLL_OK: if registered:
future._loop.add_reader(fileno, _done_wait_fd, fileno, future, read) else: future._loop.add_writer(fileno, _done_wait_fd, fileno, future, read) # switch back to parent greenlet parent.switch(future) # Back on the child greenlet. Raise error if there is one future.result() def _done_wait_fd(fd, future, read): try: if read: future._loop.remove_reader(fd) else: future._loop.remove_writer(fd) except Exception as exc: future.set_exception(exc) else: future.set_result(None) try: extensions.POLL_OK except AttributeError: # pragma nocover from pulsar import ImproperlyConfigured raise ImproperlyConfigured( 'Psycopg2 does not have support for asynchronous connections. ' 'You need at least version 2.2.0 of Psycopg2.') extensions.set_wait_callback(psycopg2_wait_callback)
def make_psycopg_green(): if not hasattr(extensions, 'set_wait_callback'): raise ImportError( "Support for coroutines is available only from Psycopg 2.2.0") extensions.set_wait_callback(gevent_wait_callback)
def patch(): gevent_psycopg2.monkey_patch() extensions.set_wait_callback(gevent_wait_callback)
def patch_psycopg(): extensions.set_wait_callback(_psycopg_gevent_wait)
# Cast all database input to unicode automatically. # See http://initd.org/psycopg/docs/usage.html#unicode-handling for more info. ext.register_type(ext.UNICODE) ext.register_type(ext.UNICODEARRAY) ext.register_type(ext.new_type((705,), "UNKNOWN", ext.UNICODE)) # See https://github.com/dbcli/pgcli/issues/426 for more details. # This registers a unicode type caster for datatype 'RECORD'. ext.register_type(ext.new_type((2249,), "RECORD", ext.UNICODE)) # Cast bytea fields to text. By default, this will render as hex strings with # Postgres 9+ and as escaped binary in earlier versions. ext.register_type(ext.new_type((17,), 'BYTEA_TEXT', psycopg2.STRING)) # When running a query, make pressing CTRL+C raise a KeyboardInterrupt # See http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ ext.set_wait_callback(psycopg2.extras.wait_select) def register_date_typecasters(connection): """ Casts date and timestamp values to string, resolves issues with out of range dates (e.g. BC) which psycopg2 can't handle """ def cast_date(value, cursor): return value cursor = connection.cursor() cursor.execute('SELECT NULL::date') date_oid = cursor.description[0][1] cursor.execute('SELECT NULL::timestamp') timestamp_oid = cursor.description[0][1] cursor.execute('SELECT NULL::timestamptz')
_logger = logging.getLogger(__name__) # Cast all database input to unicode automatically. # See http://initd.org/psycopg/docs/usage.html#unicode-handling for more info. ext.register_type(ext.UNICODE) ext.register_type(ext.UNICODEARRAY) ext.register_type(ext.new_type((705,), "UNKNOWN", ext.UNICODE)) # Cast bytea fields to text. By default, this will render as hex strings with # Postgres 9+ and as escaped binary in earlier versions. ext.register_type(ext.new_type((17,), 'BYTEA_TEXT', psycopg2.STRING)) # When running a query, make pressing CTRL+C raise a KeyboardInterrupt # See http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ ext.set_wait_callback(psycopg2.extras.wait_select) def register_json_typecasters(conn, loads_fn): """Set the function for converting JSON data for a connection. Use the supplied function to decode JSON data returned from the database via the given connection. The function should accept a single argument of the data as a string encoded in the database's character encoding. psycopg2's default handler for JSON data is json.loads. http://initd.org/psycopg/docs/extras.html#json-adaptation This function attempts to register the typecaster for both JSON and JSONB types. Returns a set that is a subset of {'json', 'jsonb'} indicating which types
def create_engine(self): """Create the database engine""" log.debug("Creating ENGINE") options = { 'convert_unicode': self.native_unicode, 'poolclass': GreenQueuePool, 'pool_size': self.pool_size, 'pool_recycle': self.pool_recycle, 'pool_timeout': self.pool_timeout } if self.database_uri.startswith('sqlite:'): options.pop('pool_timeout') match = _sqlite_re.match(self.database_uri) if match is None: raise ArgumentError('Could not parse rfc1738 URL') database, query = match.groups() if database is None: database = ':memory:' if query: query = url_decode(query).to_dict() else: query = {} info = URL('sqlite', database=database, query=query) pool_size = options.get('pool_size', 0) # we go to memory and the pool size was explicitly set to 0 # which is fail. Let the user know that if info.database in (None, '', ':memory:'): if pool_size == 0: raise RuntimeError('SQLite in memory database with an ' 'empty queue not possible due to data ' 'loss.') # if pool size is None or explicitly set to 0 we assume the # user did not want a queue for this sqlite connection and # hook in the null pool. elif not pool_size: log.warn("SQLite database is using the NullPool") from sqlalchemy.pool import NullPool options['poolclass'] = NullPool else: info = make_url(self.database_uri) # if mysql is the database engine, god forbid, and no connection # encoding is provided we set it to utf-8 if info.drivername == 'mysql': info.query.setdefault('charset', 'utf8') options.setdefault('pool_size', 10) options.setdefault('pool_recycle', 7200) elif info.drivername.startswith('postgresql+psycopg2'): from psycopg2 import extensions, OperationalError from gevent.socket import wait_read, wait_write options['use_native_unicode'] = self.native_unicode def wait_callback(conn, timeout=None): """ A wait callback useful to allow gevent to work with Psycopg. https://bitbucket.org/dvarrazzo/psycogreen/src/tip/gevent/ """ while True: state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == extensions.POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise OperationalError( "Bad result from poll: %r" % state ) if hasattr(extensions, 'set_wait_callback'): extensions.set_wait_callback(wait_callback) dialect_cls = info.get_dialect() # get the correct DBAPI base on connection url dbapi_args = {} dbapi = dialect_cls.dbapi(**dbapi_args) # create the dialect dialect_args = {'dbapi':dbapi} dialect = dialect_cls(**dialect_args) # assemble connection arguments for this dialect (cargs, connection_params) = dialect.create_connect_args(info) log.debug("CARGS: %s; CONNECTION_PARAMS: %s;", cargs, connection_params) log.debug("Creating db engine. info: %s; options: %s;", info, options) engine = sqlalchemy.create_engine(info, **options) database_engine_created.send(self, engine=engine)
def monkey_patch(): """Configure psycopg2 to be used with gevent in non-blocking way.""" extensions.set_wait_callback(gevent_wait_callback)
def pulsar_wait_callback(conn, timeout=None): """A wait callback useful to allow gevent to work with Psycopg.""" while 1: state = conn.poll() if state == extensions.POLL_OK: break elif state == extensions.POLL_READ: wait_read(conn.fileno(), timeout=timeout) elif state == extensions.POLL_WRITE: wait_write(conn.fileno(), timeout=timeout) else: raise psycopg2.OperationalError( "Bad result from poll: %r" % state) extensions.set_wait_callback(pulsar_wait_callback) class Async(object): def wait(self, callback, errback=None, registered=False): exc = None loop = self._loop try: state = self.connection.poll() except Exception: exc = sys.exc_info() if registered: loop.remove_connector(self._sock_fd) else: if state == POLL_OK: