def __init__(self, host, database, user, password, debug=False):
self.host = host
self.database = database
# 用于判断是否打印sql
self.debug = debug
args = {'database': database}
if user is not None:
args['user'] = user
if password is not None:
args['password'] = password
pair = host.split(':')
if len(pair) == 2:
args['host'] = pair[0]
args['port'] = int(pair[1])
else:
args['host'] = pair[0]
args['port'] = 3388
self.args = args
self._db = None
# 单位为秒
self._db_operation_timeout = 300
# 连接池
self.conn_pool = pool.manage(
MySQLdb, pool_size=20, max_overflow=40, timeout=self._db_operation_timeout, recycle=1800)
self.reconnection()
def testbadargs(self):
manager = pool.manage(mock_dbapi)
try:
connection = manager.connect(None)
except:
pass
def testnonthreadlocalmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal=False)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
self.assert_(connection.cursor() is not None)
self.assert_(connection is not connection2)
def test_non_thread_local_manager(self):
manager = pool.manage(MockDBAPI(), use_threadlocal=False)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
self.assert_(connection.cursor() is not None)
self.assert_(connection is not connection2)
def testnonthreadlocalmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal=False)
connection = manager.connect("foo.db")
connection2 = manager.connect("foo.db")
self.assert_(connection.cursor() is not None)
self.assert_(connection is not connection2)
def test_non_thread_local_manager(self):
manager = pool.manage(MockDBAPI(), use_threadlocal=False)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
self.assert_(connection.cursor() is not None)
self.assert_(connection is not connection2)
def patch_oracle():
try:
from django.db.backends.oracle import base as oracle_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(oracle_base, "_Database"):
oracle_base._Database = oracle_base.Database
oracle_base.Database = manage(oracle_base._Database, **POOL_SETTINGS)
def patch_postgresql():
try:
from django.db.backends.postgresql_psycopg2 import base as pgsql_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(pgsql_base, "_Database"):
pgsql_base._Database = pgsql_base.Database
pgsql_base.Database = manage(pgsql_base._Database, **POOL_SETTINGS)
def patch_sqlite3():
try:
from django.db.backends.sqlite3 import base as sqlite3_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(sqlite3_base, "_Database"):
sqlite3_base._Database = sqlite3_base.Database
sqlite3_base.Database = manage(sqlite3_base._Database, **POOL_SETTINGS)
def patch_postgresql():
try:
from django.db.backends.postgresql_psycopg2 import base as pgsql_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(pgsql_base, "_Database"):
pgsql_base._Database = pgsql_base.Database
pgsql_base.Database = manage(pgsql_base._Database, **POOL_SETTINGS)
def patch_sqlite3():
try:
from django.db.backends.sqlite3 import base as sqlite3_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(sqlite3_base, "_Database"):
sqlite3_base._Database = sqlite3_base.Database
sqlite3_base.Database = manage(sqlite3_base._Database, **POOL_SETTINGS)
def testmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal=True)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
connection3 = manager.connect('bar.db')
self.assert_(connection.cursor() is not None)
self.assert_(connection is connection2)
self.assert_(connection2 is not connection3)
def testmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal=True)
connection = manager.connect("foo.db")
connection2 = manager.connect("foo.db")
connection3 = manager.connect("bar.db")
self.assert_(connection.cursor() is not None)
self.assert_(connection is connection2)
self.assert_(connection2 is not connection3)
def testnonthreadlocalmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal = False)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
self.echo( "connection " + repr(connection))
self.assert_(connection.cursor() is not None)
self.assert_(connection is not connection2)
def _get_pool():
"""
Creates one and only one pool using the configured settings
"""
global MYSQLPOOL
if MYSQLPOOL is None:
MYSQLPOOL = ManagerProxy(pool.manage(Database, **POOL_SETTINGS))
setattr(MYSQLPOOL, '_pid', os.getpid())
if getattr(MYSQLPOOL, '_pid', None) != os.getpid():
pool.clear_managers()
return MYSQLPOOL
def testmanager(self):
manager = pool.manage(mock_dbapi, use_threadlocal=True)
connection = manager.connect('foo.db')
connection2 = manager.connect('foo.db')
connection3 = manager.connect('bar.db')
print "connection " + repr(connection)
self.assert_(connection.cursor() is not None)
self.assert_(connection is connection2)
self.assert_(connection2 is not connection3)
def get_pool():
"Creates one and only one pool using the configured settings."
global MYSQLPOOL
if MYSQLPOOL is None:
backend = getattr(settings, 'MYSQLPOOL_BACKEND', MYSQLPOOL_BACKEND)
backend = getattr(pool, backend)
kwargs = getattr(settings, 'MYSQLPOOL_ARGUMENTS', {})
kwargs.setdefault('poolclass', backend)
# The user can override this, but set it by default for safety.
kwargs.setdefault('recycle', MYSQLPOOL_TIMEOUT)
MYSQLPOOL = pool.manage(OldDatabase, **kwargs)
return MYSQLPOOL
def _get_pool():
"""
Private function to get the database pool instance, or create one if it's not available yet.
:returns: The connection pool instance
"""
global _pool
if _pool is None:
# FIXME: Use pool size and other parameters from config
_pool = dbpool.manage(MySQLdb, pool_size=1, use_threadlocal=True)
return _pool
def _get_pool():
"""
Private function to get the database pool instance, or create one if it's not available yet.
:returns: The connection pool instance
"""
global _pool
if _pool is None:
# FIXME: Use pool size and other parameters from config
_pool = dbpool.manage(MySQLdb, pool_size=1, use_threadlocal=True)
return _pool
def _connect(self):
if cfg['CFG_MISCUTIL_SQL_USE_SQLALCHEMY']:
try:
import sqlalchemy.pool as pool
import MySQLdb as mysqldb
mysqldb = pool.manage(mysqldb, use_threadlocal=True)
connect = mysqldb.connect
except ImportError:
cfg['CFG_MISCUTIL_SQL_USE_SQLALCHEMY'] = False
from MySQLdb import connect
else:
from MySQLdb import connect
return connect
def _connect(self):
if cfg['CFG_MISCUTIL_SQL_USE_SQLALCHEMY']:
try:
import sqlalchemy.pool as pool
import MySQLdb as mysqldb
mysqldb = pool.manage(mysqldb, use_threadlocal=True)
connect = mysqldb.connect
except ImportError:
cfg['CFG_MISCUTIL_SQL_USE_SQLALCHEMY'] = False
from MySQLdb import connect
else:
from MySQLdb import connect
return connect
class MySQL(DatabaseBackend):
'''MySQL Backend Adapter'''
import MySQLdb
import MySQLdb.cursors
import _mysql_exceptions
import sqlalchemy.pool as pool
LABEL = 'MySQL'
CURSOR_DEFAULT = MySQLdb.cursors.Cursor
CURSOR_DICT = MySQLdb.cursors.DictCursor
# Connection pooling
DatabasePool = pool.manage(MySQLdb, recycle=120)
def __init__(self, config):
self.config = config
if self.config.socket.type == 'UNIX':
self.name = self.config.socket.path
self.connection_params = {'unix_socket': self.config.socket.path}
else:
self.name = self.config.socket.host
self.connection_params = {
'host': self.config.socket.host,
'port': self.config.socket.port,
}
self.connection_params.update({
'user': self.config.username,
'passwd': self.config.password
})
def _log(self, _log, message):
_log('%s: %s' % (self.LABEL, message))
def connect(self, unicode=True, cursorclass=CURSOR_DEFAULT):
self.connection_params.update({
'use_unicode': unicode,
'cursorclass': cursorclass,
})
self._log(log.info, 'Asking pool for a connection')
try:
connection = self.DatabasePool.connect(**self.connection_params)
except self.MySQLdb.OperationalError, e:
self._log(log.error, 'Unable to aquire a connection to server %s' \
% self.name)
raise exception.DatabaseError(
'%s: Could not connect to server: %s' % (self.LABEL, e))
self._log(log.debug, 'Successfully aquired connection from pool')
return connection
def test_manager_with_key(self):
class NoKws(object):
def connect(self, arg):
return MockConnection()
manager = pool.manage(NoKws(), use_threadlocal=True)
c1 = manager.connect('foo.db', sa_pool_key="a")
c2 = manager.connect('foo.db', sa_pool_key="b")
c3 = manager.connect('bar.db', sa_pool_key="a")
assert c1.cursor() is not None
assert c1 is not c2
assert c1 is c3
def test_manager(self):
manager = pool.manage(MockDBAPI(), use_threadlocal=True)
c1 = manager.connect('foo.db')
c2 = manager.connect('foo.db')
c3 = manager.connect('bar.db')
c4 = manager.connect("foo.db", bar="bat")
c5 = manager.connect("foo.db", bar="hoho")
c6 = manager.connect("foo.db", bar="bat")
assert c1.cursor() is not None
assert c1 is c2
assert c1 is not c3
assert c4 is c6
assert c4 is not c5
def get_pool():
"""Create one and only one pool using the configured settings."""
global MYSQLPOOL
if MYSQLPOOL is None:
backend_name = getattr(settings, "SHIELD_MYSQL_POOL_BACKEND", DEFAULT_BACKEND)
backend = getattr(local_pool, backend_name)
kwargs = getattr(settings, "SHIELD_MYSQL_POOL_ARGUMENTS", {})
kwargs.setdefault("poolclass", backend)
kwargs.setdefault("recycle", DEFAULT_POOL_TIMEOUT)
MYSQLPOOL = sa_pool.manage(OldDatabase, **kwargs)
setattr(MYSQLPOOL, "_pid", os.getpid())
if getattr(MYSQLPOOL, "_pid", None) != os.getpid():
sa_pool.clear_managers()
return MYSQLPOOL
def test_manager(self):
manager = pool.manage(MockDBAPI(), use_threadlocal=True)
c1 = manager.connect('foo.db')
c2 = manager.connect('foo.db')
c3 = manager.connect('bar.db')
c4 = manager.connect("foo.db", bar="bat")
c5 = manager.connect("foo.db", bar="hoho")
c6 = manager.connect("foo.db", bar="bat")
assert c1.cursor() is not None
assert c1 is c2
assert c1 is not c3
assert c4 is c6
assert c4 is not c5
def get_pool():
"""Create one and only one pool using the configured settings."""
global MYSQLPOOL
if MYSQLPOOL is None:
backend_name = getattr(settings, 'MYSQLPOOL_BACKEND', DEFAULT_BACKEND)
backend = getattr(pool, backend_name)
kwargs = getattr(settings, 'MYSQLPOOL_ARGUMENTS', {})
kwargs.setdefault('poolclass', backend)
kwargs.setdefault('recycle', DEFAULT_POOL_TIMEOUT)
MYSQLPOOL = pool.manage(OldDatabase, **kwargs)
setattr(MYSQLPOOL, '_pid', os.getpid())
if getattr(MYSQLPOOL, '_pid', None) != os.getpid():
pool.clear_managers()
return MYSQLPOOL
def get_pool():
"""Create one and only one pool using the configured settings."""
global MYSQLPOOL
if MYSQLPOOL is None:
backend_name = getattr(settings, 'MYSQLPOOL_BACKEND', DEFAULT_BACKEND)
backend = getattr(pool, backend_name)
kwargs = getattr(settings, 'MYSQLPOOL_ARGUMENTS', {})
kwargs.setdefault('poolclass', backend)
kwargs.setdefault('recycle', DEFAULT_POOL_TIMEOUT)
MYSQLPOOL = pool.manage(OldDatabase, **kwargs)
setattr(MYSQLPOOL, '_pid', os.getpid())
if getattr(MYSQLPOOL, '_pid', None) != os.getpid():
pool.clear_managers()
return MYSQLPOOL
def patch_mysql():
class hashabledict(dict):
def __hash__(self):
# in python3 dict is sorted
# return hash(tuple(sorted(self.items())))
return hash(tuple(self.items()))
class hashablelist(list):
def __hash__(self):
return hash(tuple(sorted(self)))
class ManagerProxy(object):
def __init__(self, manager):
self.manager = manager
def __getattr__(self, key):
return getattr(self.manager, key)
def connect(self, *args, **kwargs):
if 'conv' in kwargs:
conv = kwargs['conv']
if isinstance(conv, dict):
items = []
for k, v in conv.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['conv'] = hashabledict(items)
if 'ssl' in kwargs:
ssl = kwargs['ssl']
if isinstance(ssl, dict):
items = []
for k, v in ssl.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['ssl'] = hashabledict(items)
return self.manager.connect(*args, **kwargs)
try:
from django.db.backends.mysql import base as mysql_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(mysql_base, "_Database"):
mysql_base._Database = mysql_base.Database
mysql_base.Database = ManagerProxy(
manage(mysql_base._Database, **POOL_SETTINGS))
def init_pool():
if not globals().get('pool_initialized', False):
global pool_initialized
pool_initialized = True
try:
backendname = settings.DATABASES['default']['ENGINE']
backend = load_backend(backendname)
#replace the database object with a proxy.
backend.Database = pool.manage(backend.Database)
backend.DatabaseError = backend.Database.DatabaseError
backend.IntegrityError = backend.Database.IntegrityError
logging.info("Connection Pool initialized")
except:
logging.exception("Connection Pool initialization error")
def __init__(self, db, dbName):
import MySQLdb
import _mysql_exceptions
MySQLdb = pool.manage(MySQLdb)
# url = 'mysql://db=%s' % DBNAME
url = 'mysql://*****:*****@localhost:3306/%s' % DBNAME
eng = create_engine(url)
try:
cxn = eng.raw_connection()
except _mysql_exceptions.OperationalError, e:
# eng1 = create_engine('mysql://user=root')
eng1 = create_engine('mysql://*****:*****@localhost:3306/test')
try:
eng1.execute('drop database %s' % DBNAME)
except _mysql_exceptions.OperationalError, e:
pass
def init_pool():
"""From http://blog.bootstraptoday.com/2012/07/11/django-connection-pooling-using-sqlalchemy-connection-pool/"""
if not globals().get('pool_initialized', False):
global pool_initialized
pool_initialized = True
try:
backendname = settings.DATABASES['default']['ENGINE']
backend = load_backend(backendname)
#replace the database object with a proxy.
backend.Database = pool.manage(backend.Database, pool_size=settings.DB_POOL_SIZE, max_overflow=-1)
backend.DatabaseError = backend.Database.DatabaseError
backend.IntegrityError = backend.Database.IntegrityError
logging.info("Connection Pool initialized")
except:
logging.exception("Connection Pool initialization error")
def test_manager_with_key(self):
dbapi = MockDBAPI()
manager = pool.manage(dbapi, use_threadlocal=True)
c1 = manager.connect('foo.db', sa_pool_key="a")
c2 = manager.connect('foo.db', sa_pool_key="b")
c3 = manager.connect('bar.db', sa_pool_key="a")
assert c1.cursor() is not None
assert c1 is not c2
assert c1 is c3
eq_(dbapi.connect.mock_calls, [
call("foo.db"),
call("foo.db"),
])
def init_pool():
"""From http://blog.bootstraptoday.com/2012/07/11/django-connection-pooling-using-sqlalchemy-connection-pool/"""
if not globals().get('pool_initialized', False):
global pool_initialized
pool_initialized = True
try:
backendname = settings.DATABASES['default']['ENGINE']
backend = load_backend(backendname)
#replace the database object with a proxy.
backend.Database = pool.manage(backend.Database, pool_size=settings.DB_POOL_SIZE, max_overflow=-1)
backend.DatabaseError = backend.Database.DatabaseError
backend.IntegrityError = backend.Database.IntegrityError
logging.info("Connection Pool initialized")
except:
logging.exception("Connection Pool initialization error")
def patch_mysql(pool_options={}):
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
class hashablelist(list):
def __hash__(self):
return hash(tuple(sorted(self)))
class ManagerProxy(object):
def __init__(self, manager):
self.manager = manager
def __getattr__(self, key):
return getattr(self.manager, key)
def connect(self, *args, **kwargs):
if 'conv' in kwargs:
conv = kwargs['conv']
if isinstance(conv, dict):
items = []
for k, v in conv.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['conv'] = hashabledict(items)
if 'ssl' in kwargs:
ssl = kwargs['ssl']
if isinstance(ssl, dict):
items = []
for k, v in ssl.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['ssl'] = hashabledict(items)
return self.manager.connect(*args, **kwargs)
from django.db.backends.mysql import base as mysql_base
POOL_SETTINGS = pool_options
if not hasattr(mysql_base, '_Database'):
mysql_base._Database = mysql_base.Database
mysql_base.Database = ManagerProxy(
manage(mysql_base._Database, **POOL_SETTINGS))
def patch_mysql():
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
class hashablelist(list):
def __hash__(self):
return hash(tuple(sorted(self)))
class ManagerProxy(object):
def __init__(self, manager):
self.manager = manager
def __getattr__(self, key):
return getattr(self.manager, key)
def connect(self, *args, **kwargs):
if 'conv' in kwargs:
conv = kwargs['conv']
if isinstance(conv, dict):
items = []
for k, v in conv.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['conv'] = hashabledict(items)
if 'ssl' in kwargs:
ssl = kwargs['ssl']
if isinstance(ssl, dict):
items = []
for k, v in ssl.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['ssl'] = hashabledict(items)
return self.manager.connect(*args, **kwargs)
try:
from django.db.backends.mysql import base as mysql_base
except (ImproperlyConfigured, ImportError) as e:
return
if not hasattr(mysql_base, "_Database"):
mysql_base._Database = mysql_base.Database
mysql_base.Database = ManagerProxy(manage(mysql_base._Database, **POOL_SETTINGS))
def init_pool():
if not globals().get('pool_initialized', False):
global pool_initialized
pool_initialized = True
try:
backendname = settings.DATABASES['default']['ENGINE']
backend = load_backend(backendname)
#replace the database object with a proxy.
backend.Database = ManagerProxy(pool.manage(backend.Database, **POOL_SETTINGS))
backend.DatabaseError = backend.Database.DatabaseError
backend.IntegrityError = backend.Database.IntegrityError
logger.debug("Initialzied Connection Pool")
except Exception,e:
import traceback
traceback.print_exc()
pass
def test_manager_with_key(self):
dbapi = MockDBAPI()
manager = pool.manage(dbapi, use_threadlocal=True)
c1 = manager.connect('foo.db', sa_pool_key="a")
c2 = manager.connect('foo.db', sa_pool_key="b")
c3 = manager.connect('bar.db', sa_pool_key="a")
assert c1.cursor() is not None
assert c1 is not c2
assert c1 is c3
eq_(dbapi.connect.mock_calls,
[
call("foo.db"),
call("foo.db"),
]
)
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# Initialize config with the basic options
config.init_app(global_conf,
app_conf,
package='srjvkk',
template_engine='mako',
paths=paths)
config['routes.map'] = make_map()
config['pylons.g'] = app_globals.Globals()
config['pylons.h'] = srjvkk.lib.helpers
# Customize templating options via this variable
tmpl_options = config['buffet.template_options']
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
#import psycopg2.extensions
#psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
config['pylons.g'].sa_engine = sa.engine_from_config(config, 'sqlalchemy.')
#config['pylons.g'].sa_engine.connect().connection.set_client_encoding("utf8")
engine = config['pylons.g'].sa_engine
from srjvkk import model
model.init_model(config, engine)
import psycopg2 as psycopg
psycopg = pool.manage(psycopg)
def init_pool():
if not globals().get('pool_initialized', False):
global pool_initialized
pool_initialized = True
try:
backendname = settings.DATABASES['default']['ENGINE']
backend = load_backend(backendname)
#replace the database object with a proxy.
backend.Database = pool.manage(backend.Database,
poolclass=pool.QueuePool,
echo=settings.DEBUG,
recycle=settings.DBPOOL_WAIT_TIMEOUT,
pool_size=settings.DBPOOL_SIZE,
max_overflow=settings.DBPOOL_MAX,
timeout=settings.DBPOOL_INTERNAL_CONN_TIMEOUT)
backend.DatabaseError = backend.Database.DatabaseError
backend.IntegrityError = backend.Database.IntegrityError
logging.info("Connection Pool initialized")
except:
logging.exception("Connection Pool initialization error")
def connect(self, backend=None, host=None, database=None,
user=None, password=None):
"""Connects a database with the given parameters"""
if backend is None:
raise FpdbError('Database backend not defined')
self.backend = backend
self.host = host
self.user = user
self.password = password
self.database = database
if backend == fpdb_db.MYSQL_INNODB:
import MySQLdb
if use_pool:
MySQLdb = pool.manage(MySQLdb, pool_size=5)
try:
self.db = MySQLdb.connect(host=host, user=user, passwd=password, db=database, use_unicode=True)
#TODO: Add port option
except MySQLdb.Error, ex:
if ex.args[0] == 1045:
raise FpdbMySQLAccessDenied(ex.args[0], ex.args[1])
elif ex.args[0] == 2002 or ex.args[0] == 2003: # 2002 is no unix socket, 2003 is no tcp socket
raise FpdbMySQLNoDatabase(ex.args[0], ex.args[1])
else:
print "*** WARNING UNKNOWN MYSQL ERROR", ex
def __init__(self, host, database, user, password, debug=False):
"""
#ORACEL UTF-8 WINDOW XP 很操蛋
"""
if 'NLS_LANG' not in os.environ or os.environ["NLS_LANG"] != 'AMERICAN_AMERICA.AL32UTF8':
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
self.host = host
self.database = database
# 用于判断是否打印sql
self.debug = debug
args = {'database': database}
if user is not None:
args['user'] = user
if password is not None:
args['password'] = password
pair = host.split(':')
if len(pair) == 2:
args['host'] = pair[0]
args['port'] = int(pair[1])
else:
args['host'] = pair[0]
args['port'] = 1521
self.args = args
self._db = None
# 单位为秒
self._db_operation_timeout = 300
# 连接池
self.conn_pool = pool.manage(
psycopg2, pool_size=20, max_overflow=40, timeout=self._db_operation_timeout, recycle=1800)
self.reconnection()
#!/usr/bin/env python
# encoding: utf-8
import sys
import os
import csv
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import numpy as np
#import cx_Oracle as ora
import psycopg2 as pg
import sqlalchemy.pool as pool
pg = pool.manage(pg)
from wpp.config import dbsvrs, wpp_tables, sqls, DB_UPLOAD, MAX_AREA_TRY, CRAWL_LIMIT, \
tbl_field, tbl_forms, tbl_idx, tbl_files
# Joining UNICODE & ASCII chars in WppDB.addAreaLocation(), when returned area
# name(CN) by getAreaName() is not decoded from UTF-8(default in postgres) to ASCII.
#reload(sys)
#sys.setdefaultencoding('utf-8')
def usage():
import time
print """
db.py - Copyleft 2009-%s Yan Xiaotian, [email protected].
Abstraction layer for WPP radiomap DB handling.
usage:
import os
import string
import time
import marshal
import re
from zlib import compress, decompress
from thread import get_ident
from invenio.config import CFG_ACCESS_CONTROL_LEVEL_SITE, \
CFG_MISCUTIL_SQL_USE_SQLALCHEMY, \
CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT
if CFG_MISCUTIL_SQL_USE_SQLALCHEMY:
try:
import sqlalchemy.pool as pool
import MySQLdb as mysqldb
mysqldb = pool.manage(mysqldb, use_threadlocal=True)
connect = mysqldb.connect
except ImportError:
CFG_MISCUTIL_SQL_USE_SQLALCHEMY = False
from MySQLdb import connect
else:
from MySQLdb import connect
## DB config variables. These variables are to be set in
## invenio-local.conf by admins and then replaced in situ in this file
## by calling "inveniocfg --update-dbexec".
## Note that they are defined here and not in config.py in order to
## prevent them from being exported accidentally elsewhere, as no-one
## should know DB credentials but this file.
## FIXME: this is more of a blast-from-the-past that should be fixed
## both here and in inveniocfg when the time permits.
import string
import time
import marshal
import re
import atexit
from zlib import compress, decompress
from thread import get_ident
from invenio.config import CFG_ACCESS_CONTROL_LEVEL_SITE, \
CFG_MISCUTIL_SQL_USE_SQLALCHEMY, \
CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT
if CFG_MISCUTIL_SQL_USE_SQLALCHEMY:
try:
import sqlalchemy.pool as pool
import MySQLdb as mysqldb
mysqldb = pool.manage(mysqldb, use_threadlocal=True)
connect = mysqldb.connect
except ImportError:
CFG_MISCUTIL_SQL_USE_SQLALCHEMY = False
from MySQLdb import connect
else:
from MySQLdb import connect
## DB config variables. These variables are to be set in
## invenio-local.conf by admins and then replaced in situ in this file
## by calling "inveniocfg --update-dbexec".
## Note that they are defined here and not in config.py in order to
## prevent them from being exported accidentally elsewhere, as no-one
## should know DB credentials but this file.
## FIXME: this is more of a blast-from-the-past that should be fixed
## both here and in inveniocfg when the time permits.
def test_bad_args(self):
manager = pool.manage(MockDBAPI())
manager.connect(None)
# -*- coding: utf-8 -*-
import datetime
from os.path import expanduser
from flask import g, request
import oursql
from sqlalchemy.pool import manage
oursql = manage(oursql)
__all__ = ["Query", "cache", "get_db", "get_notice", "httpsfix", "urlstrip"]
class Query(object):
def __init__(self, method="GET"):
self.query = {}
data = request.form if method == "POST" else request.args
for key in data:
self.query[key] = data.getlist(key)[-1]
def __getattr__(self, key):
return self.query.get(key)
def __setattr__(self, key, value):
if key == "query":
super(Query, self).__setattr__(key, value)
else:
self.query[key] = value
class _AppCache(object):
def connect(self, *args, **kwargs):
if 'conv' in kwargs:
conv = kwargs['conv']
if isinstance(conv, dict):
items = []
for k, v in conv.items():
if isinstance(v, list):
v = hashablelist(v)
items.append((k, v))
kwargs['conv'] = hashabledict(items)
return self.manager.connect(*args, **kwargs)
try:
from django.db.backends.postgresql_psycopg2 import base as pgsql_base
pgsql_base._Database = pgsql_base.Database
pgsql_base.Database = manage(pgsql_base._Database, **POOL_SETTINGS)
except ImproperlyConfigured:
pass
try:
from django.db.backends.mysql import base as mysql_base
mysql_base._Database = mysql_base.Database
db = ManagerProxy(manage(mysql_base._Database, **POOL_SETTINGS))
mysql_base.Database = db
except ImproperlyConfigured:
pass
try:
from django.db.backends.sqlite3 import base as sqlite3_base
import vote_utils
import sqlalchemy.pool as pool
def jsonify(*args, **kwargs):
return current_app.response_class(json.dumps(dict(*args, **kwargs),
indent=None),
mimetype='application/json')
if speedups.available:
print 'shapely speedups available!!!!'
speedups.enable()
# start using sqlalchemy cursor, models etc, please
psycopg2 = pool.manage(psycopg2)
def support_jsonp(f):
"""Wraps JSONified output for JSONP"""
@wraps(f)
def decorated_function(*args, **kwargs):
callback = request.args.get('callback', False)
if callback:
content = str(callback) + '(' + str(f(*args, **kwargs).data) + ')'
return current_app.response_class(
content, mimetype='application/javascript')
else:
return f(*args, **kwargs)
return decorated_function
class DatabaseWrapper(DatabaseWrapper):
def __init__(self, *args, **kwargs):
super(DatabaseWrapper, self).__init__(*args, **kwargs)
self.operators['icontains'] = 'LIKE UPPER(unaccent(%s))'
self.operators['istartswith'] = 'LIKE UPPER(unaccent(%s))'
self.ops = DatabaseOperations(self)
# comment everything below to disable the pool
"""
POOL_SETTINGS = 'DATABASE_POOL_ARGS'
# DATABASE_POOL_ARGS should be something like:
# {'max_overflow':10, 'pool_size':5, 'recycle':300}
pool_args = getattr(settings, POOL_SETTINGS, {})
db_pool = manage(Database, **pool_args)
log = logging.getLogger('z.pool')
def _log(message, *args):
log.debug(message)
# Only hook up the listeners if we are in debug mode.
if settings.DEBUG:
event.listen(QueuePool, 'checkout', partial(_log, 'retrieved from pool'))
event.listen(QueuePool, 'checkin', partial(_log, 'returned to pool'))
event.listen(QueuePool, 'connect', partial(_log, 'new connection'))
def test_bad_args(self):
manager = pool.manage(MockDBAPI())
manager.connect(None)
__license__ = "Creative Commons Attribution 3.0 Unported"
__version__ = "V0.07"
import sys
import MySQLdb
import logging
logger = logging.getLogger(__name__)
#
#
if True: # pylint: disable=using-constant-test
try:
import sqlalchemy.pool as pool
MySQLdb = pool.manage(MySQLdb, pool_size=12, max_overflow=12, timeout=30, echo=True, recycle=1800)
except: # noqa: E722 pylint: disable=bare-except
logger.exception("Creating MYSQL connection pool failing")
from wwpdb.utils.config.ConfigInfo import ConfigInfo # noqa: E402
class MyConnectionBase(object):
def __init__(self, siteId=None, verbose=False, log=sys.stderr): # pylint: disable=unused-argument
#
self.__siteId = siteId
self._cI = ConfigInfo(self.__siteId)
self._dbCon = None
self.__authD = {}
self.__databaseName = None
def get_options():
"""Return the parsed (named) options."""
global options
_parse_options()
return options
def get_largs():
"""Return the parsed (free) options."""
global largs
_parse_options()
return largs
# Wraps SQLAlchemy's DB Pool into our own connection pool.
db_pool = pool.manage(database)
def get_db_conn(host=None, port=None, user=None, password=None, database=None):
"""Returns a database connection from the connection pool."""
global db_pool
assert options
if host is None:
host = options.memsql_host
if port is None:
port = options.memsql_port
if user is None:
user = options.memsql_user
if password is None:
password = options.memsql_password
from shapely.ops import cascaded_union
from shapely.geometry import mapping, asShape
from shapely import speedups
import geo_utils
import vote_utils
import sqlalchemy.pool as pool
def jsonify(*args, **kwargs):
return current_app.response_class(json.dumps(dict(*args, **kwargs), indent=None), mimetype='application/json')
if speedups.available:
print 'shapely speedups available!!!!'
speedups.enable()
# start using sqlalchemy cursor, models etc, please
psycopg2 = pool.manage(psycopg2)
def support_jsonp(f):
"""Wraps JSONified output for JSONP"""
@wraps(f)
def decorated_function(*args, **kwargs):
callback = request.args.get('callback', False)
if callback:
content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')'
return current_app.response_class(content, mimetype='application/javascript')
else:
return f(*args, **kwargs)
return decorated_function
"""
import re
from django.core.exceptions import ImproperlyConfigured
from django.db import utils
from django.db.backends import utils as backend_utils
from django.db.backends.base.base import BaseDatabaseWrapper
from django.utils.functional import cached_property
from sqlalchemy import pool
try:
import MySQLdb as Database
except ImportError as err:
raise ImproperlyConfigured('Error loading MySQLdb module.\n'
'Did you install mysqlclient?') from err
Database = pool.manage(Database, pool_size=9, max_overflow=19, timeout=29)
from MySQLdb.constants import CLIENT, FIELD_TYPE # isort:skip
from MySQLdb.converters import conversions # isort:skip
# Some of these import MySQLdb, so import them after checking if it's installed.
from .client import DatabaseClient # isort:skip
from .creation import DatabaseCreation # isort:skip
from .features import DatabaseFeatures # isort:skip
from .introspection import DatabaseIntrospection # isort:skip
from .operations import DatabaseOperations # isort:skip
from .schema import DatabaseSchemaEditor # isort:skip
from .validation import DatabaseValidation # isort:skip
version = Database.version_info
if version < (1, 3, 3):
raise ImproperlyConfigured(
import copy
import logging
import platform
import MySQLdb
logger = logging.getLogger(__name__)
#
#
if platform.system() == "Linux":
try:
import sqlalchemy.pool as pool # pylint: disable=import-error
MySQLdb = pool.manage(MySQLdb,
pool_size=12,
max_overflow=12,
timeout=30,
echo=False,
use_threadlocal=False)
except Exception as e:
logger.exception("Creating MYSQL connection pool failing with %s",
str(e))
class ConnectionBase(object):
def __init__(self, siteId=None, verbose=False):
self.__verbose = verbose
#
self.__siteId = siteId
self.__db = None
self._dbCon = None
