def __init__(self, pool_options=None, **kwargs):
self._configurations = kwargs
pool_options = pool_options or {}
self._pool = ConnectionPool(self._connect, **{
**self._defaults,
**pool_options
})
def test_pool(self):
cursors = []
ts = []
threads_count = 100
for i in range(threads_count):
t = threading.Thread(target = create_cursor, args = (cursors, ts, self))
ts.append(t)
t.setDaemon(True)
t.start()
ts.append(t)
for t in ts:
t.join()
self.assertEqual(len(cursors), threads_count)
self.assertEqual(len(ConnectionPool._connections), ConnectionPool._hold_count)
self.assertEqual(ConnectionPool._count, 0)
ConnectionPool.close()
def create_pool(resource_name, max_size, max_usage, idle, ttl):
def create_resource_client():
resource = ResourceEnum[resource_name].value
return resource
return ConnectionPool(create=create_resource_client,
max_size=max_size,
max_usage=max_usage,
idle=idle,
ttl=ttl)
def create_cursor(cursors, ts, tester):
c = ConnectionPool.cursor()
cursors.append(c)
t = threading.Thread(target = close_cursor, args = (c, ))
t.setDaemon(True)
t.start()
ts.append(t)
# print ConnectionPool._count, ConnectionPool.MAX_COUNT
tester.assertTrue(ConnectionPool._count <= ConnectionPool.MAX_COUNT + 1)
tester.assertTrue(len(ConnectionPool._connections) <= ConnectionPool.MAX_COUNT)
def create_pool(username, pw, host, database, max_size, max_usage, idle,
ttl):
def create_storage_client():
engine = create_engine("mysql+pymysql://{}:{}@{}/{}".format(
username, pw, host, database))
if not database_exists(engine.url):
create_database(engine.url)
Base.metadata.create_all(engine)
return engine
return ConnectionPool(create=create_storage_client,
max_size=max_size,
max_usage=max_usage,
idle=idle,
ttl=ttl)
class Autocomplete:
# create connection pool object
connection_pool = ConnectionPool()
def get_similar_titles(self, query):
print("similar titles called")
db_connection = self.connection_pool.get_pool_connection()
cursor = db_connection.cursor(buffered=True)
sql = """ SELECT title FROM full_movies_data WHERE title LIKE %s"""
search_query = f"%{query}%"
cursor.execute(sql, (search_query, ))
result = cursor.fetchall()
titles_list = []
for title in result:
titles_list.append(title[0])
# return result list to request
return titles_list
class ConnectionPooled(object):
'''数据库连接池类
创建连接池：
pooled = ConnectionPooled(host='192.0.0.1', database='foo',
pool_options=dict(max_size=5))
获取一个未使用连接池的连接，并执行 SQL：
pooled.execute(sql)
pooled.connection.execute(sql)
使用连接池执行 SQL：
with pooled.pool() as connection:
connection.execute(sql)
'''
# 这个连接，是不进入连接池的
_connection = None
_defaults = dict(max_size=POOL_SIZE,
max_usage=POOL_USAGES,
ttl=POOL_TTL,
idle=POOL_IDLE)
def __init__(self, pool_options=None, **kwargs):
self._configurations = kwargs
pool_options = pool_options or {}
self._pool = ConnectionPool(self._connect, **{
**self._defaults,
**pool_options
})
def _connect(self):
return Connection(**self._configurations)
@property
def connection(self):
if not self._connection:
self._connection = self._connect()
return self._connection
def pool(self):
return self._pool.item()
def __getattr__(self, method):
return getattr(self.connection, method)
def create_pool(max_size, max_usage, idle, ttl):
def create_k8s_client():
try:
# Load configuration inside the Pod
config.load_incluster_config()
except ConfigException:
# Load configuration for testing
config.load_kube_config()
# Create the Apis
v1_core = client.CoreV1Api()
return v1_core
return ConnectionPool(create=create_k8s_client,
max_size=max_size,
max_usage=max_usage,
idle=idle,
ttl=ttl)
def setConnectionPool(self, create_connection, max_size, max_usage, idle, ttl):
try:
if not self.connection_pool.exists(self.connector_type_key):
print(
f'🟡 _Connector.setConnectionPool - New connection pool [{self.connector_type_key}]')
self.connection_pool.update(
connector_type=self.connector_type_key,
connection_pool=ConnectionPool(
create=create_connection,
max_size=max_size,
max_usage=max_usage,
idle=idle,
ttl=ttl
)
)
except:
print('_Connector.setConnectionPool - Failed to set connection pool')
traceback.print_exc()
def connection_pool(self):
"""set up a pool of re-usable active connections"""
# merge this object's values with those of its parent provider
args_to_use, kwargs_to_use = self.get_args_to_use()
# hashing connection pool on tuple of relevant arguments. There
# may be a better way to do this
conn_pool_label_tuple = (
type(self),
*args_to_use,
*[kwargs_to_use.get(k, None) for k in self.conn_keywords],
)
if conn_pool_label_tuple not in self.connection_pools:
create_callback = partial(
self.create_connection, *args_to_use, **kwargs_to_use
)
self.connection_pools[conn_pool_label_tuple] = ConnectionPool(
create_callback, close=self.close_connection, max_size=self.pool_size
)
return self.connection_pools[conn_pool_label_tuple]
from flask import request
from flask_cors import CORS
from flask_caching import Cache
from connection_pool import ConnectionPool
from autocomplete import Autocomplete
import engine
app = Flask(__name__)
CORS(app)
# tell Flask to use the above defined config
app.config['CACHE_TYPE'] = 'simple'
app.cache = Cache(app)
# creat connection pool object
connection_pool = ConnectionPool()
title_autocomplete = Autocomplete()
@app.cache.memoize(timeout=100)
def get_similar_titles(query):
return title_autocomplete.get_similar_titles(query)
def get_full_recommendations_data(recs):
result = []
# buffered=True allows to read the result as buffered
db_connection = connection_pool.get_pool_connection()
cursor = db_connection.cursor(buffered=True)
for movie in recs:
sql = """ SELECT * FROM full_movies_data WHERE title = %s """
class TransactionManager(object):
"""
handles all transaction and db query
"""
def __init__(self):
self.conn = ConnectionPool().pool.connection()
self.cursor = self.conn.cursor()
def startTransaction(self):
"""
For databases that support transactions,
the Python interface silently starts a transaction when the cursor is created.
so we do nothing here.
"""
pass
def commitTransaction(self):
self.cursor.close()
self.conn.commit()
def endTransaction(self):
"""
结束事务
"""
pass
def rollbackTransaction(self):
"""
回滚事务
"""
self.cursor.close()
self.conn.rollback()
def queryInsert(self, sqlid, inputObject):
"""
查询插入
"""
#=======================================================================
# resultclasstype参数在没有返回值的时候用不到
#=======================================================================
sql, resultclasstype = processSql(sqlid, inputObject)
try:
var = self.cursor.execute(sql)
return var
except IntegrityError:
return -1
def queryUpdate(self, sqlid, inputObject):
"""
查询更新
"""
self.queryInsert(sqlid, inputObject)
def queryDelete(self, sqlid, inputObject):
"""
查询删除
"""
self.queryInsert(sqlid, inputObject)
def queryForObject(self, sqlid, inputObject):
"""
查询并返回一个对象
"""
sql, resultclasstype = processSql(sqlid, inputObject)
self.cursor.execute(sql)
objList = Data2Object.data2object(self.cursor, resultclasstype)
if len(objList) == 0:
return None
elif len(objList) == 1:
return objList[0]
else:
raise Exception('query for one object, but get many.');
def queryForList(self, sqlid, inputObject):
"""
查询并返回一个列表
"""
sql , resultclasstype = self.processSql(sqlid, inputObject)
self.cursor.execute(sql)
objList = self.data2object(self.cursor, resultclasstype)
return objList
def processSql(self, sqlid, inputObject):
# TODO (zj)
return '', ''
def data2object(cursor, resultclasstype):
# TODO (zj)
return []
def execute_sql(self, sql, pre_sql=None):
r = []
try:
if pre_sql:
self.cursor.execute(pre_sql)
self.cursor.execute(sql)
r = self.cursor.fetchall()
self.conn.commit()
except Exception, e:
print "Exception when execute_sql:%s, rollback" % e
self.conn.rollback()
finally:
def __init__(self):
self.conn = ConnectionPool().pool.connection()
self.cursor = self.conn.cursor()
from connection_pool import ConnectionPool
class Connection(object):
def __init__(self, **kwargs):
self.args = kwargs
self.state = 'Connected'
def close(self):
self.state = 'Closed'
def connect():
return Connection()
def close(connection):
connection.close()
pool = ConnectionPool(create=connect, close=close,
max_size=10, max_usage=10000, idle=60, ttl=120)
with pool.item() as connection:
assert connection.state == 'Connected'