def credis_bench():
credis_pool = ResourcePool(10, Connection, host=HOST, port=PORT)
credis_connection.execute("flushdb")
start = time.time()
for x in range(NUM_INSERTS):
with credis_pool.ctx() as conn:
conn.execute("hset", "words", "word|{}".format(x), "1")
print('CRedis HSET Done. Duration=', time.time() - start)
hset_dict['credis'] = time.time() - start
start = time.time()
for x in range(NUM_INSERTS):
with credis_pool.ctx() as conn:
conn.execute("hget", "words", "word|{}".format(x))
print('CRedis HGET Done. Duration=', time.time() - start)
hget_dict['credis'] = time.time() - start
def test_pool(self):
pool = ResourcePool(10, self.get_resource)
threads = [
gevent.spawn(self.random_worker(), pool) for _ in range(1000)
]
gevent.joinall(threads)
assert pool.alloc_count <= pool.max_count
assert pool.used_count == 0
assert self.counter == 10
def __init__(self, ip, port, db=0, pwd=None):
self.sip = ip
self.sport = port
self.sdb = db
self.spwd = pwd
if platform.system() == 'Linux':
self.redis_pool = ResourcePool(MAX_POOL_SIZE,
Connection,
host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
else:
self.redis_pool = redis.ConnectionPool(host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
self.ping()
def __init__(self, ip, port, db=0, pwd=None):
self.sip = ip
self.sport = port
self.sdb = db
self.spwd = pwd if pwd else None
if platform.system() == 'Linux':
self.redis_pool = ResourcePool(MAX_POOL_SIZE,
Connection,
host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
else:
self.redis_pool = redis.ConnectionPool(host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
self.ping()
def worker_using_with(pool):
try:
with pool.ctx() as res:
gevent.sleep(0.05)
raise TestException('bad worker1')
except TestException:
pass
def worker_using_with2(pool):
try:
with pool.ctx() as res:
gevent.sleep(0.05)
raise TestException('bad worker2')
except TestException:
pass
def random_worker():
return random.choice([
worker_normal, worker_exception, worker_using_with, worker_using_with2
])
pool = ResourcePool(10, get_resource)
threads = [gevent.spawn(random_worker(), pool) for i in range(1000)]
gevent.joinall(threads)
assert pool.alloc_count <= pool.max_count
assert pool.used_count == 0
assert counter == 10
class RedisClient(object):
"""
Redis 客户端管理器
"""
__metaclass__ = InstancePool
def __init__(self, ip, port, db=0, pwd=None):
self.sip = ip
self.sport = port
self.sdb = db
self.spwd = pwd
if platform.system() == 'Linux':
self.redis_pool = ResourcePool(MAX_POOL_SIZE,
Connection,
host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
else:
self.redis_pool = redis.ConnectionPool(host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
self.ping()
def ping(self):
conn = self.get_conn()
err_msg = 'redis server[%s:%s] not exist!!' % (self.sip, self.sport)
if platform.system() == 'Linux':
assert conn.execute('PING'), err_msg
else:
assert conn.ping(), err_msg
def get_conn(self):
if platform.system() == 'Linux':
conn = self.redis_pool
else:
conn = redis.Redis(connection_pool=self.redis_pool)
return conn
def set(self, *args, **kwargs):
"""
args = k,v,ex,nx,px,xx
"""
pipe = kwargs.get('pipe')
conn = self.get_conn() if pipe is None else pipe
if platform.system() == 'Linux':
set_cmd = make_redis_set_cmd('SET', *args)
res = conn.execute(*set_cmd)
else:
res = conn.set(*args)
return res
def get(self, *args, **kwargs):
"""
args = k
"""
pipe = kwargs.get('pipe')
conn = self.get_conn() if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('GET', *args)
else:
res = conn.get(*args)
return res
def expire(self, *args, **kwargs):
"""
args = k,et
"""
pipe = kwargs.get('pipe')
conn = self.get_conn() if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('EXPIRE', *args)
else:
res = conn.expire(*args)
return res
def delete(self, *args, **kwargs):
"""
args = *names
"""
pipe = kwargs.get('pipe')
conn = self.get_conn() if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('DEL', *args)
else:
res = conn.delete(*args)
return res
def keys(self, *args, **kwargs):
"""
args = *pattern
"""
pipe = kwargs.get('pipe')
conn = self.get_conn() if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('KEYS', *args)
else:
res = conn.keys(*args)
return res
def execute(self, cmd, pipe=None):
"""
执行redis命令
cmd = (oper,k,v,,,,)
"""
if cmd[0] in cmd_support['get']:
res = self.get(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['set']:
res = self.set(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['expire']:
res = self.expire(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['delete']:
res = self.delete(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['keys']:
res = self.keys(*cmd[1:], pipe=pipe)
return res
def pipe_execute(self, cmds):
"""
执行redis管道命令
cmds = [(oper,k,v),,,]
"""
if platform.system() == 'Linux':
for idx, cmd in enumerate(cmds):
if cmd[0] in cmd_support['set']:
cmds[idx] = make_redis_set_cmd(*cmd)
res = self.redis_pool.execute_pipeline(*cmds)
else:
conn = self.get_conn()
pipe = conn.pipeline()
for cmd in cmds:
self.execute(cmd, pipe)
res = pipe.execute()
return res
class RedisClient(object):
"""
Redis 客户端管理器
"""
__metaclass__ = InstancePool
def __init__(self, ip, port, db=0, pwd=None):
self.sip = ip
self.sport = port
self.sdb = db
self.spwd = pwd if pwd else None
if platform.system() == 'Linux':
self.redis_pool = ResourcePool(MAX_POOL_SIZE,
Connection,
host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
else:
self.redis_pool = redis.ConnectionPool(host=self.sip,
port=self.sport,
db=self.sdb,
password=self.spwd)
self.ping()
def ping(self):
conn = self.get_conn()
err_msg = 'redis server[%s:%s] not exist!!' % (self.sip, self.sport)
if platform.system() == 'Linux':
assert conn.execute('PING'), err_msg
else:
assert conn.ping(), err_msg
def get_conn(self):
if platform.system() == 'Linux':
conn = self.redis_pool
else:
conn = redis.Redis(connection_pool=self.redis_pool)
return conn
def set(self, *args, **kwargs):
"""
args = k,v,ex,nx,px,xx
"""
pipe = kwargs.get('pipe')
conn = self.get_conn()if pipe is None else pipe
if platform.system() == 'Linux':
set_cmd = make_redis_set_cmd('SET', *args)
res = conn.execute(*set_cmd)
else:
res = conn.set(*args)
return res
def get(self, *args, **kwargs):
"""
args = k
"""
pipe = kwargs.get('pipe')
conn = self.get_conn()if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('GET', *args)
else:
res = conn.get(*args)
return res
def expire(self, *args, **kwargs):
"""
args = k,et
"""
pipe = kwargs.get('pipe')
conn = self.get_conn()if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('EXPIRE', *args)
else:
res = conn.expire(*args)
return res
def delete(self, *args, **kwargs):
"""
args = *names
"""
pipe = kwargs.get('pipe')
conn = self.get_conn()if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('DEL', *args)
else:
res = conn.delete(*args)
return res
def keys(self, *args, **kwargs):
"""
args = *pattern
"""
pipe = kwargs.get('pipe')
conn = self.get_conn()if pipe is None else pipe
if platform.system() == 'Linux':
res = conn.execute('KEYS', *args)
else:
res = conn.keys(*args)
return res
def execute(self, cmd, pipe=None):
"""
执行redis命令
cmd = (oper,k,v,,,,)
"""
if cmd[0] in cmd_support['get']:
res = self.get(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['set']:
res = self.set(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['expire']:
res = self.expire(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['delete']:
res = self.delete(*cmd[1:], pipe=pipe)
elif cmd[0] in cmd_support['keys']:
res = self.keys(*cmd[1:], pipe=pipe)
return res
def pipe_execute(self, cmds):
"""
执行redis管道命令
cmds = [(oper,k,v),,,]
"""
if platform.system() == 'Linux':
for idx, cmd in enumerate(cmds):
if cmd[0] in cmd_support['set']:
cmds[idx] = make_redis_set_cmd(*cmd)
res = self.redis_pool.execute_pipeline(*cmds)
else:
conn = self.get_conn()
pipe = conn.pipeline()
for cmd in cmds:
self.execute(cmd, pipe)
res = pipe.execute()
return res
import time
import redis
from time import time
import asyncio
import asyncio_redis
from aredis import StrictRedis
import pandas as pd
retry_attempts = 1
hashmap_sizes = [50000]
HOST = '127.0.0.1'
PORT = 6379
# Credis connection
credis_pool = ResourcePool(10, Connection, host=HOST, port=PORT)
#Aredis client connection
aredis_client = StrictRedis(host=HOST, port=PORT, db=0)
#AsyncIO redis connection
asyncio_redis_connection = asyncio_redis.Pool.create(host=HOST,
port=PORT,
poolsize=10)
# Golang extension connection
cpipe.Connect(HOST, PORT)
# Golang extension connection with golang lib
cpipelib.Connect(HOST, PORT)
def create_pool(url):
info = parse_redis_url(url)
poolmax = int(info.pop('poolmax', 32))
return ResourcePool(poolmax, Connection, **info)