示例#1
0
def credis_bench():
    credis_pool = ResourcePool(10, Connection, host=HOST, port=PORT)
    credis_connection.execute("flushdb")
    start = time.time()
    for x in range(NUM_INSERTS):
        with credis_pool.ctx() as conn:
            conn.execute("hset", "words", "word|{}".format(x), "1")
    print('CRedis HSET Done. Duration=', time.time() - start)
    hset_dict['credis'] = time.time() - start

    start = time.time()
    for x in range(NUM_INSERTS):
        with credis_pool.ctx() as conn:
            conn.execute("hget", "words", "word|{}".format(x))
    print('CRedis HGET Done. Duration=', time.time() - start)
    hget_dict['credis'] = time.time() - start
示例#2
0
 def test_pool(self):
     pool = ResourcePool(10, self.get_resource)
     threads = [
         gevent.spawn(self.random_worker(), pool) for _ in range(1000)
     ]
     gevent.joinall(threads)
     assert pool.alloc_count <= pool.max_count
     assert pool.used_count == 0
     assert self.counter == 10
示例#3
0
    def __init__(self, ip, port, db=0, pwd=None):
        self.sip = ip
        self.sport = port
        self.sdb = db
        self.spwd = pwd

        if platform.system() == 'Linux':
            self.redis_pool = ResourcePool(MAX_POOL_SIZE,
                                           Connection,
                                           host=self.sip,
                                           port=self.sport,
                                           db=self.sdb,
                                           password=self.spwd)
        else:
            self.redis_pool = redis.ConnectionPool(host=self.sip,
                                                   port=self.sport,
                                                   db=self.sdb,
                                                   password=self.spwd)
        self.ping()
示例#4
0
 def __init__(self, ip, port, db=0, pwd=None):
     self.sip = ip
     self.sport = port
     self.sdb = db
     self.spwd = pwd if pwd else None
     
     if platform.system() == 'Linux':
         self.redis_pool = ResourcePool(MAX_POOL_SIZE,
                                        Connection,
                                        host=self.sip,
                                        port=self.sport,
                                        db=self.sdb,
                                        password=self.spwd)
     else:
         self.redis_pool = redis.ConnectionPool(host=self.sip,
                                                port=self.sport,
                                                db=self.sdb,
                                                password=self.spwd)
     self.ping()
示例#5
0
def worker_using_with(pool):
    try:
        with pool.ctx() as res:
            gevent.sleep(0.05)
            raise TestException('bad worker1')
    except TestException:
        pass


def worker_using_with2(pool):
    try:
        with pool.ctx() as res:
            gevent.sleep(0.05)
            raise TestException('bad worker2')
    except TestException:
        pass


def random_worker():
    return random.choice([
        worker_normal, worker_exception, worker_using_with, worker_using_with2
    ])


pool = ResourcePool(10, get_resource)
threads = [gevent.spawn(random_worker(), pool) for i in range(1000)]
gevent.joinall(threads)
assert pool.alloc_count <= pool.max_count
assert pool.used_count == 0
assert counter == 10
示例#6
0
class RedisClient(object):
    """
    Redis 客户端管理器
    """
    __metaclass__ = InstancePool

    def __init__(self, ip, port, db=0, pwd=None):
        self.sip = ip
        self.sport = port
        self.sdb = db
        self.spwd = pwd

        if platform.system() == 'Linux':
            self.redis_pool = ResourcePool(MAX_POOL_SIZE,
                                           Connection,
                                           host=self.sip,
                                           port=self.sport,
                                           db=self.sdb,
                                           password=self.spwd)
        else:
            self.redis_pool = redis.ConnectionPool(host=self.sip,
                                                   port=self.sport,
                                                   db=self.sdb,
                                                   password=self.spwd)
        self.ping()

    def ping(self):
        conn = self.get_conn()
        err_msg = 'redis server[%s:%s] not exist!!' % (self.sip, self.sport)
        if platform.system() == 'Linux':
            assert conn.execute('PING'), err_msg
        else:
            assert conn.ping(), err_msg

    def get_conn(self):
        if platform.system() == 'Linux':
            conn = self.redis_pool
        else:
            conn = redis.Redis(connection_pool=self.redis_pool)
        return conn

    def set(self, *args, **kwargs):
        """
        args = k,v,ex,nx,px,xx
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn() if pipe is None else pipe

        if platform.system() == 'Linux':
            set_cmd = make_redis_set_cmd('SET', *args)
            res = conn.execute(*set_cmd)
        else:
            res = conn.set(*args)
        return res

    def get(self, *args, **kwargs):
        """
        args = k
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn() if pipe is None else pipe

        if platform.system() == 'Linux':
            res = conn.execute('GET', *args)
        else:
            res = conn.get(*args)
        return res

    def expire(self, *args, **kwargs):
        """
        args = k,et
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn() if pipe is None else pipe

        if platform.system() == 'Linux':
            res = conn.execute('EXPIRE', *args)
        else:
            res = conn.expire(*args)
        return res

    def delete(self, *args, **kwargs):
        """
        args = *names
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn() if pipe is None else pipe

        if platform.system() == 'Linux':
            res = conn.execute('DEL', *args)
        else:
            res = conn.delete(*args)
        return res

    def keys(self, *args, **kwargs):
        """
        args = *pattern
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn() if pipe is None else pipe

        if platform.system() == 'Linux':
            res = conn.execute('KEYS', *args)
        else:
            res = conn.keys(*args)
        return res

    def execute(self, cmd, pipe=None):
        """
        执行redis命令
        cmd = (oper,k,v,,,,)
        """
        if cmd[0] in cmd_support['get']:
            res = self.get(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['set']:
            res = self.set(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['expire']:
            res = self.expire(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['delete']:
            res = self.delete(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['keys']:
            res = self.keys(*cmd[1:], pipe=pipe)
        return res

    def pipe_execute(self, cmds):
        """
        执行redis管道命令
        cmds = [(oper,k,v),,,]
        """
        if platform.system() == 'Linux':
            for idx, cmd in enumerate(cmds):
                if cmd[0] in cmd_support['set']:
                    cmds[idx] = make_redis_set_cmd(*cmd)
            res = self.redis_pool.execute_pipeline(*cmds)
        else:
            conn = self.get_conn()
            pipe = conn.pipeline()
            for cmd in cmds:
                self.execute(cmd, pipe)
            res = pipe.execute()
        return res
示例#7
0
class RedisClient(object):
    """
    Redis 客户端管理器
    """
    __metaclass__ = InstancePool

    def __init__(self, ip, port, db=0, pwd=None):
        self.sip = ip
        self.sport = port
        self.sdb = db
        self.spwd = pwd if pwd else None
        
        if platform.system() == 'Linux':
            self.redis_pool = ResourcePool(MAX_POOL_SIZE,
                                           Connection,
                                           host=self.sip,
                                           port=self.sport,
                                           db=self.sdb,
                                           password=self.spwd)
        else:
            self.redis_pool = redis.ConnectionPool(host=self.sip,
                                                   port=self.sport,
                                                   db=self.sdb,
                                                   password=self.spwd)
        self.ping()
    
    def ping(self):
        conn = self.get_conn()
        err_msg = 'redis server[%s:%s] not exist!!' % (self.sip, self.sport)
        if platform.system() == 'Linux':
            assert conn.execute('PING'), err_msg
        else:
            assert conn.ping(), err_msg
    
    def get_conn(self):
        if platform.system() == 'Linux':
            conn = self.redis_pool
        else:
            conn = redis.Redis(connection_pool=self.redis_pool)
        return conn
    
    def set(self, *args, **kwargs):
        """
        args = k,v,ex,nx,px,xx
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn()if pipe is None else pipe
            
        if platform.system() == 'Linux':
            set_cmd = make_redis_set_cmd('SET', *args)
            res = conn.execute(*set_cmd)
        else:
            res = conn.set(*args)
        return res
    
    def get(self, *args, **kwargs):
        """
        args = k
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn()if pipe is None else pipe
                    
        if platform.system() == 'Linux':
            res = conn.execute('GET', *args)
        else:
            res = conn.get(*args)
        return res
    
    def expire(self, *args, **kwargs):
        """
        args = k,et
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn()if pipe is None else pipe
        
        if platform.system() == 'Linux':
            res = conn.execute('EXPIRE', *args)
        else:
            res = conn.expire(*args)
        return res
    
    def delete(self, *args, **kwargs):
        """
        args = *names
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn()if pipe is None else pipe
        
        if platform.system() == 'Linux':
            res = conn.execute('DEL', *args)
        else:
            res = conn.delete(*args)
        return res
    
    def keys(self, *args, **kwargs):
        """
        args = *pattern
        """
        pipe = kwargs.get('pipe')
        conn = self.get_conn()if pipe is None else pipe
        
        if platform.system() == 'Linux':
            res = conn.execute('KEYS', *args)
        else:
            res = conn.keys(*args)
        return res
    
    def execute(self, cmd, pipe=None):
        """
        执行redis命令
        cmd = (oper,k,v,,,,)
        """
        if cmd[0] in cmd_support['get']:
            res = self.get(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['set']:
            res = self.set(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['expire']:
            res = self.expire(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['delete']:
            res = self.delete(*cmd[1:], pipe=pipe)
        elif cmd[0] in cmd_support['keys']:
            res = self.keys(*cmd[1:], pipe=pipe)
        return res
    
    def pipe_execute(self, cmds):
        """
        执行redis管道命令
        cmds = [(oper,k,v),,,]
        """
        if platform.system() == 'Linux':
            for idx, cmd in enumerate(cmds):
                if cmd[0] in cmd_support['set']:
                    cmds[idx] = make_redis_set_cmd(*cmd)
            res = self.redis_pool.execute_pipeline(*cmds)
        else:
            conn = self.get_conn()
            pipe = conn.pipeline()
            for cmd in cmds:
                self.execute(cmd, pipe)
            res = pipe.execute()
        return res
示例#8
0
import time
import redis
from time import time
import asyncio
import asyncio_redis
from aredis import StrictRedis
import pandas as pd

retry_attempts = 1
hashmap_sizes = [50000]

HOST = '127.0.0.1'
PORT = 6379

# Credis connection
credis_pool = ResourcePool(10, Connection, host=HOST, port=PORT)

#Aredis client connection
aredis_client = StrictRedis(host=HOST, port=PORT, db=0)

#AsyncIO redis connection
asyncio_redis_connection = asyncio_redis.Pool.create(host=HOST,
                                                     port=PORT,
                                                     poolsize=10)

# Golang extension connection
cpipe.Connect(HOST, PORT)

# Golang extension connection with golang lib
cpipelib.Connect(HOST, PORT)
示例#9
0
def create_pool(url):
    info = parse_redis_url(url)
    poolmax = int(info.pop('poolmax', 32))
    return ResourcePool(poolmax, Connection, **info)