class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0' def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval( """ redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0' def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval(""" redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') self.client.delete('test7') self.client.delete('test8') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')]) def test_eval(self): self.client.eval( """ return redis.call('set', KEYS[1], ARGV[1]) """, 1, 'test7', '7') eq_(self.client.get('test7'), b('7')) def test_evalsha(self): sha = self.client.script_load(""" return redis.call('set', KEYS[1], ARGV[1]) """) eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK')) eq_(self.client.get('test8'), b('8'))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') self.client.delete('test7') self.client.delete('test8') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')]) def test_eval(self): self.client.eval(""" return redis.call('set', KEYS[1], ARGV[1]) """, 1, 'test7', '7') eq_(self.client.get('test7'), b('7')) def test_evalsha(self): sha = self.client.script_load(""" return redis.call('set', KEYS[1], ARGV[1]) """) eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK')) eq_(self.client.get('test8'), b('8'))
class Memcache(object): def __init__(self): self.shard = RedisShardAPI(CACHE) @staticmethod def cache_key(func, *args): return u":".join([func.__name__] + map(unicode, args)) def cache(self): def wraped(func): @functools.wraps(func) def fcall(obj, *args): key = self.cache_key(func, *args) rst = self.shard.get(key) logging.warning(key) if not rst: data = func(obj, *args) self.shard.set(key, json.dumps(data)) self.shard.expire(key, 3600) return data return rst return fcall return wraped def invalidate(self, func, *args): key = self.cache_key(func, *args) self.shard.delete(key)
def decorator(*args, **kwargs): servers = current_app.config.get("REDIS_SHARD_URL") client = RedisShardAPI(servers, hash_method='md5') now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S") ip = request.remote_addr key = "%s{%s}" % (ip, now) client.incr(key) if client.get(key): num = int(client.get(key)) else: num = 0 client.expire(key, 10) if num > times: return jsonify(dict( code=40003, msg='warning: Refused to too many requests !')) return f(*args, **kwargs)
def decorator(*args, **kwargs): servers = current_app.config.get("REDIS_SHARD_URL") client = RedisShardAPI(servers, hash_method='md5') now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S") ip = request.remote_addr key = "%s{%s}" % (ip, now) client.incr(key) if client.get(key): num = int(client.get(key)) else: num = 0 client.expire(key, 10) if num > times: return jsonify( dict(code=40003, msg='warning: Refused to too many requests !')) return f(*args, **kwargs)
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0'
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0'
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')])
}, #{'name':'server5','host':'127.0.0.1','port':14000,'db':0}, #{'name':'server6','host':'127.0.0.1','port':15000,'db':0}, ] ########NEW FILE######## __FILENAME__ = myapp #!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers import time client = RedisShardAPI(servers) client.set('test', 1) print client.get('test') client.zadd('testset', 'first', 1) client.zadd('testset', 'second', 2) print client.zrange('testset', 0, -1) print client.zrank('testset', 'second') print client.zrank('testset2', 'second') client.set('foo', 2) client.set('a{foo}', 5) client.set('b{foo}', 5) client.set('c{foo}', 5) client.set('{foo}d', 6) client.set('{foo}e', 7) client.set('e{foo}f', 8) print client.get_server_name('foo') print client.get_server_name('c{foo}')
class SessionStore(SessionBase): """ Implements Redis database session store. """ def __init__(self, session_key=None): super(SessionStore, self).__init__(session_key) try: unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None) except AttributeError: unix_socket_path = None if unix_socket_path is None: self.server = RedisShardAPI(settings.SESSION_REDIS_HOSTS) else: self.server = redis.StrictRedis( unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', '/var/run/redis/redis.sock'), db=getattr(settings, 'SESSION_REDIS_DB', 0), password=getattr(settings, 'SESSION_REDIS_PASSWORD', None), ) def load(self): try: session_data = self.server.get(self.get_real_stored_key(self._get_or_create_session_key())) return self.decode(force_unicode(session_data)) except: self.create() return {} def exists(self, session_key): return self.server.exists(self.get_real_stored_key(session_key)) def create(self): while True: self._session_key = self._get_new_session_key() try: self.save(must_create=True) except CreateError: continue self.modified = True return def save(self, must_create=False): if not getattr(settings, 'SITE_READ_ONLY', False): if must_create and self.exists(self._get_or_create_session_key()): raise CreateError this_session = self._get_session(no_load=must_create) data = self.encode(self._get_session(no_load=must_create)) if redis.VERSION[0] >= 2: self.server.setex(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age(), data) else: self.server.set(self.get_real_stored_key(self._get_or_create_session_key()), data) self.server.expire(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age()) def delete(self, session_key=None): if session_key is None: if self.session_key is None: return session_key = self.session_key try: self.server.delete(self.get_real_stored_key(session_key)) except: pass def get_real_stored_key(self, session_key): """Return the real key name in redis storage @return string """ prefix = getattr(settings, 'SESSION_REDIS_PREFIX', '') if not prefix: return session_key return ':'.join([prefix, session_key])
from redis_shard.shard import RedisShardAPI import re import uuid server_list = [ {'name':'server1', 'host':'127.0.0.1', 'port':6379}, {'name':'server2', 'host':'127.0.0.1', 'port':6380}, {'name':'server3', 'host':'127.0.0.1', 'port':6381}, {'name':'server4', 'host':'127.0.0.1', 'port':6382}, {'name':'server5', 'host':'127.0.0.1', 'port':6383}, ] jedis = RedisShardAPI(server_list) print("test_key_1"); foo = jedis.get("{test_key}_1"); print("test_key_2"); foo = jedis.get("{test_key}_2"); print("test_key_3"); foo = jedis.get("{test_key}_3"); print("test_key_4"); foo = jedis.get("{test_key}_4"); print("test_key_5"); foo = jedis.get("test_key_5"); print("test_key_6"); foo = jedis.get("test_key_6"); print("test_key_7"); foo = jedis.get("test_key_7"); print("test_key_8"); foo = jedis.get("test_key_8"); print("test_key_9"); foo = jedis.get("test_key_9");
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() pipe.reset() eq_(self.client.get('test'), b'2') eq_(self.client.zscore('testzset', 'first'), 2.0) eq_(self.client.zscore('testzset', 'second'), 2.0) with self.client.pipeline() as pipe: pipe.set('test', '3') pipe.zadd('testzset', 'first', 4) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 5) pipe.execute() eq_(self.client.get('test'), b'3') eq_(self.client.zscore('testzset', 'first'), 5.0) eq_(self.client.zscore('testzset', 'second'), 5.0) with self.client.pipeline() as pipe: pipe.watch('test') eq_(self.client.get('test'), b'3') pipe.multi() pipe.incr('test') eq_(pipe.execute(), [4]) eq_(self.client.get('test'), b'4') with self.client.pipeline() as pipe: pipe.watch('test') pipe.multi() pipe.incr('test') self.client.decr('test') self.assertRaises(WatchError, pipe.execute) eq_(self.client.get('test'), b'3') keys_of_names = {} with self.client.pipeline() as pipe: for key in xrange(100): key = str(key) name = pipe.shard_api.get_server_name(key) if name not in keys_of_names: keys_of_names[name] = key else: key1 = key key2 = keys_of_names[name] pipe.watch(key1, key2) pipe.multi() pipe.set(key1, 1) pipe.set(key2, 2) pipe.execute() eq_(self.client.get(key1), b'1') eq_(self.client.get(key2), b'2') break def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval(""" redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
#!/usr/bin/env python import time from config import servers from redis_shard.shard import RedisShardAPI client = RedisShardAPI(servers) client.set("test", 1) print client.get("test") client.zadd("testset", "first", 1) client.zadd("testset", "second", 2) print client.zrange("testset", 0, -1) print client.zrank("testset", "second") print client.zrank("testset2", "second") client.set("foo", 2) client.set("a{foo}", 5) client.set("b{foo}", 5) client.set("c{foo}", 5) client.set("{foo}d", 6) client.set("{foo}e", 7) client.set("e{foo}f", 8) print client.get_server_name("foo") print client.get_server_name("c{foo}") print client.get_server_name("{foo}d") print client.get_server_name("{foo}e") print client.get_server_name("e{foo}f") t0 = time.time() print client.tag_keys("*{foo}*") t1 = time.time()
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', {'first': 1}) pipe.zincrby('testzset', 1, 'first') pipe.zadd('testzset', {'second': 2}) pipe.execute() pipe.reset() eq_(self.client.get('test'), b'2') eq_(self.client.zscore('testzset', 'first'), 2.0) eq_(self.client.zscore('testzset', 'second'), 2.0) with self.client.pipeline() as pipe: pipe.set('test', '3') pipe.zadd('testzset', {'first': 4}) pipe.zincrby('testzset', 1, 'first') pipe.zadd('testzset', {'second': 5}) pipe.execute() eq_(self.client.get('test'), b'3') eq_(self.client.zscore('testzset', 'first'), 5.0) eq_(self.client.zscore('testzset', 'second'), 5.0) with self.client.pipeline() as pipe: pipe.watch('test') eq_(self.client.get('test'), b'3') pipe.multi() pipe.incr('test') eq_(pipe.execute(), [4]) eq_(self.client.get('test'), b'4') with self.client.pipeline() as pipe: pipe.watch('test') pipe.multi() pipe.incr('test') self.client.decr('test') self.assertRaises(WatchError, pipe.execute) eq_(self.client.get('test'), b'3') keys_of_names = {} with self.client.pipeline() as pipe: for key in xrange(100): key = str(key) name = pipe.shard_api.get_server_name(key) if name not in keys_of_names: keys_of_names[name] = key else: key1 = key key2 = keys_of_names[name] pipe.watch(key1, key2) pipe.multi() pipe.set(key1, 1) pipe.set(key2, 2) pipe.execute() eq_(self.client.get(key1), b'1') eq_(self.client.get(key2), b'2') break def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval( """ redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
#!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers import time client = RedisShardAPI(servers) client.set('test',1) print client.get('test') client.zadd('testset','first',1) client.zadd('testset','second',2) print client.zrange('testset',0,-1) print client.zrank('testset','second') print client.zrank('testset2','second') client.set('foo',2) client.set('a{foo}',5) client.set('b{foo}',5) client.set('c{foo}',5) client.set('{foo}d',6) client.set('{foo}e',7) client.set('e{foo}f',8) print client.get_server_name('foo') print client.get_server_name('c{foo}') print client.get_server_name('{foo}d') print client.get_server_name('{foo}e') print client.get_server_name('e{foo}f') t0 = time.time() print client.tag_keys('*{foo}*') t1 = time.time() print t1-t0
#!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers import time client = RedisShardAPI(servers) client.set('test',1) print(client.get('test')) client.zadd('testset','first',1) client.zadd('testset','second',2) print(client.zrange('testset',0,-1)) print(client.zrank('testset','second')) print(client.zrank('testset2','second')) client.set('foo',2) client.set('a{foo}',5) client.set('b{foo}',5) client.set('c{foo}',5) client.set('{foo}d',6) client.set('{foo}e',7) client.set('e{foo}f',8) print(client.get_server_name('foo')) print(client.get_server_name('c{foo}')) print(client.get_server_name('{foo}d')) print(client.get_server_name('{foo}e')) print(client.get_server_name('e{foo}f')) t0 = time.time() print(client.tag_keys('*{foo}*')) t1 = time.time() print(t1-t0)