def __init__(self,seed_url): self.visited_url = set() self.un_visited_url = [seed_url] self.seed_url = seed_url self.client = RedisShardAPI(REDIS_SEARCH_SERVERS) self.link_index_dict = {} self.title_index_dict = {}
def setup(self, app): for other in app.plugins: if not isinstance(other, RedisShardPlugin): continue if other.keyword == self.keyword: raise PluginError("Found another redis shard plugin with "\ "conflicting settings (non-unique keyword).") if self.redisdb is None: self.redisdb = RedisShardAPI(self.config)
class Memcache(object): def __init__(self): self.shard = RedisShardAPI(CACHE) @staticmethod def cache_key(func, *args): return u":".join([func.__name__] + map(unicode, args)) def cache(self): def wraped(func): @functools.wraps(func) def fcall(obj, *args): key = self.cache_key(func, *args) rst = self.shard.get(key) logging.warning(key) if not rst: data = func(obj, *args) self.shard.set(key, json.dumps(data)) self.shard.expire(key, 3600) return data return rst return fcall return wraped def invalidate(self, func, *args): key = self.cache_key(func, *args) self.shard.delete(key)
def reconnect_redis(self): if self.shared_client and Storage.storage: return Storage.storage storage = RedisShardAPI(self.context.config.REDIS_SERVERS, hash_method='md5') if self.shared_client: Storage.storage = storage return storage
def reconnect_redis(self): '''Reconnect to redis. :return: Redis client instance :rettype: redis.Redis ''' if self.shared_client and Storage.storage: return Storage.storage storage = RedisShardAPI(self.context.config.REDIS_SERVERS, hash_method='md5') if self.shared_client: Storage.storage = storage return storage
def decorator(*args, **kwargs): servers = current_app.config.get("REDIS_SHARD_URL") client = RedisShardAPI(servers, hash_method='md5') now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S") ip = request.remote_addr key = "%s{%s}" % (ip, now) client.incr(key) if client.get(key): num = int(client.get(key)) else: num = 0 client.expire(key, 10) if num > times: return jsonify( dict(code=40003, msg='warning: Refused to too many requests !')) return f(*args, **kwargs)
def __init__(self, session_key=None): super(SessionStore, self).__init__(session_key) try: unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None) except AttributeError: unix_socket_path = None if unix_socket_path is None: self.server = RedisShardAPI(settings.SESSION_REDIS_HOSTS) else: self.server = redis.StrictRedis( unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', '/var/run/redis/redis.sock'), db=getattr(settings, 'SESSION_REDIS_DB', 0), password=getattr(settings, 'SESSION_REDIS_PASSWORD', None), )
def decorator(*args, **kwargs): servers = current_app.config.get("REDIS_SHARD_URL") client = RedisShardAPI(servers, hash_method='md5') now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S") ip = request.remote_addr key = "%s{%s}" % (ip, now) client.incr(key) if client.get(key): num = int(client.get(key)) else: num = 0 client.expire(key, 10) if num > times: return jsonify(dict( code=40003, msg='warning: Refused to too many requests !')) return f(*args, **kwargs)
'port': 6379, 'db': 0 }, #{'name':'server5','host':'127.0.0.1','port':14000,'db':0}, #{'name':'server6','host':'127.0.0.1','port':15000,'db':0}, ] ########NEW FILE######## __FILENAME__ = myapp #!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers import time client = RedisShardAPI(servers) client.set('test', 1) print client.get('test') client.zadd('testset', 'first', 1) client.zadd('testset', 'second', 2) print client.zrange('testset', 0, -1) print client.zrank('testset', 'second') print client.zrank('testset2', 'second') client.set('foo', 2) client.set('a{foo}', 5) client.set('b{foo}', 5) client.set('c{foo}', 5) client.set('{foo}d', 6) client.set('{foo}e', 7) client.set('e{foo}f', 8)
def init_app(self, app): servers = app.config.get("REDIS_SHARD_URL") client = RedisShardAPI(servers, hash_method='md5') self._cache = client return client
#!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers import time client = RedisShardAPI(servers) client.set('test',1) print client.get('test') client.zadd('testset','first',1) client.zadd('testset','second',2) print client.zrange('testset',0,-1) print client.zrank('testset','second') print client.zrank('testset2','second') client.set('foo',2) client.set('a{foo}',5) client.set('b{foo}',5) client.set('c{foo}',5) client.set('{foo}d',6) client.set('{foo}e',7) client.set('e{foo}f',8) print client.get_server_name('foo') print client.get_server_name('c{foo}') print client.get_server_name('{foo}d') print client.get_server_name('{foo}e') print client.get_server_name('e{foo}f') t0 = time.time() print client.tag_keys('*{foo}*') t1 = time.time() print t1-t0
'port': 10000, 'db': 0 }, 'node2': { 'host': '127.0.0.1', 'port': 11000, 'db': 0 }, 'node3': { 'host': '127.0.0.1', 'port': 12000, 'db': 0 }, } redis = RedisShardAPI(RedisServers) PATH = os.getcwd() favicon_path = os.path.join(PATH, 'favicon.ico') settings = { 'debug': True, 'static_path': os.path.join(PATH, 'static'), } Engine = tenjin.Engine(layout='_layout2.html', path=[os.path.join('views', theme) for theme in THEMES] + ['views'], cache=tenjin.MemoryCacheStorage(), preprocess=True) # if __name__ == "__main__": # html= Engine.render('home.html', {
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', {'first': 1}) pipe.zincrby('testzset', 1, 'first') pipe.zadd('testzset', {'second': 2}) pipe.execute() pipe.reset() eq_(self.client.get('test'), b'2') eq_(self.client.zscore('testzset', 'first'), 2.0) eq_(self.client.zscore('testzset', 'second'), 2.0) with self.client.pipeline() as pipe: pipe.set('test', '3') pipe.zadd('testzset', {'first': 4}) pipe.zincrby('testzset', 1, 'first') pipe.zadd('testzset', {'second': 5}) pipe.execute() eq_(self.client.get('test'), b'3') eq_(self.client.zscore('testzset', 'first'), 5.0) eq_(self.client.zscore('testzset', 'second'), 5.0) with self.client.pipeline() as pipe: pipe.watch('test') eq_(self.client.get('test'), b'3') pipe.multi() pipe.incr('test') eq_(pipe.execute(), [4]) eq_(self.client.get('test'), b'4') with self.client.pipeline() as pipe: pipe.watch('test') pipe.multi() pipe.incr('test') self.client.decr('test') self.assertRaises(WatchError, pipe.execute) eq_(self.client.get('test'), b'3') keys_of_names = {} with self.client.pipeline() as pipe: for key in xrange(100): key = str(key) name = pipe.shard_api.get_server_name(key) if name not in keys_of_names: keys_of_names[name] = key else: key1 = key key2 = keys_of_names[name] pipe.watch(key1, key2) pipe.multi() pipe.set(key1, 1) pipe.set(key2, 2) pipe.execute() eq_(self.client.get(key1), b'1') eq_(self.client.get(key2), b'2') break def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval( """ redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') self.client.delete('test7') self.client.delete('test8') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')]) def test_eval(self): self.client.eval(""" return redis.call('set', KEYS[1], ARGV[1]) """, 1, 'test7', '7') eq_(self.client.get('test7'), b('7')) def test_evalsha(self): sha = self.client.script_load(""" return redis.call('set', KEYS[1], ARGV[1]) """) eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK')) eq_(self.client.get('test8'), b('8'))
def setUp(self): self.client = RedisShardAPI(servers) self.clear_db()
from redis_shard.shard import RedisShardAPI import re import uuid server_list = [ {'name':'server1', 'host':'127.0.0.1', 'port':6379}, {'name':'server2', 'host':'127.0.0.1', 'port':6380}, {'name':'server3', 'host':'127.0.0.1', 'port':6381}, {'name':'server4', 'host':'127.0.0.1', 'port':6382}, {'name':'server5', 'host':'127.0.0.1', 'port':6383}, ] jedis = RedisShardAPI(server_list) print("test_key_1"); foo = jedis.get("{test_key}_1"); print("test_key_2"); foo = jedis.get("{test_key}_2"); print("test_key_3"); foo = jedis.get("{test_key}_3"); print("test_key_4"); foo = jedis.get("{test_key}_4"); print("test_key_5"); foo = jedis.get("test_key_5"); print("test_key_6"); foo = jedis.get("test_key_6"); print("test_key_7"); foo = jedis.get("test_key_7"); print("test_key_8"); foo = jedis.get("test_key_8"); print("test_key_9"); foo = jedis.get("test_key_9");
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() pipe.reset() eq_(self.client.get('test'), b'2') eq_(self.client.zscore('testzset', 'first'), 2.0) eq_(self.client.zscore('testzset', 'second'), 2.0) with self.client.pipeline() as pipe: pipe.set('test', '3') pipe.zadd('testzset', 'first', 4) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 5) pipe.execute() eq_(self.client.get('test'), b'3') eq_(self.client.zscore('testzset', 'first'), 5.0) eq_(self.client.zscore('testzset', 'second'), 5.0) with self.client.pipeline() as pipe: pipe.watch('test') eq_(self.client.get('test'), b'3') pipe.multi() pipe.incr('test') eq_(pipe.execute(), [4]) eq_(self.client.get('test'), b'4') with self.client.pipeline() as pipe: pipe.watch('test') pipe.multi() pipe.incr('test') self.client.decr('test') self.assertRaises(WatchError, pipe.execute) eq_(self.client.get('test'), b'3') keys_of_names = {} with self.client.pipeline() as pipe: for key in xrange(100): key = str(key) name = pipe.shard_api.get_server_name(key) if name not in keys_of_names: keys_of_names[name] = key else: key1 = key key2 = keys_of_names[name] pipe.watch(key1, key2) pipe.multi() pipe.set(key1, 1) pipe.set(key2, 2) pipe.execute() eq_(self.client.get(key1), b'1') eq_(self.client.get(key2), b'2') break def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval(""" redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')])
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0'
def __init__(self): self.shard = RedisShardAPI(CACHE)
def get_cache_redis(self): return RedisShardAPI(REDIS_CONFIG_CACHE)
'name': 'feed19', 'host': 'feedproxy', 'port': 6319, 'db': 0 }, { 'name': 'feed20', 'host': 'feedproxy', 'port': 6320, 'db': 0 }, ] FEED_KEY = "{feed%(user_id)s}:list" old_shard = RedisShardAPI(old_servers) new_shard = RedisShardAPI(new_servers) def main(): parser = argparse.ArgumentParser(description='Reshard newsfeed instance') parser.add_argument('--start', type=int, required=True, help='start user id') parser.add_argument('--end', type=int, required=True, help='start user id') parser.add_argument('--show_only', type=bool, default=False, help='only showw migrate process') parser.add_argument('--delete',
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0' def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval( """ redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2']
class TestSentinelShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(**sentinel_settings) self.clear_db() def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') self.client.delete('test1') self.client.delete('test2') self.client.delete('test3') self.client.delete('test7') self.client.delete('test8') def test_zset(self): self.client.zadd('testzset', 'first', 1) self.client.zadd('testzset', 'second', 2) self.client.zrange('testzset', 0, -1) == ['first', 'second'] def test_list(self): self.client.rpush('testlist', 0) self.client.rpush('testlist', 1) self.client.rpush('testlist', 2) self.client.rpush('testlist', 3) self.client.rpop('testlist') self.client.lpop('testlist') self.client.lrange('testlist', 0, -1) == ['1', '2'] def test_mget(self): self.client.set('test1', 1) self.client.set('test2', 2) self.client.set('test3', 3) eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')]) def test_mset(self): self.client.mset({'test4': 4, 'test5': 5, 'test6': 6}) eq_(self.client.get('test4'), b('4')) eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')]) def test_eval(self): self.client.eval(""" return redis.call('set', KEYS[1], ARGV[1]) """, 1, 'test7', '7') eq_(self.client.get('test7'), b('7')) def test_evalsha(self): sha = self.client.script_load(""" return redis.call('set', KEYS[1], ARGV[1]) """) eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK')) eq_(self.client.get('test8'), b('8'))
class Crawler: def __init__(self,seed_url): self.visited_url = set() self.un_visited_url = [seed_url] self.seed_url = seed_url self.client = RedisShardAPI(REDIS_SEARCH_SERVERS) self.link_index_dict = {} self.title_index_dict = {} def read_url(self,url): try: openUrl = urllib2.urlopen(url) content = openUrl.read() openUrl.close() return content except: return "" def pop_url(self): if len(self.un_visited_url) > 0: return self.un_visited_url.pop() else: return None def add_url(self,url): self.un_visited_url.append(url) def save_url(self,url): self.visited_url.add(url) def is_visited_url(self,url): if url in self.visited_url: return True return False def compare_url(self,url): if not url.startswith("http"): return "http://"+url else: return url def index_one_link(self,url,name): words = segment(name) for w in words: if self.link_index_dict.has_key(w): dct = self.link_index_dict[w] if dct.has_key(url): self.link_index_dict[w][url] += 1 else: self.link_index_dict[w][url] = 1 else: self.link_index_dict[w] = {} self.link_index_dict[w][url] = 1 self.client.hset(TABLE_URL_TITLE_LINK,url,name) def index_all_links(self): for key,value in self.link_index_dict.items(): value = cPickle.dumps(value) self.client.hset(INVERT_TABLE_LINK,key,value) for key,value in self.title_index_dict.items(): value = cPickle.dumps(value) self.client.hset(INVERT_TABLE_TITLE,key,value) def index_title(self,url,title): #print url,title words = segment(title) for w in words: if self.title_index_dict.has_key(w): dct = self.title_index_dict[w] if dct.has_key(url): self.title_index_dict[w][url] += 1 else: self.title_index_dict[w][url] = 1 else: self.title_index_dict[w] = {} self.title_index_dict[w][url] = 1 self.client.hset(TABLE_URL_TITLE_TITLE,url,title) def get_title(self,content): p = re.compile(r'<title>.*</title>',re.I) title = p.findall(content) sname = "" if title: try: sname = title[0] sname = sname.decode('gbk') except: return "" else: return "" sname = sname.lower() sname = sname[sname.find("<title>")+7:sname.find("</title")] return sname def get_urls(self,url): content = self.read_url(url) title = self.get_title(content) if title: self.index_title(url,title) if url.rfind("/") == -1: host = url else: host = url[:url.rfind("/")+1] if not host.endswith("/"): host += "/" result = set() if not content: return [] else: p = re.compile( r'<a.+?href=.+?>.+?</a>' ) pname = re.compile( r'(?<=>).*?(?=</a>)' ) phref = re.compile( r'(?<=href\=\").*?(?=\")' ) items = p.findall(content) for item in items: can_index = True sname = pname.findall(item) if sname: sname = sname[0].strip() try: sname = sname.decode('gbk') except: can_index = False shref = phref.findall(item) if shref: shref = shref[0].strip() if not shref.startswith("www") and not shref.startswith("http:"): if shref=="#": continue #本网页的相对地址转换成绝对地址 if host: if shref.startswith("/"): shref = host + shref[1:] elif shref.startswith("./"): shref = host + shref[2:] else: shref = "" if shref.find("ustl.edu.cn") == -1 and shref.find("202.199.240.66") == -1: continue elif shref.find("javascript") != -1: continue elif shref.endswith("jpg") or shref.endswith("rmvb") or shref.endswith("avi") or shref.endswith("doc"): continue else: result.add(shref) if sname and shref and can_index: self.index_one_link(shref,sname) return result def do_crawl(self): """ Start to crawle the ustl pages """ start_time = time.time() sum_url = 0 url = self.pop_url() while url != None: self.save_url(url) urls = self.get_urls(url) for link in urls: link = self.compare_url(link) if self.is_visited_url(link) == False: self.save_url(link) self.add_url(link) else: continue print url,"\t now sum is: ",sum_url sum_url += 1 #if sum_url > 5: #break url = self.pop_url() print "All url nums is: ",sum_url,"\t Cost time is ",str(time.time()-start_time) return sum_url
#!/usr/bin/env python import time from config import servers from redis_shard.shard import RedisShardAPI client = RedisShardAPI(servers) client.set("test", 1) print client.get("test") client.zadd("testset", "first", 1) client.zadd("testset", "second", 2) print client.zrange("testset", 0, -1) print client.zrank("testset", "second") print client.zrank("testset2", "second") client.set("foo", 2) client.set("a{foo}", 5) client.set("b{foo}", 5) client.set("c{foo}", 5) client.set("{foo}d", 6) client.set("{foo}e", 7) client.set("e{foo}f", 8) print client.get_server_name("foo") print client.get_server_name("c{foo}") print client.get_server_name("{foo}d") print client.get_server_name("{foo}e") print client.get_server_name("e{foo}f") t0 = time.time() print client.tag_keys("*{foo}*") t1 = time.time()
def setUp(self): self.client = RedisShardAPI(**settings) self.clear_db()
redis_url = urlparse(Configuration.REDIS_URL) redis = Redis(host=redis_url.hostname, port=redis_url.port, password=redis_url.password) scheduler = Scheduler('scheduled', interval=60, connection=redis) wigo_queued_db = WigoQueuedDB(redis) if Configuration.RDBMS_REPLICATE else None wigo_rdbms = WigoRdbms() if Configuration.ENVIRONMENT != 'test': servers = [] for index, redis_url in enumerate(Configuration.REDIS_URLS): parsed = urlparse(redis_url) servers.append({ 'name': 'redis_{}'.format(index), 'host': parsed.hostname, 'port': parsed.port, 'password': parsed.password }) from redis_shard import shard shard.SHARD_METHODS = set(shard.SHARD_METHODS) shard.SHARD_METHODS.add('zscan') shard.SHARD_METHODS.add('zscan_iter') sharded_redis = RedisShardAPI(servers, hash_method='md5') wigo_db = WigoRedisDB(sharded_redis, wigo_queued_db) else: wigo_db = WigoRedisDB(redis, wigo_queued_db)
def setUp(self): self.client = RedisShardAPI(**sentinel_settings) self.clear_db()
class SessionStore(SessionBase): """ Implements Redis database session store. """ def __init__(self, session_key=None): super(SessionStore, self).__init__(session_key) try: unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None) except AttributeError: unix_socket_path = None if unix_socket_path is None: self.server = RedisShardAPI(settings.SESSION_REDIS_HOSTS) else: self.server = redis.StrictRedis( unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', '/var/run/redis/redis.sock'), db=getattr(settings, 'SESSION_REDIS_DB', 0), password=getattr(settings, 'SESSION_REDIS_PASSWORD', None), ) def load(self): try: session_data = self.server.get(self.get_real_stored_key(self._get_or_create_session_key())) return self.decode(force_unicode(session_data)) except: self.create() return {} def exists(self, session_key): return self.server.exists(self.get_real_stored_key(session_key)) def create(self): while True: self._session_key = self._get_new_session_key() try: self.save(must_create=True) except CreateError: continue self.modified = True return def save(self, must_create=False): if not getattr(settings, 'SITE_READ_ONLY', False): if must_create and self.exists(self._get_or_create_session_key()): raise CreateError this_session = self._get_session(no_load=must_create) data = self.encode(self._get_session(no_load=must_create)) if redis.VERSION[0] >= 2: self.server.setex(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age(), data) else: self.server.set(self.get_real_stored_key(self._get_or_create_session_key()), data) self.server.expire(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age()) def delete(self, session_key=None): if session_key is None: if self.session_key is None: return session_key = self.session_key try: self.server.delete(self.get_real_stored_key(session_key)) except: pass def get_real_stored_key(self, session_key): """Return the real key name in redis storage @return string """ prefix = getattr(settings, 'SESSION_REDIS_PREFIX', '') if not prefix: return session_key return ':'.join([prefix, session_key])
class TestShard(unittest.TestCase): def setUp(self): self.client = RedisShardAPI(servers) self.clear_db() def tearDown(self): pass def clear_db(self): self.client.delete('testset') self.client.delete('testzset') self.client.delete('testlist') def test_pipeline(self): self.client.set('test', '1') pipe = self.client.pipeline() pipe.set('test', '2') pipe.zadd('testzset', 'first', 1) pipe.zincrby('testzset', 'first') pipe.zadd('testzset', 'second', 2) pipe.execute() self.client.get('test') == '2' self.client.zscore('testzset', 'fist') == '3.0' def test_pipeline_script(self): pipe = self.client.pipeline() for i in range(100): pipe.eval(""" redis.call('set', KEYS[1], ARGV[1]) """, 1, 'testx%d' % i, i) pipe.execute() for i in range(100): eq_(self.client.get('testx%d' % i), b('%d' % i))
'password': '******', 'db': '0' } redis_info_2 = { 'host': 'mb.y.redis.sohucs.com', 'port': '22829', 'password': '******', 'db': '0' } servers = [ {'name': 'server1', 'host': 'mb.y.redis.sohucs.com', 'port': 22812, 'db': 0, 'password': '******'}, {'name': 'server2', 'host': 'mb.y.redis.sohucs.com', 'port': 22829, 'db': 0, 'password': '******'} ] client = RedisShardAPI(servers, hash_method='md5') def connect_db(host, port, password, db): pool = redis.ConnectionPool(host=host, port=port, password=password, decode_responses=True, db=db) r = redis.StrictRedis(connection_pool=pool) return r def recommend(clk_cid): # 得到点击历史 clk_his = user_clk_his[clk_cid] # print '点击历史', clk_his # 建立召回列表 recall_list = []
#!/usr/bin/env python # -*- coding: utf-8 -*- from redis_shard.shard import RedisShardAPI from config import servers client = RedisShardAPI(servers) client.set('test',1) print client.get('test') client.zadd('testset','first',1) client.zadd('testset','second',2) print client.zrange('testset',0,-1) client.set('foo',2) client.set('a{foo}',5) client.set('b{foo}',5) client.set('c{foo}',5) print client.get('foo')
def __init__(self, server): try: self.client = RedisShardAPI(server) except: raise Exception, 'Failed to connect redis sharding'