def __init__(self,seed_url):
self.visited_url = set()
self.un_visited_url = [seed_url]
self.seed_url = seed_url
self.client = RedisShardAPI(REDIS_SEARCH_SERVERS)
self.link_index_dict = {}
self.title_index_dict = {}
def setup(self, app):
for other in app.plugins:
if not isinstance(other, RedisShardPlugin): continue
if other.keyword == self.keyword:
raise PluginError("Found another redis shard plugin with "\
"conflicting settings (non-unique keyword).")
if self.redisdb is None:
self.redisdb = RedisShardAPI(self.config)
class Memcache(object):
def __init__(self):
self.shard = RedisShardAPI(CACHE)
@staticmethod
def cache_key(func, *args):
return u":".join([func.__name__] + map(unicode, args))
def cache(self):
def wraped(func):
@functools.wraps(func)
def fcall(obj, *args):
key = self.cache_key(func, *args)
rst = self.shard.get(key)
logging.warning(key)
if not rst:
data = func(obj, *args)
self.shard.set(key, json.dumps(data))
self.shard.expire(key, 3600)
return data
return rst
return fcall
return wraped
def invalidate(self, func, *args):
key = self.cache_key(func, *args)
self.shard.delete(key)
def reconnect_redis(self):
if self.shared_client and Storage.storage:
return Storage.storage
storage = RedisShardAPI(self.context.config.REDIS_SERVERS,
hash_method='md5')
if self.shared_client:
Storage.storage = storage
return storage
def reconnect_redis(self):
'''Reconnect to redis.
:return: Redis client instance
:rettype: redis.Redis
'''
if self.shared_client and Storage.storage:
return Storage.storage
storage = RedisShardAPI(self.context.config.REDIS_SERVERS,
hash_method='md5')
if self.shared_client:
Storage.storage = storage
return storage
def decorator(*args, **kwargs):
servers = current_app.config.get("REDIS_SHARD_URL")
client = RedisShardAPI(servers, hash_method='md5')
now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
ip = request.remote_addr
key = "%s{%s}" % (ip, now)
client.incr(key)
if client.get(key):
num = int(client.get(key))
else:
num = 0
client.expire(key, 10)
if num > times:
return jsonify(
dict(code=40003,
msg='warning: Refused to too many requests !'))
return f(*args, **kwargs)
def __init__(self, session_key=None):
super(SessionStore, self).__init__(session_key)
try:
unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None)
except AttributeError:
unix_socket_path = None
if unix_socket_path is None:
self.server = RedisShardAPI(settings.SESSION_REDIS_HOSTS)
else:
self.server = redis.StrictRedis(
unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', '/var/run/redis/redis.sock'),
db=getattr(settings, 'SESSION_REDIS_DB', 0),
password=getattr(settings, 'SESSION_REDIS_PASSWORD', None),
)
def decorator(*args, **kwargs):
servers = current_app.config.get("REDIS_SHARD_URL")
client = RedisShardAPI(servers, hash_method='md5')
now = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
ip = request.remote_addr
key = "%s{%s}" % (ip, now)
client.incr(key)
if client.get(key):
num = int(client.get(key))
else:
num = 0
client.expire(key, 10)
if num > times:
return jsonify(dict(
code=40003,
msg='warning: Refused to too many requests !'))
return f(*args, **kwargs)
'port': 6379,
'db': 0
},
#{'name':'server5','host':'127.0.0.1','port':14000,'db':0},
#{'name':'server6','host':'127.0.0.1','port':15000,'db':0},
]
########NEW FILE########
__FILENAME__ = myapp
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from redis_shard.shard import RedisShardAPI
from config import servers
import time
client = RedisShardAPI(servers)
client.set('test', 1)
print client.get('test')
client.zadd('testset', 'first', 1)
client.zadd('testset', 'second', 2)
print client.zrange('testset', 0, -1)
print client.zrank('testset', 'second')
print client.zrank('testset2', 'second')
client.set('foo', 2)
client.set('a{foo}', 5)
client.set('b{foo}', 5)
client.set('c{foo}', 5)
client.set('{foo}d', 6)
client.set('{foo}e', 7)
client.set('e{foo}f', 8)
def init_app(self, app):
servers = app.config.get("REDIS_SHARD_URL")
client = RedisShardAPI(servers, hash_method='md5')
self._cache = client
return client
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from redis_shard.shard import RedisShardAPI
from config import servers
import time
client = RedisShardAPI(servers)
client.set('test',1)
print client.get('test')
client.zadd('testset','first',1)
client.zadd('testset','second',2)
print client.zrange('testset',0,-1)
print client.zrank('testset','second')
print client.zrank('testset2','second')
client.set('foo',2)
client.set('a{foo}',5)
client.set('b{foo}',5)
client.set('c{foo}',5)
client.set('{foo}d',6)
client.set('{foo}e',7)
client.set('e{foo}f',8)
print client.get_server_name('foo')
print client.get_server_name('c{foo}')
print client.get_server_name('{foo}d')
print client.get_server_name('{foo}e')
print client.get_server_name('e{foo}f')
t0 = time.time()
print client.tag_keys('*{foo}*')
t1 = time.time()
print t1-t0
'port': 10000,
'db': 0
},
'node2': {
'host': '127.0.0.1',
'port': 11000,
'db': 0
},
'node3': {
'host': '127.0.0.1',
'port': 12000,
'db': 0
},
}
redis = RedisShardAPI(RedisServers)
PATH = os.getcwd()
favicon_path = os.path.join(PATH, 'favicon.ico')
settings = {
'debug': True,
'static_path': os.path.join(PATH, 'static'),
}
Engine = tenjin.Engine(layout='_layout2.html',
path=[os.path.join('views', theme)
for theme in THEMES] + ['views'],
cache=tenjin.MemoryCacheStorage(),
preprocess=True)
# if __name__ == "__main__":
# html= Engine.render('home.html', {
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(**settings)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', {'first': 1})
pipe.zincrby('testzset', 1, 'first')
pipe.zadd('testzset', {'second': 2})
pipe.execute()
pipe.reset()
eq_(self.client.get('test'), b'2')
eq_(self.client.zscore('testzset', 'first'), 2.0)
eq_(self.client.zscore('testzset', 'second'), 2.0)
with self.client.pipeline() as pipe:
pipe.set('test', '3')
pipe.zadd('testzset', {'first': 4})
pipe.zincrby('testzset', 1, 'first')
pipe.zadd('testzset', {'second': 5})
pipe.execute()
eq_(self.client.get('test'), b'3')
eq_(self.client.zscore('testzset', 'first'), 5.0)
eq_(self.client.zscore('testzset', 'second'), 5.0)
with self.client.pipeline() as pipe:
pipe.watch('test')
eq_(self.client.get('test'), b'3')
pipe.multi()
pipe.incr('test')
eq_(pipe.execute(), [4])
eq_(self.client.get('test'), b'4')
with self.client.pipeline() as pipe:
pipe.watch('test')
pipe.multi()
pipe.incr('test')
self.client.decr('test')
self.assertRaises(WatchError, pipe.execute)
eq_(self.client.get('test'), b'3')
keys_of_names = {}
with self.client.pipeline() as pipe:
for key in xrange(100):
key = str(key)
name = pipe.shard_api.get_server_name(key)
if name not in keys_of_names:
keys_of_names[name] = key
else:
key1 = key
key2 = keys_of_names[name]
pipe.watch(key1, key2)
pipe.multi()
pipe.set(key1, 1)
pipe.set(key2, 2)
pipe.execute()
eq_(self.client.get(key1), b'1')
eq_(self.client.get(key2), b'2')
break
def test_pipeline_script(self):
pipe = self.client.pipeline()
for i in range(100):
pipe.eval(
"""
redis.call('set', KEYS[1], ARGV[1])
""", 1, 'testx%d' % i, i)
pipe.execute()
for i in range(100):
eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(**settings)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
self.client.delete('test1')
self.client.delete('test2')
self.client.delete('test3')
self.client.delete('test7')
self.client.delete('test8')
def test_zset(self):
self.client.zadd('testzset', 'first', 1)
self.client.zadd('testzset', 'second', 2)
self.client.zrange('testzset', 0, -1) == ['first', 'second']
def test_list(self):
self.client.rpush('testlist', 0)
self.client.rpush('testlist', 1)
self.client.rpush('testlist', 2)
self.client.rpush('testlist', 3)
self.client.rpop('testlist')
self.client.lpop('testlist')
self.client.lrange('testlist', 0, -1) == ['1', '2']
def test_mget(self):
self.client.set('test1', 1)
self.client.set('test2', 2)
self.client.set('test3', 3)
eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')])
def test_mset(self):
self.client.mset({'test4': 4, 'test5': 5, 'test6': 6})
eq_(self.client.get('test4'), b('4'))
eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')])
def test_eval(self):
self.client.eval("""
return redis.call('set', KEYS[1], ARGV[1])
""", 1, 'test7', '7')
eq_(self.client.get('test7'), b('7'))
def test_evalsha(self):
sha = self.client.script_load("""
return redis.call('set', KEYS[1], ARGV[1])
""")
eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK'))
eq_(self.client.get('test8'), b('8'))
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
from redis_shard.shard import RedisShardAPI
import re
import uuid
server_list = [
{'name':'server1', 'host':'127.0.0.1', 'port':6379},
{'name':'server2', 'host':'127.0.0.1', 'port':6380},
{'name':'server3', 'host':'127.0.0.1', 'port':6381},
{'name':'server4', 'host':'127.0.0.1', 'port':6382},
{'name':'server5', 'host':'127.0.0.1', 'port':6383},
]
jedis = RedisShardAPI(server_list)
print("test_key_1");
foo = jedis.get("{test_key}_1");
print("test_key_2");
foo = jedis.get("{test_key}_2");
print("test_key_3");
foo = jedis.get("{test_key}_3");
print("test_key_4");
foo = jedis.get("{test_key}_4");
print("test_key_5");
foo = jedis.get("test_key_5");
print("test_key_6");
foo = jedis.get("test_key_6");
print("test_key_7");
foo = jedis.get("test_key_7");
print("test_key_8");
foo = jedis.get("test_key_8");
print("test_key_9");
foo = jedis.get("test_key_9");
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(**settings)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', 'first', 1)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 2)
pipe.execute()
pipe.reset()
eq_(self.client.get('test'), b'2')
eq_(self.client.zscore('testzset', 'first'), 2.0)
eq_(self.client.zscore('testzset', 'second'), 2.0)
with self.client.pipeline() as pipe:
pipe.set('test', '3')
pipe.zadd('testzset', 'first', 4)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 5)
pipe.execute()
eq_(self.client.get('test'), b'3')
eq_(self.client.zscore('testzset', 'first'), 5.0)
eq_(self.client.zscore('testzset', 'second'), 5.0)
with self.client.pipeline() as pipe:
pipe.watch('test')
eq_(self.client.get('test'), b'3')
pipe.multi()
pipe.incr('test')
eq_(pipe.execute(), [4])
eq_(self.client.get('test'), b'4')
with self.client.pipeline() as pipe:
pipe.watch('test')
pipe.multi()
pipe.incr('test')
self.client.decr('test')
self.assertRaises(WatchError, pipe.execute)
eq_(self.client.get('test'), b'3')
keys_of_names = {}
with self.client.pipeline() as pipe:
for key in xrange(100):
key = str(key)
name = pipe.shard_api.get_server_name(key)
if name not in keys_of_names:
keys_of_names[name] = key
else:
key1 = key
key2 = keys_of_names[name]
pipe.watch(key1, key2)
pipe.multi()
pipe.set(key1, 1)
pipe.set(key2, 2)
pipe.execute()
eq_(self.client.get(key1), b'1')
eq_(self.client.get(key2), b'2')
break
def test_pipeline_script(self):
pipe = self.client.pipeline()
for i in range(100):
pipe.eval("""
redis.call('set', KEYS[1], ARGV[1])
""", 1, 'testx%d' % i, i)
pipe.execute()
for i in range(100):
eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
self.client.delete('test1')
self.client.delete('test2')
self.client.delete('test3')
def test_zset(self):
self.client.zadd('testzset', 'first', 1)
self.client.zadd('testzset', 'second', 2)
self.client.zrange('testzset', 0, -1) == ['first', 'second']
def test_list(self):
self.client.rpush('testlist', 0)
self.client.rpush('testlist', 1)
self.client.rpush('testlist', 2)
self.client.rpush('testlist', 3)
self.client.rpop('testlist')
self.client.lpop('testlist')
self.client.lrange('testlist', 0, -1) == ['1', '2']
def test_mget(self):
self.client.set('test1', 1)
self.client.set('test2', 2)
self.client.set('test3', 3)
eq_(self.client.mget('test1', 'test2', 'test3'),
[b('1'), b('2'), b('3')])
def test_mset(self):
self.client.mset({'test4': 4, 'test5': 5, 'test6': 6})
eq_(self.client.get('test4'), b('4'))
eq_(self.client.mget('test4', 'test5', 'test6'),
[b('4'), b('5'), b('6')])
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', 'first', 1)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 2)
pipe.execute()
self.client.get('test') == '2'
self.client.zscore('testzset', 'fist') == '3.0'
def __init__(self):
self.shard = RedisShardAPI(CACHE)
def get_cache_redis(self):
return RedisShardAPI(REDIS_CONFIG_CACHE)
'name': 'feed19',
'host': 'feedproxy',
'port': 6319,
'db': 0
},
{
'name': 'feed20',
'host': 'feedproxy',
'port': 6320,
'db': 0
},
]
FEED_KEY = "{feed%(user_id)s}:list"
old_shard = RedisShardAPI(old_servers)
new_shard = RedisShardAPI(new_servers)
def main():
parser = argparse.ArgumentParser(description='Reshard newsfeed instance')
parser.add_argument('--start',
type=int,
required=True,
help='start user id')
parser.add_argument('--end', type=int, required=True, help='start user id')
parser.add_argument('--show_only',
type=bool,
default=False,
help='only showw migrate process')
parser.add_argument('--delete',
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', 'first', 1)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 2)
pipe.execute()
self.client.get('test') == '2'
self.client.zscore('testzset', 'fist') == '3.0'
def test_pipeline_script(self):
pipe = self.client.pipeline()
for i in range(100):
pipe.eval(
"""
redis.call('set', KEYS[1], ARGV[1])
""", 1, 'testx%d' % i, i)
pipe.execute()
for i in range(100):
eq_(self.client.get('testx%d' % i), b('%d' % i))
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_zset(self):
self.client.zadd('testzset', 'first', 1)
self.client.zadd('testzset', 'second', 2)
self.client.zrange('testzset', 0, -1) == ['first', 'second']
def test_list(self):
self.client.rpush('testlist', 0)
self.client.rpush('testlist', 1)
self.client.rpush('testlist', 2)
self.client.rpush('testlist', 3)
self.client.rpop('testlist')
self.client.lpop('testlist')
self.client.lrange('testlist', 0, -1) == ['1', '2']
class TestSentinelShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(**sentinel_settings)
self.clear_db()
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
self.client.delete('test1')
self.client.delete('test2')
self.client.delete('test3')
self.client.delete('test7')
self.client.delete('test8')
def test_zset(self):
self.client.zadd('testzset', 'first', 1)
self.client.zadd('testzset', 'second', 2)
self.client.zrange('testzset', 0, -1) == ['first', 'second']
def test_list(self):
self.client.rpush('testlist', 0)
self.client.rpush('testlist', 1)
self.client.rpush('testlist', 2)
self.client.rpush('testlist', 3)
self.client.rpop('testlist')
self.client.lpop('testlist')
self.client.lrange('testlist', 0, -1) == ['1', '2']
def test_mget(self):
self.client.set('test1', 1)
self.client.set('test2', 2)
self.client.set('test3', 3)
eq_(self.client.mget('test1', 'test2', 'test3'), [b('1'), b('2'), b('3')])
def test_mset(self):
self.client.mset({'test4': 4, 'test5': 5, 'test6': 6})
eq_(self.client.get('test4'), b('4'))
eq_(self.client.mget('test4', 'test5', 'test6'), [b('4'), b('5'), b('6')])
def test_eval(self):
self.client.eval("""
return redis.call('set', KEYS[1], ARGV[1])
""", 1, 'test7', '7')
eq_(self.client.get('test7'), b('7'))
def test_evalsha(self):
sha = self.client.script_load("""
return redis.call('set', KEYS[1], ARGV[1])
""")
eq_(self.client.evalsha(sha, 1, 'test8', b('8')), b('OK'))
eq_(self.client.get('test8'), b('8'))
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', 'first', 1)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 2)
pipe.execute()
self.client.get('test') == '2'
self.client.zscore('testzset', 'fist') == '3.0'
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
class Crawler:
def __init__(self,seed_url):
self.visited_url = set()
self.un_visited_url = [seed_url]
self.seed_url = seed_url
self.client = RedisShardAPI(REDIS_SEARCH_SERVERS)
self.link_index_dict = {}
self.title_index_dict = {}
def read_url(self,url):
try:
openUrl = urllib2.urlopen(url)
content = openUrl.read()
openUrl.close()
return content
except:
return ""
def pop_url(self):
if len(self.un_visited_url) > 0:
return self.un_visited_url.pop()
else:
return None
def add_url(self,url):
self.un_visited_url.append(url)
def save_url(self,url):
self.visited_url.add(url)
def is_visited_url(self,url):
if url in self.visited_url:
return True
return False
def compare_url(self,url):
if not url.startswith("http"):
return "http://"+url
else:
return url
def index_one_link(self,url,name):
words = segment(name)
for w in words:
if self.link_index_dict.has_key(w):
dct = self.link_index_dict[w]
if dct.has_key(url):
self.link_index_dict[w][url] += 1
else:
self.link_index_dict[w][url] = 1
else:
self.link_index_dict[w] = {}
self.link_index_dict[w][url] = 1
self.client.hset(TABLE_URL_TITLE_LINK,url,name)
def index_all_links(self):
for key,value in self.link_index_dict.items():
value = cPickle.dumps(value)
self.client.hset(INVERT_TABLE_LINK,key,value)
for key,value in self.title_index_dict.items():
value = cPickle.dumps(value)
self.client.hset(INVERT_TABLE_TITLE,key,value)
def index_title(self,url,title):
#print url,title
words = segment(title)
for w in words:
if self.title_index_dict.has_key(w):
dct = self.title_index_dict[w]
if dct.has_key(url):
self.title_index_dict[w][url] += 1
else:
self.title_index_dict[w][url] = 1
else:
self.title_index_dict[w] = {}
self.title_index_dict[w][url] = 1
self.client.hset(TABLE_URL_TITLE_TITLE,url,title)
def get_title(self,content):
p = re.compile(r'<title>.*</title>',re.I)
title = p.findall(content)
sname = ""
if title:
try:
sname = title[0]
sname = sname.decode('gbk')
except:
return ""
else:
return ""
sname = sname.lower()
sname = sname[sname.find("<title>")+7:sname.find("</title")]
return sname
def get_urls(self,url):
content = self.read_url(url)
title = self.get_title(content)
if title:
self.index_title(url,title)
if url.rfind("/") == -1:
host = url
else:
host = url[:url.rfind("/")+1]
if not host.endswith("/"):
host += "/"
result = set()
if not content:
return []
else:
p = re.compile( r'<a.+?href=.+?>.+?</a>' )
pname = re.compile( r'(?<=>).*?(?=</a>)' )
phref = re.compile( r'(?<=href\=\").*?(?=\")' )
items = p.findall(content)
for item in items:
can_index = True
sname = pname.findall(item)
if sname:
sname = sname[0].strip()
try:
sname = sname.decode('gbk')
except:
can_index = False
shref = phref.findall(item)
if shref:
shref = shref[0].strip()
if not shref.startswith("www") and not shref.startswith("http:"):
if shref=="#":
continue
#本网页的相对地址转换成绝对地址
if host:
if shref.startswith("/"):
shref = host + shref[1:]
elif shref.startswith("./"):
shref = host + shref[2:]
else:
shref = ""
if shref.find("ustl.edu.cn") == -1 and shref.find("202.199.240.66") == -1:
continue
elif shref.find("javascript") != -1:
continue
elif shref.endswith("jpg") or shref.endswith("rmvb") or shref.endswith("avi") or shref.endswith("doc"):
continue
else:
result.add(shref)
if sname and shref and can_index:
self.index_one_link(shref,sname)
return result
def do_crawl(self):
"""
Start to crawle the ustl pages
"""
start_time = time.time()
sum_url = 0
url = self.pop_url()
while url != None:
self.save_url(url)
urls = self.get_urls(url)
for link in urls:
link = self.compare_url(link)
if self.is_visited_url(link) == False:
self.save_url(link)
self.add_url(link)
else:
continue
print url,"\t now sum is: ",sum_url
sum_url += 1
#if sum_url > 5:
#break
url = self.pop_url()
print "All url nums is: ",sum_url,"\t Cost time is ",str(time.time()-start_time)
return sum_url
#!/usr/bin/env python
import time
from config import servers
from redis_shard.shard import RedisShardAPI
client = RedisShardAPI(servers)
client.set("test", 1)
print client.get("test")
client.zadd("testset", "first", 1)
client.zadd("testset", "second", 2)
print client.zrange("testset", 0, -1)
print client.zrank("testset", "second")
print client.zrank("testset2", "second")
client.set("foo", 2)
client.set("a{foo}", 5)
client.set("b{foo}", 5)
client.set("c{foo}", 5)
client.set("{foo}d", 6)
client.set("{foo}e", 7)
client.set("e{foo}f", 8)
print client.get_server_name("foo")
print client.get_server_name("c{foo}")
print client.get_server_name("{foo}d")
print client.get_server_name("{foo}e")
print client.get_server_name("e{foo}f")
t0 = time.time()
print client.tag_keys("*{foo}*")
t1 = time.time()
def setUp(self):
self.client = RedisShardAPI(**settings)
self.clear_db()
redis_url = urlparse(Configuration.REDIS_URL)
redis = Redis(host=redis_url.hostname,
port=redis_url.port,
password=redis_url.password)
scheduler = Scheduler('scheduled', interval=60, connection=redis)
wigo_queued_db = WigoQueuedDB(redis) if Configuration.RDBMS_REPLICATE else None
wigo_rdbms = WigoRdbms()
if Configuration.ENVIRONMENT != 'test':
servers = []
for index, redis_url in enumerate(Configuration.REDIS_URLS):
parsed = urlparse(redis_url)
servers.append({
'name': 'redis_{}'.format(index),
'host': parsed.hostname,
'port': parsed.port,
'password': parsed.password
})
from redis_shard import shard
shard.SHARD_METHODS = set(shard.SHARD_METHODS)
shard.SHARD_METHODS.add('zscan')
shard.SHARD_METHODS.add('zscan_iter')
sharded_redis = RedisShardAPI(servers, hash_method='md5')
wigo_db = WigoRedisDB(sharded_redis, wigo_queued_db)
else:
wigo_db = WigoRedisDB(redis, wigo_queued_db)
def setUp(self):
self.client = RedisShardAPI(**sentinel_settings)
self.clear_db()
class SessionStore(SessionBase):
"""
Implements Redis database session store.
"""
def __init__(self, session_key=None):
super(SessionStore, self).__init__(session_key)
try:
unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', None)
except AttributeError:
unix_socket_path = None
if unix_socket_path is None:
self.server = RedisShardAPI(settings.SESSION_REDIS_HOSTS)
else:
self.server = redis.StrictRedis(
unix_socket_path=getattr(settings, 'SESSION_REDIS_UNIX_DOMAIN_SOCKET_PATH', '/var/run/redis/redis.sock'),
db=getattr(settings, 'SESSION_REDIS_DB', 0),
password=getattr(settings, 'SESSION_REDIS_PASSWORD', None),
)
def load(self):
try:
session_data = self.server.get(self.get_real_stored_key(self._get_or_create_session_key()))
return self.decode(force_unicode(session_data))
except:
self.create()
return {}
def exists(self, session_key):
return self.server.exists(self.get_real_stored_key(session_key))
def create(self):
while True:
self._session_key = self._get_new_session_key()
try:
self.save(must_create=True)
except CreateError:
continue
self.modified = True
return
def save(self, must_create=False):
if not getattr(settings, 'SITE_READ_ONLY', False):
if must_create and self.exists(self._get_or_create_session_key()):
raise CreateError
this_session = self._get_session(no_load=must_create)
data = self.encode(self._get_session(no_load=must_create))
if redis.VERSION[0] >= 2:
self.server.setex(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age(), data)
else:
self.server.set(self.get_real_stored_key(self._get_or_create_session_key()), data)
self.server.expire(self.get_real_stored_key(self._get_or_create_session_key()), self.get_expiry_age())
def delete(self, session_key=None):
if session_key is None:
if self.session_key is None:
return
session_key = self.session_key
try:
self.server.delete(self.get_real_stored_key(session_key))
except:
pass
def get_real_stored_key(self, session_key):
"""Return the real key name in redis storage
@return string
"""
prefix = getattr(settings, 'SESSION_REDIS_PREFIX', '')
if not prefix:
return session_key
return ':'.join([prefix, session_key])
class TestShard(unittest.TestCase):
def setUp(self):
self.client = RedisShardAPI(servers)
self.clear_db()
def tearDown(self):
pass
def clear_db(self):
self.client.delete('testset')
self.client.delete('testzset')
self.client.delete('testlist')
def test_pipeline(self):
self.client.set('test', '1')
pipe = self.client.pipeline()
pipe.set('test', '2')
pipe.zadd('testzset', 'first', 1)
pipe.zincrby('testzset', 'first')
pipe.zadd('testzset', 'second', 2)
pipe.execute()
self.client.get('test') == '2'
self.client.zscore('testzset', 'fist') == '3.0'
def test_pipeline_script(self):
pipe = self.client.pipeline()
for i in range(100):
pipe.eval("""
redis.call('set', KEYS[1], ARGV[1])
""", 1, 'testx%d' % i, i)
pipe.execute()
for i in range(100):
eq_(self.client.get('testx%d' % i), b('%d' % i))
'password': '******',
'db': '0'
}
redis_info_2 = {
'host': 'mb.y.redis.sohucs.com',
'port': '22829',
'password': '******',
'db': '0'
}
servers = [
{'name': 'server1', 'host': 'mb.y.redis.sohucs.com', 'port': 22812, 'db': 0,
'password': '******'},
{'name': 'server2', 'host': 'mb.y.redis.sohucs.com', 'port': 22829, 'db': 0,
'password': '******'}
]
client = RedisShardAPI(servers, hash_method='md5')
def connect_db(host, port, password, db):
pool = redis.ConnectionPool(host=host, port=port, password=password, decode_responses=True, db=db)
r = redis.StrictRedis(connection_pool=pool)
return r
def recommend(clk_cid):
# 得到点击历史
clk_his = user_clk_his[clk_cid]
# print '点击历史', clk_his
# 建立召回列表
recall_list = []
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from redis_shard.shard import RedisShardAPI
from config import servers
client = RedisShardAPI(servers)
client.set('test',1)
print client.get('test')
client.zadd('testset','first',1)
client.zadd('testset','second',2)
print client.zrange('testset',0,-1)
client.set('foo',2)
client.set('a{foo}',5)
client.set('b{foo}',5)
client.set('c{foo}',5)
print client.get('foo')
def __init__(self, server):
try:
self.client = RedisShardAPI(server)
except:
raise Exception, 'Failed to connect redis sharding'
