def test_get_sharded_connections(self):
_fake_shards_for(self.conn, 'shard', 2, 2)
for i in xrange(10):
get_sharded_connection('shard', i, 2).sadd('foo', i)
s0 = redis_x.Redis(db=14).scard('foo')
s1 = redis_x.Redis(db=13).scard('foo')
self.assertTrue(s0 < 10)
self.assertTrue(s1 < 10)
self.assertEquals(s0 + s1, 10)
def test_sharded_follow_user_and_syndicate_status(self):
_fake_shards_for(self.conn, 'timelines', 8, 4)
_fake_shards_for(self.conn, 'followers', 4, 4)
sharded_followers.shards = 4
sharded_timelines['profile:1'].zadd('profile:1', 1, time.time())
for u2 in xrange(2, 11):
sharded_timelines['profile:%i' % u2].zadd('profile:%i' % u2, u2,
time.time() + u2)
follow_user(self.conn, 1, u2)
follow_user(self.conn, u2, 1)
allkeys = defaultdict(int)
for db in xrange(14, 10, -1):
c = redis_x.Redis(db=db)
for k in c.keys():
allkeys[k] += c.zcard(k)
for k, v in allkeys.iteritems():
part, _, owner = k.partition(':')
if part in ('following', 'followers', 'home'):
self.assertEquals(v, 9 if owner == '1' else 1)
elif part == 'profile':
self.assertEquals(v, 1)
self.assertEquals(
len(
sharded_zrangebyscore('followers', 4, 'followers:1', '0',
'inf', 100)), 9)
syndicate_status(1, {'11': time.time()})
self.assertEquals(
len(
sharded_zrangebyscore('timelines', 4, 'home:2', '0', 'inf',
100)), 2)
def get_redis_connection(component, wait=1):
key = 'config:redis:' + component
old_config = CONFIGS.get(key, object()) #A
config = get_config( #B
config_connection, 'redis', component, wait) #B
if config != old_config: #C
REDIS_CONNECTIONS[key] = redis_x.Redis(**config) #C
return REDIS_CONNECTIONS.get(key) #D
def call(*args, **kwargs): #E
old_config = CONFIGS.get(key, object()) #F
_config = get_config( #G
config_connection, 'redis', component, wait) #G
config = {}
for k, v in _config.iteritems(): #L
config[k.encode('utf-8')] = v #L
if config != old_config: #H
REDIS_CONNECTIONS[key] = redis_x.Redis(**config) #H
return function( #I
REDIS_CONNECTIONS.get(key), *args, **kwargs) #I
def test_count_visit(self):
shards = {'db': 13}, {'db': 14}
self.conn.set('config:redis:unique', json.dumps({'db': 15}))
for i in xrange(16):
self.conn.set('config:redis:unique:%s' % i,
json.dumps(shards[i & 1]))
for i in xrange(100):
count_visit(str(uuid.uuid4()))
base = 'unique:%s' % date.today().isoformat()
total = 0
for c in shards:
conn = redis_x.Redis(**c)
keys = conn.keys(base + ':*')
for k in keys:
cnt = conn.scard(k)
total += cnt
self.assertEquals(total, 100)
self.assertEquals(self.conn.get(base), '100')
def syndicate_status(uid, post, start=0, on_lists=False):
root = 'followers'
key = 'followers:%s' % uid
base = 'home:%s'
if on_lists:
root = 'list:out'
key = 'list:out:%s' % uid
base = 'list:statuses:%s'
followers = sharded_zrangebyscore(
root, #A
sharded_followers.shards,
key,
start,
'inf',
POSTS_PER_PASS) #A
to_send = defaultdict(list) #B
for follower, start in followers:
timeline = base % follower #C
shard = shard_key(
'timelines', #D
timeline,
sharded_timelines.shards,
2) #D
to_send[shard].append(timeline) #E
for timelines in to_send.itervalues():
pipe = sharded_timelines[timelines[0]].pipeline(False) #F
for timeline in timelines:
pipe.zadd(timeline, **post) #G
pipe.zremrangebyrank( #G
timeline, 0, -HOME_TIMELINE_SIZE - 1) #G
pipe.execute()
conn = redis_x.Redis()
if len(followers) >= POSTS_PER_PASS:
execute_later(conn, 'default', 'syndicate_status',
[uid, post, start, on_lists])
elif not on_lists:
execute_later(conn, 'default', 'syndicate_status',
[uid, post, 0, True])
def test_sharded_follow_user(self):
_fake_shards_for(self.conn, 'timelines', 8, 4)
sharded_timelines['profile:1'].zadd('profile:1', 1, time.time())
for u2 in xrange(2, 11):
sharded_timelines['profile:%i' % u2].zadd('profile:%i' % u2, u2,
time.time() + u2)
_follow_user(self.conn, 1, u2)
_follow_user(self.conn, u2, 1)
self.assertEquals(self.conn.zcard('followers:1'), 9)
self.assertEquals(self.conn.zcard('following:1'), 9)
self.assertEquals(sharded_timelines['home:1'].zcard('home:1'), 9)
for db in xrange(14, 10, -1):
self.assertTrue(len(redis_x.Redis(db=db).keys()) > 0)
for u2 in xrange(2, 11):
self.assertEquals(self.conn.zcard('followers:%i' % u2), 1)
self.assertEquals(self.conn.zcard('following:%i' % u2), 1)
self.assertEquals(
sharded_timelines['home:%i' % u2].zcard('home:%i' % u2), 1)
def setUp(self):
self.conn = redis_x.Redis(db=15)
self.conn.flushdb()
def setUp(self):
global config_connection
import redis_x
self.conn = config_connection = redis_x.Redis(db=15)
self.conn.flushdb()
def setUp(self):
import redis_x
self.conn = redis_x.Redis(db=15)
def setUp(self):
self.conn = redis_x.Redis(db=15)
self._flush()
global config_connection
config_connection = self.conn
self.conn.set('config:redis:test', json.dumps({'db': 15}))
def _flush(self):
self.conn.flushdb()
redis_x.Redis(db=14).flushdb()
redis_x.Redis(db=13).flushdb()
redis_x.Redis(db=12).flushdb()
redis_x.Redis(db=11).flushdb()