class AsyncLRUTest(AsyncTestCase):
def setUp(self):
super(AsyncLRUTest, self).setUp()
self.sr = redis.StrictRedis()
self.ar = AsyncStrictRedis()
self.m = MotorClient()
self.c = self.m.my_test
self.io_loop.run_sync(self.setup_coro)
@gen.coroutine
def setup_coro(self):
yield self.ar.flushdb()
yield self.m.drop_database('my_test')
self.redis_delegator = RedisDelegate(self.ar, self.c)
tag = Tags()
users = Users()
fblog = Fblog()
static_all_resources = StaticAllResources()
self.redis_delegator.add_collection(tag)
self.redis_delegator.add_collection(users)
self.redis_delegator.add_collection(fblog)
self.redis_delegator.add_collection(static_all_resources, 'all_resources')
@gen_test
def test_SetField_get_and_set(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
res = yield file_ids.get()
self.assertEqual(res, set())
file_ids_list = ['1','2','3']
yield file_ids.set(file_ids_list)
self.assertEqual(sr.scard(file_ids.key_name), 3)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, file_ids.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, file_ids.key_name) is not None)
res = yield file_ids.get()
self.assertEqual(res, set(file_ids_list))
self.stop()
@gen_test
def test_SetField_scard_sadd_srem_sismember(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
res = yield file_ids.scard()
self.assertEqual(res, 0)
res = yield file_ids.sadd('1')
self.assertEqual(res, 1)
res = yield file_ids.scard()
self.assertEqual(res, 1)
res = yield file_ids.sadd('1', '2', '3')
self.assertEqual(res, 2)
res = yield file_ids.sismember('2')
self.assertTrue(res)
res = yield file_ids.srem('2')
self.assertEqual(res, 1)
res = yield file_ids.sismember('2')
self.assertFalse(res)
res = yield file_ids.scard()
self.assertEqual(res, 2)
self.stop()
@gen_test
def test_SetField_make_data_in_redis(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
file_ids_list = ['1','2','3', '4']
yield self.c.tags.insert({'uid': 1, 'file_ids': file_ids_list})
res = yield file_ids.get()
self.assertEqual(res, set(file_ids_list))
self.stop()
@gen_test
def test_ListField_get_and_set(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
res = yield log.get()
self.assertEqual(res, list())
log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}]
yield log.set(log_list)
self.assertEqual(sr.llen(log.key_name), 3)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, log.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, log.key_name) is not None)
res = yield log.get()
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ListField_llen_ltrim(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'4':[2,3,4]}]
yield log.set(log_list)
res = yield log.llen()
self.assertEqual(res, 4)
res = yield log.ltrim(0, 5)
self.assertTrue(res)
res = yield log.llen()
self.assertEqual(res, 4)
res = yield log.ltrim(0, 1)
#self.assertEqual(res, 2)
res = yield log.llen()
self.assertEqual(res, 2)
res = yield log.get()
self.assertEqual(res, log_list[:2])
self.stop()
@gen_test
def test_ListField_lindex(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'4':[2,3,4]}]
yield log.set(log_list)
res = yield log.lindex(0)
self.assertEqual(res, log_list[0])
res = yield log.lindex(1)
self.assertEqual(res, log_list[1])
res = yield log.lindex(-1)
self.assertEqual(res, log_list[-1])
res = yield log.lindex(-5)
self.assertEqual(res, None)
res = yield log.lindex(5)
self.assertEqual(res, None)
self.stop()
@gen_test
def test_ListField_lrem(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'2': {'2':1}}, {'2': {'2':1}}, {'4':[2,3,4]}, {'2': {'2':1}}]
yield log.set(log_list)
res = yield log.lrem(1, {'2': {'2':1}})
res = yield log.get()
del log_list[1]
self.assertEqual(res, log_list)
res = yield log.lrem(-1, {'2': {'2':1}})
res = yield log.get()
del log_list[-1]
self.assertEqual(res, log_list)
res = yield log.lrem(0, {'2': {'2':1}})
res = yield log.get()
del log_list[2:4]
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ListField_lindex_rpush_lpop_lrange(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{'1':[1,2]},{'4':[2,3,4]}, {'2': {'2':1}},{'3':[1,2]}]
res = yield log.rpush({'1':[1,2]})
self.assertEqual(res, 1)
res = yield log.rpush({'4':[2,3,4]})
self.assertEqual(res, 2)
res = yield log.rpush({'2': {'2':1}},{'3':[1,2]})
self.assertEqual(res, 4)
res = yield log.lrange(0, -1)
self.assertEqual(res, log_list)
res = yield log.lrange(0, 5)
self.assertEqual(res, log_list)
res = yield log.lrange(0, -5)
self.assertEqual(res, [])
res = yield log.lrange(0, 2)
self.assertEqual(res, log_list[:3])
res = yield log.lrange(1, 3)
self.assertEqual(res, log_list[1:4])
self.stop()
@gen_test
def test_ListField_make_data_in_redis(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{'1':[1,2]},{'4':[2,3,4]}, {'2': {'2':1}},{'3':[1,2]}]
yield self.c.fblog.insert({'uid': 1, 'online_time': 1.0, 'log': log_list})
#res = yield self.c.fblog.find_one()
#print res
res = yield log.get()
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ZsetField_get_and_set(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 0}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield friends.set(friends_list)
self.assertEqual(sr.zcard(friends.key_name), 4)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, friends.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, friends.key_name) is not None)
res = yield friends.get()
self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar']))
self.stop()
@gen_test
def test_ZsetField_zscore(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield friends.set(friends_list)
res = yield friends.zscore(1)
self.assertEqual(res, 5)
res = yield friends.zscore(3)
self.assertEqual(res, 1)
res = yield friends.zscore(8)
self.assertEqual(res, None)
self.stop()
@gen_test
def test_ZsetField_zadd_zrem_zrange(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield friends.zadd(5, 1, 0, 2, 1, 3, 0, 4)
res = yield friends.zcard()
self.assertEqual(res, len(friends_list))
res = yield friends.zrange(0, -1)
self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar']))
res = yield friends.zrem(1)
self.assertEqual(res, 1)
res = yield friends.zrange(0, -1)
self.assertEqual(res, sorted(friends_list[1:], key = lambda x: x['isStar']))
res = yield friends.zrem(5)
self.assertEqual(res, 0)
res = yield friends.zrange(0, -1)
self.assertEqual(res, sorted(friends_list[1:], key = lambda x: x['isStar']))
res = yield friends.zrem(2, 3)
self.assertEqual(res, 2)
res = yield friends.zrange(0, -1)
self.assertEqual(res, sorted(friends_list[3:], key = lambda x: x['isStar']))
self.stop()
@gen_test
def test_ZsetField_make_data_in_redis(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz','friends': friends_list})
#res = yield self.c.fblog.find_one()
#print res
res = yield friends.get()
self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar']))
self.stop()
@gen_test
def test_set_None(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list})
doc = yield self.c.users.find_one({'uid': 1})
res = yield users.get('haslog')
self.assertEqual(res, 1)
res = yield users.get('test')
self.assertEqual(res, 'xyz')
res = yield users.find(1, ['test', 'xyz'])
self.assertEqual(res, {'haslog': 1, 'test': 'xyz'})
yield users.set('test', None)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.assertEqual(sr.hgetall(users._key), {'haslog': '1'})
doc1 = yield self.c.users.find_one({'uid': 1})
doc['test'] = None
self.assertEqual(doc, doc1)
self.stop()
@gen_test
def test_common_field_get_and_set(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list})
yield users.set('haslog', 0)
res = yield users.get('haslog')
self.assertEqual(res, 0)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.stop()
@gen_test
def test_unset_field(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list})
doc = yield self.c.users.find_one({'uid': 1})
res = yield users.get('test')
self.assertEqual(res, 'xyz')
yield users.delete(1, 'test')
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.assertEqual(sr.hgetall(users._key), {'haslog': '1'})
doc1 = yield self.c.users.find_one({'uid': 1})
doc.pop('test')
self.assertEqual(doc, doc1)
self.stop()
@gen_test
def test_static_find(self):
sr = self.sr
test_resources = [{'file_id': str(i), 'file_name': 'file' + str(i), 'main_type': i % 2, 'mtime': i, 'download_num': i} for i in range(15)]
test_resources_copy = copy.deepcopy(test_resources)
for t in test_resources_copy:
self.c.all_resources.insert(t)
res = yield self.redis_delegator.all_resources.find('0')
self.assertEqual(res, test_resources[0])
self.assertTrue(sr.exists('all_resources:0'))
res = yield self.redis_delegator.all_resources.find([str(_) for _ in range(6)])
res = sorted(res, key=lambda x: x['file_id'])
self.assertEqual(res, test_resources[:6])
res = yield self.redis_delegator.all_resources.find([str(_) for _ in range(7, 12)], {'main_type': 1})
res = sorted(res, key=lambda x: int(x['file_id']))
self.assertEqual(res, test_resources[7:12:2])
@gen_test
def test_find_update(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name))
self.assertFalse(sr.zrank(LRU_QUEUE, users.friends.key_name) is None)
update_doc = {'haslog': None}
yield users.update(1, update_doc)
#doc.update(update_doc)
doc.pop('haslog')
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
self.stop()
@gen_test
def test_write_back(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
#self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
#self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None)
yield users.write_back(1)
#self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
#self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
yield users.write_back(1, 'friends')
yield users.write_back(1)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
self.stop()
@gen_test(timeout=10)
def test_try_write_back(self):
def side_effect(*args, **kwargs):
kwargs['lock_timeout'] = 1
return acquire_lock_with_timeout(*args, **kwargs)
with mock.patch('redis_async_lru_scheduler.acquire_lock_with_timeout', side_effect=side_effect) as whate_ever:
#with mock.patch('redis_async_lru_scheduler.LOCK_TIMEOUT', 1) as whate_ever:
LOCK_TIMEOUT = 1
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}]
doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}
yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
time.sleep(LOCK_TIMEOUT+1)
res = yield self.redis_delegator.try_write_back(self.ar, 'users:1')
self.assertTrue(res)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, users.friends.key_name) is not None)
time.sleep(LOCK_TIMEOUT+1)
res = yield self.redis_delegator.try_write_back(self.ar, 'users:1.friends')
self.assertTrue(res)
res = yield self.redis_delegator.try_write_back(self.ar, 'users:1')
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, users.friends.key_name) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
self.stop()
def get_app(self):
db = MotorClient(os.getenv('DB_HOST', 'mongodb://localhost:27017'), )
loop = asyncio.get_event_loop()
loop.run_until_complete(db.drop_database(os.getenv('DB_NAME')))
return application
class TestTornadoMotorAcl:
io_loop = IOLoop.instance()
@gen_test
def setup(self):
print "-setup"
self.client = MotorClient()
#print "c, ", self.client
self.db = self.client['test_database']
self.permissions = MotorCollection(self.db, 'acl_permissions')
self.groups = MotorCollection(self.db, 'acl_groups')
self.resources = MotorCollection(self.db, 'acl_resources')
self.users = MotorCollection(self.db, 'users')
self.user_ids = yield self.users.insert([{'name':'burger'},{'name':'paul'}])
self.user_ids = [x for x in self.user_ids]
self.admin_user = User("burger", self.user_ids[0])
self.user_user = User("paul", self.user_ids[1])
self.perm_ids = yield self.permissions.insert([x.to_primitive() for x in [
Permission({"name":"read"}),
Permission({"name":"write"}),
Permission({"name":"update"}),
Permission({"name":"delete"}),
]])
self.res_ids = yield self.resources.insert([x.to_primitive() for x in [
Resource({'name':'own data'}),
Resource({'name':'others data'}),
Resource({'name':'all data'})
]])
self.group_ids = yield self.groups.insert([x.to_primitive() for x in [
Group({"name":"brugere",
"permissions":[
ResourcePermissionPair({"resource":"own data",
"permissions":[
"read", "write"
]
})
],
"members":[self.user_ids[0], self.user_ids[1]],
}),
Group({"name":"super brugere",
"permissions":[
ResourcePermissionPair({"resource":'own data',
"permissions":[
"read", "write", "update"
]
}),
ResourcePermissionPair({"resource":"others data",
"permissions":[
"read"
]
})
],
"members":[self.user_ids[0]],
})
]])
doc = yield self.groups.find_one({"name": "brugere"})
members =doc['members']
#print "members:", members
#print "users", self.user_ids
# assert self.admin_user._id in members
assert True
@gen_test
def teardown(self):
self.db = None
self.client.drop_database('test_database')
@gen_test
def test_setup(self):
assert len(self.user_ids) == 2
assert len(self.perm_ids) == 4
assert len(self.res_ids) == 3
assert len(self.group_ids) == 2
@gen_test
def test_that_user_can_read_own_data(self):
handler = OwnDataHandler(self.db, self.user_user)
yield handler.post()
assert handler.status == 200
#assert handler.status == 200
@gen_test
def test_that_user_can_not_read_others_data(self):
handler = OthersDataHandler(self.db, self.user_user)
yield handler.post()
assert handler.status == 403
@gen_test
def test_that_admin_can_read_own_data(self):
handler = OwnDataHandler(self.db, self.admin_user)
yield handler.post()
assert handler.status == 200
#assert handler.status == 200
@gen_test
def test_that_admin_can_read_others_data(self):
handler = OthersDataHandler(self.db, self.admin_user)
yield handler.post()
assert handler.status == 200
@gen_test
def test_that_admin_can_read_others_and_own_data(self):
handler = OwnAndOthersDataHandler(self.db, self.admin_user)
yield handler.post()
assert handler.status == 200
class FileNameSearcherTest(AsyncTestCase):
def setUp(self):
super(FileNameSearcherTest, self).setUp()
self.sr = redis.StrictRedis()
self.ar = AsyncStrictRedis()
self.m = MotorClient()
self.c = self.m.fbt
self.patcher1 = mock.patch('redis_handler.RedisHandler.redis_client', return_value=mock_redis)
self.patcher1.start()
'''
self.patcher2 = mock.patch("pymongo.MongoReplicaSetClient", return_value=sync_db)
self.patcher3 = mock.patch("motor.MotorReplicaSetClient", return_value=db)
#self.patcher4 = mock.patch("async_redis.asyncRedis.AsyncStrictRedis", spec=AsyncStrictRedis, return_value=AsyncStrictRedis())
self.patcher2.start()
self.patcher3.start()
#self.patcher4.start()
'''
self.io_loop.run_sync(self.setup_coro)
def tearDown(self):
self.patcher1.stop()
'''
self.patcher2.stop()
self.patcher3.stop()
#self.patcher4.stop()
'''
@gen.coroutine
def setup_coro(self):
yield self.ar.flushdb()
yield self.m.drop_database('fbt')
yield self.ar.sadd(USER_IP_CACHE_SET_KEY, *[1, 2, 3, 4])
yield self.c.users.insert({'uid': 1, 'friends': [{'uid': 2, 'isStar': 1}, {'uid': 3, 'isStar': 1}]})
yield self.c.resources_of_user.insert({'uid': 2, 'file_ids': [str(_) for _ in range(10)]})
yield self.c.resources_of_user.insert({'uid': 3, 'file_ids': [str(_) for _ in range(5, 15)]})
yield self.c.resources_of_user.insert({'uid': 4, 'file_ids': [str(_) for _ in range(15, 20)]})
for i, file_name in enumerate(test_filename):
file_id = str(i)
resource = {'file_id': file_id, 'mtime': test_mtime[i], 'download_num': test_download_num[i],
'public': 1, 'main_type': test_main_type[i], 'file_name': file_name}
test_resources.append(resource)
yield self.c.all_resources.insert(resource)
yield FileNameSearcher().file_id_add_title(file_id, file_name)
#yield [self.c.all_resources.insert(resource), FileNameSearcher().file_id_add_title(file_id, file_name)]
@gen_test(timeout=3)
def test_all(self):
search_key = test_keyword[3]
res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4)
res_list = list()
for i, v in enumerate(test_filename):
if search_key in v:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4, sort="mtime")
#print res_list[0:3], len(res_list)
#print res, search_key
self.assertEqual(res, (len(res_list), res_list[0:4]))
#print res
res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True)
res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4, sort="download_num")
self.assertEqual(res, (len(res_list), res_list[0:4]))
res = yield FileNameSearcher().query_file_ids_by_file_name('a b', 1, 4, sort="download_num")
self.assertEqual(res, (0, []))
res = yield FileNameSearcher().query_file_ids_by_file_name('xxxxxxxx', 1, 4, sort="download_num")
self.assertEqual(res, (0, []))
# test for query_file_ids_by_file_name_private
search_key = test_keyword[1]
print 'search_key', search_key
res_list = list()
for i, v in enumerate(test_filename[:15]):
if search_key in v:
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res = yield FileNameSearcher().query_file_ids_by_file_name_private(1, search_key, 1, 4, sort="mtime")
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
self.assertEqual(res, (len(res_list), res_list[0:4]))
# test for query_file_ids_by_file_name_private
res_list = list()
for i, v in enumerate(test_filename[:15]):
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res = yield FileNameSearcher().get_private_resources(1, 1, 4, sort="mtime")
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
self.assertEqual(res, (15, res_list[0:4]))
# test for query_file_ids_by_file_name_private
print test_main_type
main_type = test_main_type[0]
res_list = list()
for i, v in enumerate(test_filename[:15]):
if main_type == test_resources[i]['main_type']:
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True)
res = yield FileNameSearcher().get_private_resources_by_type(1, main_type, 1, 4, sort="download_num")
print len(res_list)
self.assertEqual(res, (len(res_list), res_list[0:4]))
##################### test_basic ###################
# sync
test_filename1 = ' '.join(test_keyword[:3])
test_file_id = '21'
filename_searcher = FileNameSearcher()
rdb = filename_searcher.db
mongo = filename_searcher.mongoDB
filename_searcher.file_id_add_title_sync(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertTrue(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertTrue(test_file_id in res['file_ids'])
filename_searcher.remove_file_id_sync(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertFalse(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertFalse(test_file_id in res['file_ids'])
# async
yield filename_searcher.file_id_add_title(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertTrue(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertTrue(test_file_id in res['file_ids'])
yield filename_searcher.remove_file_id(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertFalse(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertFalse(test_file_id in res['file_ids'])
# test init
filename_searcher.drop()
filename_searcher.init_from_mongo()
key_fileids_dict = defaultdict(list)
for i, f in enumerate(test_filename):
i = str(i)
for k in f.split():
key_fileids_dict[k].append(i)
for k in test_keyword:
key_name = make_key_for_keyword(k)
self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k]))
for res in mongo.key_fileids.find():
self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']]))
# test add
filename_searcher.scan_from_mongo(0)
for k in test_keyword:
key_name = make_key_for_keyword(k)
self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k]))
for res in mongo.key_fileids.find():
self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']]))
print 'add test'
last_mtime = max(test_mtime)
t_mtime = last_mtime + 1
resource = {'file_id': '21', 'mtime': t_mtime, 'download_num': 11,
'public': 1, 'main_type': 1, 'file_name': 'test'}
yield self.c.all_resources.insert(resource)
res = filename_searcher.scan_from_mongo(last_mtime)
self.assertEqual(t_mtime, res)
self.stop()
class FileNameSearcherTest(AsyncTestCase):
def setUp(self):
super(FileNameSearcherTest, self).setUp()
self.sr = redis.StrictRedis()
self.ar = AsyncStrictRedis()
self.m = MotorClient()
self.c = self.m.fbt
self.patcher1 = mock.patch('redis_handler.RedisHandler.redis_client',
return_value=mock_redis)
self.patcher1.start()
'''
self.patcher2 = mock.patch("pymongo.MongoReplicaSetClient", return_value=sync_db)
self.patcher3 = mock.patch("motor.MotorReplicaSetClient", return_value=db)
#self.patcher4 = mock.patch("async_redis.asyncRedis.AsyncStrictRedis", spec=AsyncStrictRedis, return_value=AsyncStrictRedis())
self.patcher2.start()
self.patcher3.start()
#self.patcher4.start()
'''
self.io_loop.run_sync(self.setup_coro)
def tearDown(self):
self.patcher1.stop()
'''
self.patcher2.stop()
self.patcher3.stop()
#self.patcher4.stop()
'''
@gen.coroutine
def setup_coro(self):
yield self.ar.flushdb()
yield self.m.drop_database('fbt')
yield self.ar.sadd(USER_IP_CACHE_SET_KEY, *[1, 2, 3, 4])
yield self.c.users.insert({
'uid':
1,
'friends': [{
'uid': 2,
'isStar': 1
}, {
'uid': 3,
'isStar': 1
}]
})
yield self.c.resources_of_user.insert({
'uid':
2,
'file_ids': [str(_) for _ in range(10)]
})
yield self.c.resources_of_user.insert({
'uid':
3,
'file_ids': [str(_) for _ in range(5, 15)]
})
yield self.c.resources_of_user.insert({
'uid':
4,
'file_ids': [str(_) for _ in range(15, 20)]
})
for i, file_name in enumerate(test_filename):
file_id = str(i)
resource = {
'file_id': file_id,
'mtime': test_mtime[i],
'download_num': test_download_num[i],
'public': 1,
'main_type': test_main_type[i],
'file_name': file_name
}
test_resources.append(resource)
yield self.c.all_resources.insert(resource)
yield FileNameSearcher().file_id_add_title(file_id, file_name)
#yield [self.c.all_resources.insert(resource), FileNameSearcher().file_id_add_title(file_id, file_name)]
@gen_test(timeout=3)
def test_all(self):
search_key = test_keyword[3]
res = yield FileNameSearcher().query_file_ids_by_file_name(
search_key, 1, 4)
res_list = list()
for i, v in enumerate(test_filename):
if search_key in v:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
res = yield FileNameSearcher().query_file_ids_by_file_name(
search_key, 1, 4, sort="mtime")
#print res_list[0:3], len(res_list)
#print res, search_key
self.assertEqual(res, (len(res_list), res_list[0:4]))
#print res
res_list = sorted(res_list,
key=lambda x: x["download_num"],
reverse=True)
res = yield FileNameSearcher().query_file_ids_by_file_name(
search_key, 1, 4, sort="download_num")
self.assertEqual(res, (len(res_list), res_list[0:4]))
res = yield FileNameSearcher().query_file_ids_by_file_name(
'a b', 1, 4, sort="download_num")
self.assertEqual(res, (0, []))
res = yield FileNameSearcher().query_file_ids_by_file_name(
'xxxxxxxx', 1, 4, sort="download_num")
self.assertEqual(res, (0, []))
# test for query_file_ids_by_file_name_private
search_key = test_keyword[1]
print 'search_key', search_key
res_list = list()
for i, v in enumerate(test_filename[:15]):
if search_key in v:
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res = yield FileNameSearcher().query_file_ids_by_file_name_private(
1, search_key, 1, 4, sort="mtime")
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
self.assertEqual(res, (len(res_list), res_list[0:4]))
# test for query_file_ids_by_file_name_private
res_list = list()
for i, v in enumerate(test_filename[:15]):
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res = yield FileNameSearcher().get_private_resources(1,
1,
4,
sort="mtime")
res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True)
self.assertEqual(res, (15, res_list[0:4]))
# test for query_file_ids_by_file_name_private
print test_main_type
main_type = test_main_type[0]
res_list = list()
for i, v in enumerate(test_filename[:15]):
if main_type == test_resources[i]['main_type']:
if '_id' in test_resources[i]:
test_resources[i].pop('_id')
res_list.append(test_resources[i])
res_list = sorted(res_list,
key=lambda x: x["download_num"],
reverse=True)
res = yield FileNameSearcher().get_private_resources_by_type(
1, main_type, 1, 4, sort="download_num")
print len(res_list)
self.assertEqual(res, (len(res_list), res_list[0:4]))
##################### test_basic ###################
# sync
test_filename1 = ' '.join(test_keyword[:3])
test_file_id = '21'
filename_searcher = FileNameSearcher()
rdb = filename_searcher.db
mongo = filename_searcher.mongoDB
filename_searcher.file_id_add_title_sync(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertTrue(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertTrue(test_file_id in res['file_ids'])
filename_searcher.remove_file_id_sync(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertFalse(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertFalse(test_file_id in res['file_ids'])
# async
yield filename_searcher.file_id_add_title(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertTrue(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertTrue(test_file_id in res['file_ids'])
yield filename_searcher.remove_file_id(test_file_id, test_filename1)
for k in test_keyword[:3]:
key_name = make_key_for_keyword(k)
self.assertFalse(rdb.sismember(key_name, test_file_id))
for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}):
self.assertFalse(test_file_id in res['file_ids'])
# test init
filename_searcher.drop()
filename_searcher.init_from_mongo()
key_fileids_dict = defaultdict(list)
for i, f in enumerate(test_filename):
i = str(i)
for k in f.split():
key_fileids_dict[k].append(i)
for k in test_keyword:
key_name = make_key_for_keyword(k)
self.assertEqual(sorted(rdb.smembers(key_name)),
sorted(key_fileids_dict[k]))
for res in mongo.key_fileids.find():
self.assertEqual(sorted(res['file_ids']),
sorted(key_fileids_dict[res['key']]))
# test add
filename_searcher.scan_from_mongo(0)
for k in test_keyword:
key_name = make_key_for_keyword(k)
self.assertEqual(sorted(rdb.smembers(key_name)),
sorted(key_fileids_dict[k]))
for res in mongo.key_fileids.find():
self.assertEqual(sorted(res['file_ids']),
sorted(key_fileids_dict[res['key']]))
print 'add test'
last_mtime = max(test_mtime)
t_mtime = last_mtime + 1
resource = {
'file_id': '21',
'mtime': t_mtime,
'download_num': 11,
'public': 1,
'main_type': 1,
'file_name': 'test'
}
yield self.c.all_resources.insert(resource)
res = filename_searcher.scan_from_mongo(last_mtime)
self.assertEqual(t_mtime, res)
self.stop()
class AsyncLRUTest(AsyncTestCase):
def setUp(self):
super(AsyncLRUTest, self).setUp()
self.sr = redis.StrictRedis()
self.ar = AsyncStrictRedis()
self.m = MotorClient()
self.c = self.m.my_test
self.io_loop.run_sync(self.setup_coro)
@gen.coroutine
def setup_coro(self):
yield self.ar.flushdb()
yield self.m.drop_database('my_test')
self.redis_delegator = RedisDelegate(self.ar, self.c)
tag = Tags()
users = Users()
fblog = Fblog()
static_all_resources = StaticAllResources()
self.redis_delegator.add_collection(tag)
self.redis_delegator.add_collection(users)
self.redis_delegator.add_collection(fblog)
self.redis_delegator.add_collection(static_all_resources,
'all_resources')
@gen_test
def test_SetField_get_and_set(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
res = yield file_ids.get()
self.assertEqual(res, set())
file_ids_list = ['1', '2', '3']
yield file_ids.set(file_ids_list)
self.assertEqual(sr.scard(file_ids.key_name), 3)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, file_ids.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, file_ids.key_name) is not None)
res = yield file_ids.get()
self.assertEqual(res, set(file_ids_list))
self.stop()
@gen_test
def test_SetField_scard_sadd_srem_sismember(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
res = yield file_ids.scard()
self.assertEqual(res, 0)
res = yield file_ids.sadd('1')
self.assertEqual(res, 1)
res = yield file_ids.scard()
self.assertEqual(res, 1)
res = yield file_ids.sadd('1', '2', '3')
self.assertEqual(res, 2)
res = yield file_ids.sismember('2')
self.assertTrue(res)
res = yield file_ids.srem('2')
self.assertEqual(res, 1)
res = yield file_ids.sismember('2')
self.assertFalse(res)
res = yield file_ids.scard()
self.assertEqual(res, 2)
self.stop()
@gen_test
def test_SetField_make_data_in_redis(self):
file_ids = self.redis_delegator.tags(1).file_ids
sr = self.sr
file_ids_list = ['1', '2', '3', '4']
yield self.c.tags.insert({'uid': 1, 'file_ids': file_ids_list})
res = yield file_ids.get()
self.assertEqual(res, set(file_ids_list))
self.stop()
@gen_test
def test_ListField_get_and_set(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
res = yield log.get()
self.assertEqual(res, list())
log_list = [{'1': [1, 2]}, {'2': {'2': 1}}, {'3': [1, 2]}]
yield log.set(log_list)
self.assertEqual(sr.llen(log.key_name), 3)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, log.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, log.key_name) is not None)
res = yield log.get()
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ListField_llen_ltrim(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{
'1': [1, 2]
}, {
'2': {
'2': 1
}
}, {
'3': [1, 2]
}, {
'4': [2, 3, 4]
}]
yield log.set(log_list)
res = yield log.llen()
self.assertEqual(res, 4)
res = yield log.ltrim(0, 5)
self.assertTrue(res)
res = yield log.llen()
self.assertEqual(res, 4)
res = yield log.ltrim(0, 1)
#self.assertEqual(res, 2)
res = yield log.llen()
self.assertEqual(res, 2)
res = yield log.get()
self.assertEqual(res, log_list[:2])
self.stop()
@gen_test
def test_ListField_lindex(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{
'1': [1, 2]
}, {
'2': {
'2': 1
}
}, {
'3': [1, 2]
}, {
'4': [2, 3, 4]
}]
yield log.set(log_list)
res = yield log.lindex(0)
self.assertEqual(res, log_list[0])
res = yield log.lindex(1)
self.assertEqual(res, log_list[1])
res = yield log.lindex(-1)
self.assertEqual(res, log_list[-1])
res = yield log.lindex(-5)
self.assertEqual(res, None)
res = yield log.lindex(5)
self.assertEqual(res, None)
self.stop()
@gen_test
def test_ListField_lrem(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{
'1': [1, 2]
}, {
'2': {
'2': 1
}
}, {
'3': [1, 2]
}, {
'2': {
'2': 1
}
}, {
'2': {
'2': 1
}
}, {
'4': [2, 3, 4]
}, {
'2': {
'2': 1
}
}]
yield log.set(log_list)
res = yield log.lrem(1, {'2': {'2': 1}})
res = yield log.get()
del log_list[1]
self.assertEqual(res, log_list)
res = yield log.lrem(-1, {'2': {'2': 1}})
res = yield log.get()
del log_list[-1]
self.assertEqual(res, log_list)
res = yield log.lrem(0, {'2': {'2': 1}})
res = yield log.get()
del log_list[2:4]
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ListField_lindex_rpush_lpop_lrange(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{
'1': [1, 2]
}, {
'4': [2, 3, 4]
}, {
'2': {
'2': 1
}
}, {
'3': [1, 2]
}]
res = yield log.rpush({'1': [1, 2]})
self.assertEqual(res, 1)
res = yield log.rpush({'4': [2, 3, 4]})
self.assertEqual(res, 2)
res = yield log.rpush({'2': {'2': 1}}, {'3': [1, 2]})
self.assertEqual(res, 4)
res = yield log.lrange(0, -1)
self.assertEqual(res, log_list)
res = yield log.lrange(0, 5)
self.assertEqual(res, log_list)
res = yield log.lrange(0, -5)
self.assertEqual(res, [])
res = yield log.lrange(0, 2)
self.assertEqual(res, log_list[:3])
res = yield log.lrange(1, 3)
self.assertEqual(res, log_list[1:4])
self.stop()
@gen_test
def test_ListField_make_data_in_redis(self):
log = self.redis_delegator.fblog(1).log
sr = self.sr
log_list = [{
'1': [1, 2]
}, {
'4': [2, 3, 4]
}, {
'2': {
'2': 1
}
}, {
'3': [1, 2]
}]
yield self.c.fblog.insert({
'uid': 1,
'online_time': 1.0,
'log': log_list
})
#res = yield self.c.fblog.find_one()
#print res
res = yield log.get()
self.assertEqual(res, log_list)
self.stop()
@gen_test
def test_ZsetField_get_and_set(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 0
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield friends.set(friends_list)
self.assertEqual(sr.zcard(friends.key_name), 4)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, friends.key_name))
self.assertTrue(sr.zrank(LRU_QUEUE, friends.key_name) is not None)
res = yield friends.get()
self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar']))
self.stop()
@gen_test
def test_ZsetField_zscore(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield friends.set(friends_list)
res = yield friends.zscore(1)
self.assertEqual(res, 5)
res = yield friends.zscore(3)
self.assertEqual(res, 1)
res = yield friends.zscore(8)
self.assertEqual(res, None)
self.stop()
@gen_test
def test_ZsetField_zadd_zrem_zrange(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield friends.zadd(5, 1, 0, 2, 1, 3, 0, 4)
res = yield friends.zcard()
self.assertEqual(res, len(friends_list))
res = yield friends.zrange(0, -1)
self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar']))
res = yield friends.zrem(1)
self.assertEqual(res, 1)
res = yield friends.zrange(0, -1)
self.assertEqual(res,
sorted(friends_list[1:], key=lambda x: x['isStar']))
res = yield friends.zrem(5)
self.assertEqual(res, 0)
res = yield friends.zrange(0, -1)
self.assertEqual(res,
sorted(friends_list[1:], key=lambda x: x['isStar']))
res = yield friends.zrem(2, 3)
self.assertEqual(res, 2)
res = yield friends.zrange(0, -1)
self.assertEqual(res,
sorted(friends_list[3:], key=lambda x: x['isStar']))
self.stop()
@gen_test
def test_ZsetField_make_data_in_redis(self):
friends = self.redis_delegator.users(1).friends
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield self.c.users.insert({
'uid': 1,
'haslog': 1,
'test': 'xyz',
'friends': friends_list
})
#res = yield self.c.fblog.find_one()
#print res
res = yield friends.get()
self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar']))
self.stop()
@gen_test
def test_set_None(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield self.c.users.insert({
'uid': 1,
'haslog': 1,
'test': 'xyz',
'friends': friends_list
})
doc = yield self.c.users.find_one({'uid': 1})
res = yield users.get('haslog')
self.assertEqual(res, 1)
res = yield users.get('test')
self.assertEqual(res, 'xyz')
res = yield users.find(1, ['test', 'xyz'])
self.assertEqual(res, {'haslog': 1, 'test': 'xyz'})
yield users.set('test', None)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.assertEqual(sr.hgetall(users._key), {'haslog': '1'})
doc1 = yield self.c.users.find_one({'uid': 1})
doc['test'] = None
self.assertEqual(doc, doc1)
self.stop()
@gen_test
def test_common_field_get_and_set(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield self.c.users.insert({
'uid': 1,
'haslog': 1,
'test': 'xyz',
'friends': friends_list
})
yield users.set('haslog', 0)
res = yield users.get('haslog')
self.assertEqual(res, 0)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.stop()
@gen_test
def test_unset_field(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
yield self.c.users.insert({
'uid': 1,
'haslog': 1,
'test': 'xyz',
'friends': friends_list
})
doc = yield self.c.users.find_one({'uid': 1})
res = yield users.get('test')
self.assertEqual(res, 'xyz')
yield users.delete(1, 'test')
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
self.assertEqual(sr.hgetall(users._key), {'haslog': '1'})
doc1 = yield self.c.users.find_one({'uid': 1})
doc.pop('test')
self.assertEqual(doc, doc1)
self.stop()
@gen_test
def test_static_find(self):
sr = self.sr
test_resources = [{
'file_id': str(i),
'file_name': 'file' + str(i),
'main_type': i % 2,
'mtime': i,
'download_num': i
} for i in range(15)]
test_resources_copy = copy.deepcopy(test_resources)
for t in test_resources_copy:
self.c.all_resources.insert(t)
res = yield self.redis_delegator.all_resources.find('0')
self.assertEqual(res, test_resources[0])
self.assertTrue(sr.exists('all_resources:0'))
res = yield self.redis_delegator.all_resources.find(
[str(_) for _ in range(6)])
res = sorted(res, key=lambda x: x['file_id'])
self.assertEqual(res, test_resources[:6])
res = yield self.redis_delegator.all_resources.find(
[str(_) for _ in range(7, 12)], {'main_type': 1})
res = sorted(res, key=lambda x: int(x['file_id']))
self.assertEqual(res, test_resources[7:12:2])
@gen_test
def test_find_update(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
doc = {
'haslog': 1,
'test': 'xyz',
'friends': sorted(friends_list, key=lambda x: x['isStar'])
}
yield self.c.users.insert({
'uid':
1,
'haslog':
1,
'test':
'xyz',
'friends':
sorted(friends_list, key=lambda x: x['isStar'])
})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET,
users.friends.key_name))
self.assertFalse(sr.zrank(LRU_QUEUE, users.friends.key_name) is None)
update_doc = {'haslog': None}
yield users.update(1, update_doc)
#doc.update(update_doc)
doc.pop('haslog')
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
self.stop()
@gen_test
def test_write_back(self):
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
doc = {
'haslog': 1,
'test': 'xyz',
'friends': sorted(friends_list, key=lambda x: x['isStar'])
}
yield self.c.users.insert({
'uid':
1,
'haslog':
1,
'test':
'xyz',
'friends':
sorted(friends_list, key=lambda x: x['isStar'])
})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
res = yield users.find(1)
self.assertEqual(res, doc)
#self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
#self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None)
yield users.write_back(1)
#self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
#self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
yield users.write_back(1, 'friends')
yield users.write_back(1)
doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0})
self.assertEqual(doc, doc1)
self.stop()
@gen_test(timeout=10)
def test_try_write_back(self):
def side_effect(*args, **kwargs):
kwargs['lock_timeout'] = 1
return acquire_lock_with_timeout(*args, **kwargs)
with mock.patch('redis_async_lru_scheduler.acquire_lock_with_timeout',
side_effect=side_effect) as whate_ever:
#with mock.patch('redis_async_lru_scheduler.LOCK_TIMEOUT', 1) as whate_ever:
LOCK_TIMEOUT = 1
users = self.redis_delegator.users(1)
sr = self.sr
friends_list = [{
'uid': 1,
'isStar': 5
}, {
'uid': 2,
'isStar': 0
}, {
'uid': 3,
'isStar': 1
}, {
'uid': 4,
'isStar': 0
}]
doc = {
'haslog': 1,
'test': 'xyz',
'friends': sorted(friends_list, key=lambda x: x['isStar'])
}
yield self.c.users.insert({
'uid':
1,
'haslog':
1,
'test':
'xyz',
'friends':
sorted(friends_list, key=lambda x: x['isStar'])
})
#doc = yield self.c.users.find_one({'uid': 1})
res = yield users.find(1)
self.assertEqual(res, doc)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
update_doc = {'test': '123', 'abc': 'aaa'}
yield users.update(1, update_doc)
doc.update(update_doc)
self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None)
time.sleep(LOCK_TIMEOUT + 1)
res = yield self.redis_delegator.try_write_back(self.ar, 'users:1')
self.assertTrue(res)
self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key))
self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {
'uid': 0,
'_id': 0
})
self.assertEqual(doc, doc1)
update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]}
yield users.update(1, update_doc)
doc.update(update_doc)
self.assertTrue(
sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name))
self.assertTrue(
sr.zrank(LRU_QUEUE, users.friends.key_name) is not None)
time.sleep(LOCK_TIMEOUT + 1)
res = yield self.redis_delegator.try_write_back(
self.ar, 'users:1.friends')
self.assertTrue(res)
res = yield self.redis_delegator.try_write_back(self.ar, 'users:1')
self.assertFalse(
sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name))
self.assertTrue(
sr.zrank(LRU_QUEUE, users.friends.key_name) is None)
doc1 = yield self.c.users.find_one({'uid': 1}, {
'uid': 0,
'_id': 0
})
self.assertEqual(doc, doc1)
self.stop()
