class AsyncLRUTest(AsyncTestCase): def setUp(self): super(AsyncLRUTest, self).setUp() self.sr = redis.StrictRedis() self.ar = AsyncStrictRedis() self.m = MotorClient() self.c = self.m.my_test self.io_loop.run_sync(self.setup_coro) @gen.coroutine def setup_coro(self): yield self.ar.flushdb() yield self.m.drop_database('my_test') self.redis_delegator = RedisDelegate(self.ar, self.c) tag = Tags() users = Users() fblog = Fblog() static_all_resources = StaticAllResources() self.redis_delegator.add_collection(tag) self.redis_delegator.add_collection(users) self.redis_delegator.add_collection(fblog) self.redis_delegator.add_collection(static_all_resources, 'all_resources') @gen_test def test_SetField_get_and_set(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr res = yield file_ids.get() self.assertEqual(res, set()) file_ids_list = ['1','2','3'] yield file_ids.set(file_ids_list) self.assertEqual(sr.scard(file_ids.key_name), 3) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, file_ids.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, file_ids.key_name) is not None) res = yield file_ids.get() self.assertEqual(res, set(file_ids_list)) self.stop() @gen_test def test_SetField_scard_sadd_srem_sismember(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr res = yield file_ids.scard() self.assertEqual(res, 0) res = yield file_ids.sadd('1') self.assertEqual(res, 1) res = yield file_ids.scard() self.assertEqual(res, 1) res = yield file_ids.sadd('1', '2', '3') self.assertEqual(res, 2) res = yield file_ids.sismember('2') self.assertTrue(res) res = yield file_ids.srem('2') self.assertEqual(res, 1) res = yield file_ids.sismember('2') self.assertFalse(res) res = yield file_ids.scard() self.assertEqual(res, 2) self.stop() @gen_test def test_SetField_make_data_in_redis(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr file_ids_list = ['1','2','3', '4'] yield self.c.tags.insert({'uid': 1, 'file_ids': file_ids_list}) res = yield file_ids.get() self.assertEqual(res, set(file_ids_list)) self.stop() @gen_test def test_ListField_get_and_set(self): log = self.redis_delegator.fblog(1).log sr = self.sr res = yield log.get() self.assertEqual(res, list()) log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}] yield log.set(log_list) self.assertEqual(sr.llen(log.key_name), 3) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, log.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, log.key_name) is not None) res = yield log.get() self.assertEqual(res, log_list) self.stop() @gen_test def test_ListField_llen_ltrim(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'4':[2,3,4]}] yield log.set(log_list) res = yield log.llen() self.assertEqual(res, 4) res = yield log.ltrim(0, 5) self.assertTrue(res) res = yield log.llen() self.assertEqual(res, 4) res = yield log.ltrim(0, 1) #self.assertEqual(res, 2) res = yield log.llen() self.assertEqual(res, 2) res = yield log.get() self.assertEqual(res, log_list[:2]) self.stop() @gen_test def test_ListField_lindex(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'4':[2,3,4]}] yield log.set(log_list) res = yield log.lindex(0) self.assertEqual(res, log_list[0]) res = yield log.lindex(1) self.assertEqual(res, log_list[1]) res = yield log.lindex(-1) self.assertEqual(res, log_list[-1]) res = yield log.lindex(-5) self.assertEqual(res, None) res = yield log.lindex(5) self.assertEqual(res, None) self.stop() @gen_test def test_ListField_lrem(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{'1':[1,2]},{'2': {'2':1}},{'3':[1,2]}, {'2': {'2':1}}, {'2': {'2':1}}, {'4':[2,3,4]}, {'2': {'2':1}}] yield log.set(log_list) res = yield log.lrem(1, {'2': {'2':1}}) res = yield log.get() del log_list[1] self.assertEqual(res, log_list) res = yield log.lrem(-1, {'2': {'2':1}}) res = yield log.get() del log_list[-1] self.assertEqual(res, log_list) res = yield log.lrem(0, {'2': {'2':1}}) res = yield log.get() del log_list[2:4] self.assertEqual(res, log_list) self.stop() @gen_test def test_ListField_lindex_rpush_lpop_lrange(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{'1':[1,2]},{'4':[2,3,4]}, {'2': {'2':1}},{'3':[1,2]}] res = yield log.rpush({'1':[1,2]}) self.assertEqual(res, 1) res = yield log.rpush({'4':[2,3,4]}) self.assertEqual(res, 2) res = yield log.rpush({'2': {'2':1}},{'3':[1,2]}) self.assertEqual(res, 4) res = yield log.lrange(0, -1) self.assertEqual(res, log_list) res = yield log.lrange(0, 5) self.assertEqual(res, log_list) res = yield log.lrange(0, -5) self.assertEqual(res, []) res = yield log.lrange(0, 2) self.assertEqual(res, log_list[:3]) res = yield log.lrange(1, 3) self.assertEqual(res, log_list[1:4]) self.stop() @gen_test def test_ListField_make_data_in_redis(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{'1':[1,2]},{'4':[2,3,4]}, {'2': {'2':1}},{'3':[1,2]}] yield self.c.fblog.insert({'uid': 1, 'online_time': 1.0, 'log': log_list}) #res = yield self.c.fblog.find_one() #print res res = yield log.get() self.assertEqual(res, log_list) self.stop() @gen_test def test_ZsetField_get_and_set(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{'uid': 1, 'isStar': 0}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield friends.set(friends_list) self.assertEqual(sr.zcard(friends.key_name), 4) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, friends.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, friends.key_name) is not None) res = yield friends.get() self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar'])) self.stop() @gen_test def test_ZsetField_zscore(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield friends.set(friends_list) res = yield friends.zscore(1) self.assertEqual(res, 5) res = yield friends.zscore(3) self.assertEqual(res, 1) res = yield friends.zscore(8) self.assertEqual(res, None) self.stop() @gen_test def test_ZsetField_zadd_zrem_zrange(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield friends.zadd(5, 1, 0, 2, 1, 3, 0, 4) res = yield friends.zcard() self.assertEqual(res, len(friends_list)) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar'])) res = yield friends.zrem(1) self.assertEqual(res, 1) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[1:], key = lambda x: x['isStar'])) res = yield friends.zrem(5) self.assertEqual(res, 0) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[1:], key = lambda x: x['isStar'])) res = yield friends.zrem(2, 3) self.assertEqual(res, 2) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[3:], key = lambda x: x['isStar'])) self.stop() @gen_test def test_ZsetField_make_data_in_redis(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz','friends': friends_list}) #res = yield self.c.fblog.find_one() #print res res = yield friends.get() self.assertEqual(res, sorted(friends_list, key = lambda x: x['isStar'])) self.stop() @gen_test def test_set_None(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list}) doc = yield self.c.users.find_one({'uid': 1}) res = yield users.get('haslog') self.assertEqual(res, 1) res = yield users.get('test') self.assertEqual(res, 'xyz') res = yield users.find(1, ['test', 'xyz']) self.assertEqual(res, {'haslog': 1, 'test': 'xyz'}) yield users.set('test', None) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.assertEqual(sr.hgetall(users._key), {'haslog': '1'}) doc1 = yield self.c.users.find_one({'uid': 1}) doc['test'] = None self.assertEqual(doc, doc1) self.stop() @gen_test def test_common_field_get_and_set(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list}) yield users.set('haslog', 0) res = yield users.get('haslog') self.assertEqual(res, 0) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.stop() @gen_test def test_unset_field(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list}) doc = yield self.c.users.find_one({'uid': 1}) res = yield users.get('test') self.assertEqual(res, 'xyz') yield users.delete(1, 'test') self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.assertEqual(sr.hgetall(users._key), {'haslog': '1'}) doc1 = yield self.c.users.find_one({'uid': 1}) doc.pop('test') self.assertEqual(doc, doc1) self.stop() @gen_test def test_static_find(self): sr = self.sr test_resources = [{'file_id': str(i), 'file_name': 'file' + str(i), 'main_type': i % 2, 'mtime': i, 'download_num': i} for i in range(15)] test_resources_copy = copy.deepcopy(test_resources) for t in test_resources_copy: self.c.all_resources.insert(t) res = yield self.redis_delegator.all_resources.find('0') self.assertEqual(res, test_resources[0]) self.assertTrue(sr.exists('all_resources:0')) res = yield self.redis_delegator.all_resources.find([str(_) for _ in range(6)]) res = sorted(res, key=lambda x: x['file_id']) self.assertEqual(res, test_resources[:6]) res = yield self.redis_delegator.all_resources.find([str(_) for _ in range(7, 12)], {'main_type': 1}) res = sorted(res, key=lambda x: int(x['file_id'])) self.assertEqual(res, test_resources[7:12:2]) @gen_test def test_find_update(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])} yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertFalse(sr.zrank(LRU_QUEUE, users.friends.key_name) is None) update_doc = {'haslog': None} yield users.update(1, update_doc) #doc.update(update_doc) doc.pop('haslog') res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) self.stop() @gen_test def test_write_back(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])} yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) #self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) #self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None) yield users.write_back(1) #self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) #self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) yield users.write_back(1, 'friends') yield users.write_back(1) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) self.stop() @gen_test(timeout=10) def test_try_write_back(self): def side_effect(*args, **kwargs): kwargs['lock_timeout'] = 1 return acquire_lock_with_timeout(*args, **kwargs) with mock.patch('redis_async_lru_scheduler.acquire_lock_with_timeout', side_effect=side_effect) as whate_ever: #with mock.patch('redis_async_lru_scheduler.LOCK_TIMEOUT', 1) as whate_ever: LOCK_TIMEOUT = 1 users = self.redis_delegator.users(1) sr = self.sr friends_list = [{'uid': 1, 'isStar': 5}, {'uid': 2, 'isStar': 0}, {'uid': 3, 'isStar': 1}, {'uid': 4, 'isStar': 0}] doc = {'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])} yield self.c.users.insert({'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key = lambda x: x['isStar'])}) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) time.sleep(LOCK_TIMEOUT+1) res = yield self.redis_delegator.try_write_back(self.ar, 'users:1') self.assertTrue(res) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, users.friends.key_name) is not None) time.sleep(LOCK_TIMEOUT+1) res = yield self.redis_delegator.try_write_back(self.ar, 'users:1.friends') self.assertTrue(res) res = yield self.redis_delegator.try_write_back(self.ar, 'users:1') self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, users.friends.key_name) is None) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) self.stop()
def get_app(self): db = MotorClient(os.getenv('DB_HOST', 'mongodb://localhost:27017'), ) loop = asyncio.get_event_loop() loop.run_until_complete(db.drop_database(os.getenv('DB_NAME'))) return application
class TestTornadoMotorAcl: io_loop = IOLoop.instance() @gen_test def setup(self): print "-setup" self.client = MotorClient() #print "c, ", self.client self.db = self.client['test_database'] self.permissions = MotorCollection(self.db, 'acl_permissions') self.groups = MotorCollection(self.db, 'acl_groups') self.resources = MotorCollection(self.db, 'acl_resources') self.users = MotorCollection(self.db, 'users') self.user_ids = yield self.users.insert([{'name':'burger'},{'name':'paul'}]) self.user_ids = [x for x in self.user_ids] self.admin_user = User("burger", self.user_ids[0]) self.user_user = User("paul", self.user_ids[1]) self.perm_ids = yield self.permissions.insert([x.to_primitive() for x in [ Permission({"name":"read"}), Permission({"name":"write"}), Permission({"name":"update"}), Permission({"name":"delete"}), ]]) self.res_ids = yield self.resources.insert([x.to_primitive() for x in [ Resource({'name':'own data'}), Resource({'name':'others data'}), Resource({'name':'all data'}) ]]) self.group_ids = yield self.groups.insert([x.to_primitive() for x in [ Group({"name":"brugere", "permissions":[ ResourcePermissionPair({"resource":"own data", "permissions":[ "read", "write" ] }) ], "members":[self.user_ids[0], self.user_ids[1]], }), Group({"name":"super brugere", "permissions":[ ResourcePermissionPair({"resource":'own data', "permissions":[ "read", "write", "update" ] }), ResourcePermissionPair({"resource":"others data", "permissions":[ "read" ] }) ], "members":[self.user_ids[0]], }) ]]) doc = yield self.groups.find_one({"name": "brugere"}) members =doc['members'] #print "members:", members #print "users", self.user_ids # assert self.admin_user._id in members assert True @gen_test def teardown(self): self.db = None self.client.drop_database('test_database') @gen_test def test_setup(self): assert len(self.user_ids) == 2 assert len(self.perm_ids) == 4 assert len(self.res_ids) == 3 assert len(self.group_ids) == 2 @gen_test def test_that_user_can_read_own_data(self): handler = OwnDataHandler(self.db, self.user_user) yield handler.post() assert handler.status == 200 #assert handler.status == 200 @gen_test def test_that_user_can_not_read_others_data(self): handler = OthersDataHandler(self.db, self.user_user) yield handler.post() assert handler.status == 403 @gen_test def test_that_admin_can_read_own_data(self): handler = OwnDataHandler(self.db, self.admin_user) yield handler.post() assert handler.status == 200 #assert handler.status == 200 @gen_test def test_that_admin_can_read_others_data(self): handler = OthersDataHandler(self.db, self.admin_user) yield handler.post() assert handler.status == 200 @gen_test def test_that_admin_can_read_others_and_own_data(self): handler = OwnAndOthersDataHandler(self.db, self.admin_user) yield handler.post() assert handler.status == 200
class FileNameSearcherTest(AsyncTestCase): def setUp(self): super(FileNameSearcherTest, self).setUp() self.sr = redis.StrictRedis() self.ar = AsyncStrictRedis() self.m = MotorClient() self.c = self.m.fbt self.patcher1 = mock.patch('redis_handler.RedisHandler.redis_client', return_value=mock_redis) self.patcher1.start() ''' self.patcher2 = mock.patch("pymongo.MongoReplicaSetClient", return_value=sync_db) self.patcher3 = mock.patch("motor.MotorReplicaSetClient", return_value=db) #self.patcher4 = mock.patch("async_redis.asyncRedis.AsyncStrictRedis", spec=AsyncStrictRedis, return_value=AsyncStrictRedis()) self.patcher2.start() self.patcher3.start() #self.patcher4.start() ''' self.io_loop.run_sync(self.setup_coro) def tearDown(self): self.patcher1.stop() ''' self.patcher2.stop() self.patcher3.stop() #self.patcher4.stop() ''' @gen.coroutine def setup_coro(self): yield self.ar.flushdb() yield self.m.drop_database('fbt') yield self.ar.sadd(USER_IP_CACHE_SET_KEY, *[1, 2, 3, 4]) yield self.c.users.insert({'uid': 1, 'friends': [{'uid': 2, 'isStar': 1}, {'uid': 3, 'isStar': 1}]}) yield self.c.resources_of_user.insert({'uid': 2, 'file_ids': [str(_) for _ in range(10)]}) yield self.c.resources_of_user.insert({'uid': 3, 'file_ids': [str(_) for _ in range(5, 15)]}) yield self.c.resources_of_user.insert({'uid': 4, 'file_ids': [str(_) for _ in range(15, 20)]}) for i, file_name in enumerate(test_filename): file_id = str(i) resource = {'file_id': file_id, 'mtime': test_mtime[i], 'download_num': test_download_num[i], 'public': 1, 'main_type': test_main_type[i], 'file_name': file_name} test_resources.append(resource) yield self.c.all_resources.insert(resource) yield FileNameSearcher().file_id_add_title(file_id, file_name) #yield [self.c.all_resources.insert(resource), FileNameSearcher().file_id_add_title(file_id, file_name)] @gen_test(timeout=3) def test_all(self): search_key = test_keyword[3] res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4) res_list = list() for i, v in enumerate(test_filename): if search_key in v: test_resources[i].pop('_id') res_list.append(test_resources[i]) res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4, sort="mtime") #print res_list[0:3], len(res_list) #print res, search_key self.assertEqual(res, (len(res_list), res_list[0:4])) #print res res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True) res = yield FileNameSearcher().query_file_ids_by_file_name(search_key, 1, 4, sort="download_num") self.assertEqual(res, (len(res_list), res_list[0:4])) res = yield FileNameSearcher().query_file_ids_by_file_name('a b', 1, 4, sort="download_num") self.assertEqual(res, (0, [])) res = yield FileNameSearcher().query_file_ids_by_file_name('xxxxxxxx', 1, 4, sort="download_num") self.assertEqual(res, (0, [])) # test for query_file_ids_by_file_name_private search_key = test_keyword[1] print 'search_key', search_key res_list = list() for i, v in enumerate(test_filename[:15]): if search_key in v: if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res = yield FileNameSearcher().query_file_ids_by_file_name_private(1, search_key, 1, 4, sort="mtime") res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) self.assertEqual(res, (len(res_list), res_list[0:4])) # test for query_file_ids_by_file_name_private res_list = list() for i, v in enumerate(test_filename[:15]): if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res = yield FileNameSearcher().get_private_resources(1, 1, 4, sort="mtime") res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) self.assertEqual(res, (15, res_list[0:4])) # test for query_file_ids_by_file_name_private print test_main_type main_type = test_main_type[0] res_list = list() for i, v in enumerate(test_filename[:15]): if main_type == test_resources[i]['main_type']: if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True) res = yield FileNameSearcher().get_private_resources_by_type(1, main_type, 1, 4, sort="download_num") print len(res_list) self.assertEqual(res, (len(res_list), res_list[0:4])) ##################### test_basic ################### # sync test_filename1 = ' '.join(test_keyword[:3]) test_file_id = '21' filename_searcher = FileNameSearcher() rdb = filename_searcher.db mongo = filename_searcher.mongoDB filename_searcher.file_id_add_title_sync(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertTrue(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertTrue(test_file_id in res['file_ids']) filename_searcher.remove_file_id_sync(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertFalse(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertFalse(test_file_id in res['file_ids']) # async yield filename_searcher.file_id_add_title(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertTrue(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertTrue(test_file_id in res['file_ids']) yield filename_searcher.remove_file_id(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertFalse(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertFalse(test_file_id in res['file_ids']) # test init filename_searcher.drop() filename_searcher.init_from_mongo() key_fileids_dict = defaultdict(list) for i, f in enumerate(test_filename): i = str(i) for k in f.split(): key_fileids_dict[k].append(i) for k in test_keyword: key_name = make_key_for_keyword(k) self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k])) for res in mongo.key_fileids.find(): self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']])) # test add filename_searcher.scan_from_mongo(0) for k in test_keyword: key_name = make_key_for_keyword(k) self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k])) for res in mongo.key_fileids.find(): self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']])) print 'add test' last_mtime = max(test_mtime) t_mtime = last_mtime + 1 resource = {'file_id': '21', 'mtime': t_mtime, 'download_num': 11, 'public': 1, 'main_type': 1, 'file_name': 'test'} yield self.c.all_resources.insert(resource) res = filename_searcher.scan_from_mongo(last_mtime) self.assertEqual(t_mtime, res) self.stop()
class FileNameSearcherTest(AsyncTestCase): def setUp(self): super(FileNameSearcherTest, self).setUp() self.sr = redis.StrictRedis() self.ar = AsyncStrictRedis() self.m = MotorClient() self.c = self.m.fbt self.patcher1 = mock.patch('redis_handler.RedisHandler.redis_client', return_value=mock_redis) self.patcher1.start() ''' self.patcher2 = mock.patch("pymongo.MongoReplicaSetClient", return_value=sync_db) self.patcher3 = mock.patch("motor.MotorReplicaSetClient", return_value=db) #self.patcher4 = mock.patch("async_redis.asyncRedis.AsyncStrictRedis", spec=AsyncStrictRedis, return_value=AsyncStrictRedis()) self.patcher2.start() self.patcher3.start() #self.patcher4.start() ''' self.io_loop.run_sync(self.setup_coro) def tearDown(self): self.patcher1.stop() ''' self.patcher2.stop() self.patcher3.stop() #self.patcher4.stop() ''' @gen.coroutine def setup_coro(self): yield self.ar.flushdb() yield self.m.drop_database('fbt') yield self.ar.sadd(USER_IP_CACHE_SET_KEY, *[1, 2, 3, 4]) yield self.c.users.insert({ 'uid': 1, 'friends': [{ 'uid': 2, 'isStar': 1 }, { 'uid': 3, 'isStar': 1 }] }) yield self.c.resources_of_user.insert({ 'uid': 2, 'file_ids': [str(_) for _ in range(10)] }) yield self.c.resources_of_user.insert({ 'uid': 3, 'file_ids': [str(_) for _ in range(5, 15)] }) yield self.c.resources_of_user.insert({ 'uid': 4, 'file_ids': [str(_) for _ in range(15, 20)] }) for i, file_name in enumerate(test_filename): file_id = str(i) resource = { 'file_id': file_id, 'mtime': test_mtime[i], 'download_num': test_download_num[i], 'public': 1, 'main_type': test_main_type[i], 'file_name': file_name } test_resources.append(resource) yield self.c.all_resources.insert(resource) yield FileNameSearcher().file_id_add_title(file_id, file_name) #yield [self.c.all_resources.insert(resource), FileNameSearcher().file_id_add_title(file_id, file_name)] @gen_test(timeout=3) def test_all(self): search_key = test_keyword[3] res = yield FileNameSearcher().query_file_ids_by_file_name( search_key, 1, 4) res_list = list() for i, v in enumerate(test_filename): if search_key in v: test_resources[i].pop('_id') res_list.append(test_resources[i]) res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) res = yield FileNameSearcher().query_file_ids_by_file_name( search_key, 1, 4, sort="mtime") #print res_list[0:3], len(res_list) #print res, search_key self.assertEqual(res, (len(res_list), res_list[0:4])) #print res res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True) res = yield FileNameSearcher().query_file_ids_by_file_name( search_key, 1, 4, sort="download_num") self.assertEqual(res, (len(res_list), res_list[0:4])) res = yield FileNameSearcher().query_file_ids_by_file_name( 'a b', 1, 4, sort="download_num") self.assertEqual(res, (0, [])) res = yield FileNameSearcher().query_file_ids_by_file_name( 'xxxxxxxx', 1, 4, sort="download_num") self.assertEqual(res, (0, [])) # test for query_file_ids_by_file_name_private search_key = test_keyword[1] print 'search_key', search_key res_list = list() for i, v in enumerate(test_filename[:15]): if search_key in v: if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res = yield FileNameSearcher().query_file_ids_by_file_name_private( 1, search_key, 1, 4, sort="mtime") res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) self.assertEqual(res, (len(res_list), res_list[0:4])) # test for query_file_ids_by_file_name_private res_list = list() for i, v in enumerate(test_filename[:15]): if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res = yield FileNameSearcher().get_private_resources(1, 1, 4, sort="mtime") res_list = sorted(res_list, key=lambda x: x["mtime"], reverse=True) self.assertEqual(res, (15, res_list[0:4])) # test for query_file_ids_by_file_name_private print test_main_type main_type = test_main_type[0] res_list = list() for i, v in enumerate(test_filename[:15]): if main_type == test_resources[i]['main_type']: if '_id' in test_resources[i]: test_resources[i].pop('_id') res_list.append(test_resources[i]) res_list = sorted(res_list, key=lambda x: x["download_num"], reverse=True) res = yield FileNameSearcher().get_private_resources_by_type( 1, main_type, 1, 4, sort="download_num") print len(res_list) self.assertEqual(res, (len(res_list), res_list[0:4])) ##################### test_basic ################### # sync test_filename1 = ' '.join(test_keyword[:3]) test_file_id = '21' filename_searcher = FileNameSearcher() rdb = filename_searcher.db mongo = filename_searcher.mongoDB filename_searcher.file_id_add_title_sync(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertTrue(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertTrue(test_file_id in res['file_ids']) filename_searcher.remove_file_id_sync(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertFalse(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertFalse(test_file_id in res['file_ids']) # async yield filename_searcher.file_id_add_title(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertTrue(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertTrue(test_file_id in res['file_ids']) yield filename_searcher.remove_file_id(test_file_id, test_filename1) for k in test_keyword[:3]: key_name = make_key_for_keyword(k) self.assertFalse(rdb.sismember(key_name, test_file_id)) for res in mongo.key_fileids.find({'key': {"$in": test_keyword[:3]}}): self.assertFalse(test_file_id in res['file_ids']) # test init filename_searcher.drop() filename_searcher.init_from_mongo() key_fileids_dict = defaultdict(list) for i, f in enumerate(test_filename): i = str(i) for k in f.split(): key_fileids_dict[k].append(i) for k in test_keyword: key_name = make_key_for_keyword(k) self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k])) for res in mongo.key_fileids.find(): self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']])) # test add filename_searcher.scan_from_mongo(0) for k in test_keyword: key_name = make_key_for_keyword(k) self.assertEqual(sorted(rdb.smembers(key_name)), sorted(key_fileids_dict[k])) for res in mongo.key_fileids.find(): self.assertEqual(sorted(res['file_ids']), sorted(key_fileids_dict[res['key']])) print 'add test' last_mtime = max(test_mtime) t_mtime = last_mtime + 1 resource = { 'file_id': '21', 'mtime': t_mtime, 'download_num': 11, 'public': 1, 'main_type': 1, 'file_name': 'test' } yield self.c.all_resources.insert(resource) res = filename_searcher.scan_from_mongo(last_mtime) self.assertEqual(t_mtime, res) self.stop()
class AsyncLRUTest(AsyncTestCase): def setUp(self): super(AsyncLRUTest, self).setUp() self.sr = redis.StrictRedis() self.ar = AsyncStrictRedis() self.m = MotorClient() self.c = self.m.my_test self.io_loop.run_sync(self.setup_coro) @gen.coroutine def setup_coro(self): yield self.ar.flushdb() yield self.m.drop_database('my_test') self.redis_delegator = RedisDelegate(self.ar, self.c) tag = Tags() users = Users() fblog = Fblog() static_all_resources = StaticAllResources() self.redis_delegator.add_collection(tag) self.redis_delegator.add_collection(users) self.redis_delegator.add_collection(fblog) self.redis_delegator.add_collection(static_all_resources, 'all_resources') @gen_test def test_SetField_get_and_set(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr res = yield file_ids.get() self.assertEqual(res, set()) file_ids_list = ['1', '2', '3'] yield file_ids.set(file_ids_list) self.assertEqual(sr.scard(file_ids.key_name), 3) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, file_ids.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, file_ids.key_name) is not None) res = yield file_ids.get() self.assertEqual(res, set(file_ids_list)) self.stop() @gen_test def test_SetField_scard_sadd_srem_sismember(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr res = yield file_ids.scard() self.assertEqual(res, 0) res = yield file_ids.sadd('1') self.assertEqual(res, 1) res = yield file_ids.scard() self.assertEqual(res, 1) res = yield file_ids.sadd('1', '2', '3') self.assertEqual(res, 2) res = yield file_ids.sismember('2') self.assertTrue(res) res = yield file_ids.srem('2') self.assertEqual(res, 1) res = yield file_ids.sismember('2') self.assertFalse(res) res = yield file_ids.scard() self.assertEqual(res, 2) self.stop() @gen_test def test_SetField_make_data_in_redis(self): file_ids = self.redis_delegator.tags(1).file_ids sr = self.sr file_ids_list = ['1', '2', '3', '4'] yield self.c.tags.insert({'uid': 1, 'file_ids': file_ids_list}) res = yield file_ids.get() self.assertEqual(res, set(file_ids_list)) self.stop() @gen_test def test_ListField_get_and_set(self): log = self.redis_delegator.fblog(1).log sr = self.sr res = yield log.get() self.assertEqual(res, list()) log_list = [{'1': [1, 2]}, {'2': {'2': 1}}, {'3': [1, 2]}] yield log.set(log_list) self.assertEqual(sr.llen(log.key_name), 3) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, log.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, log.key_name) is not None) res = yield log.get() self.assertEqual(res, log_list) self.stop() @gen_test def test_ListField_llen_ltrim(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{ '1': [1, 2] }, { '2': { '2': 1 } }, { '3': [1, 2] }, { '4': [2, 3, 4] }] yield log.set(log_list) res = yield log.llen() self.assertEqual(res, 4) res = yield log.ltrim(0, 5) self.assertTrue(res) res = yield log.llen() self.assertEqual(res, 4) res = yield log.ltrim(0, 1) #self.assertEqual(res, 2) res = yield log.llen() self.assertEqual(res, 2) res = yield log.get() self.assertEqual(res, log_list[:2]) self.stop() @gen_test def test_ListField_lindex(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{ '1': [1, 2] }, { '2': { '2': 1 } }, { '3': [1, 2] }, { '4': [2, 3, 4] }] yield log.set(log_list) res = yield log.lindex(0) self.assertEqual(res, log_list[0]) res = yield log.lindex(1) self.assertEqual(res, log_list[1]) res = yield log.lindex(-1) self.assertEqual(res, log_list[-1]) res = yield log.lindex(-5) self.assertEqual(res, None) res = yield log.lindex(5) self.assertEqual(res, None) self.stop() @gen_test def test_ListField_lrem(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{ '1': [1, 2] }, { '2': { '2': 1 } }, { '3': [1, 2] }, { '2': { '2': 1 } }, { '2': { '2': 1 } }, { '4': [2, 3, 4] }, { '2': { '2': 1 } }] yield log.set(log_list) res = yield log.lrem(1, {'2': {'2': 1}}) res = yield log.get() del log_list[1] self.assertEqual(res, log_list) res = yield log.lrem(-1, {'2': {'2': 1}}) res = yield log.get() del log_list[-1] self.assertEqual(res, log_list) res = yield log.lrem(0, {'2': {'2': 1}}) res = yield log.get() del log_list[2:4] self.assertEqual(res, log_list) self.stop() @gen_test def test_ListField_lindex_rpush_lpop_lrange(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{ '1': [1, 2] }, { '4': [2, 3, 4] }, { '2': { '2': 1 } }, { '3': [1, 2] }] res = yield log.rpush({'1': [1, 2]}) self.assertEqual(res, 1) res = yield log.rpush({'4': [2, 3, 4]}) self.assertEqual(res, 2) res = yield log.rpush({'2': {'2': 1}}, {'3': [1, 2]}) self.assertEqual(res, 4) res = yield log.lrange(0, -1) self.assertEqual(res, log_list) res = yield log.lrange(0, 5) self.assertEqual(res, log_list) res = yield log.lrange(0, -5) self.assertEqual(res, []) res = yield log.lrange(0, 2) self.assertEqual(res, log_list[:3]) res = yield log.lrange(1, 3) self.assertEqual(res, log_list[1:4]) self.stop() @gen_test def test_ListField_make_data_in_redis(self): log = self.redis_delegator.fblog(1).log sr = self.sr log_list = [{ '1': [1, 2] }, { '4': [2, 3, 4] }, { '2': { '2': 1 } }, { '3': [1, 2] }] yield self.c.fblog.insert({ 'uid': 1, 'online_time': 1.0, 'log': log_list }) #res = yield self.c.fblog.find_one() #print res res = yield log.get() self.assertEqual(res, log_list) self.stop() @gen_test def test_ZsetField_get_and_set(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 0 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield friends.set(friends_list) self.assertEqual(sr.zcard(friends.key_name), 4) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, friends.key_name)) self.assertTrue(sr.zrank(LRU_QUEUE, friends.key_name) is not None) res = yield friends.get() self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar'])) self.stop() @gen_test def test_ZsetField_zscore(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield friends.set(friends_list) res = yield friends.zscore(1) self.assertEqual(res, 5) res = yield friends.zscore(3) self.assertEqual(res, 1) res = yield friends.zscore(8) self.assertEqual(res, None) self.stop() @gen_test def test_ZsetField_zadd_zrem_zrange(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield friends.zadd(5, 1, 0, 2, 1, 3, 0, 4) res = yield friends.zcard() self.assertEqual(res, len(friends_list)) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar'])) res = yield friends.zrem(1) self.assertEqual(res, 1) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[1:], key=lambda x: x['isStar'])) res = yield friends.zrem(5) self.assertEqual(res, 0) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[1:], key=lambda x: x['isStar'])) res = yield friends.zrem(2, 3) self.assertEqual(res, 2) res = yield friends.zrange(0, -1) self.assertEqual(res, sorted(friends_list[3:], key=lambda x: x['isStar'])) self.stop() @gen_test def test_ZsetField_make_data_in_redis(self): friends = self.redis_delegator.users(1).friends sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list }) #res = yield self.c.fblog.find_one() #print res res = yield friends.get() self.assertEqual(res, sorted(friends_list, key=lambda x: x['isStar'])) self.stop() @gen_test def test_set_None(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list }) doc = yield self.c.users.find_one({'uid': 1}) res = yield users.get('haslog') self.assertEqual(res, 1) res = yield users.get('test') self.assertEqual(res, 'xyz') res = yield users.find(1, ['test', 'xyz']) self.assertEqual(res, {'haslog': 1, 'test': 'xyz'}) yield users.set('test', None) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.assertEqual(sr.hgetall(users._key), {'haslog': '1'}) doc1 = yield self.c.users.find_one({'uid': 1}) doc['test'] = None self.assertEqual(doc, doc1) self.stop() @gen_test def test_common_field_get_and_set(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list }) yield users.set('haslog', 0) res = yield users.get('haslog') self.assertEqual(res, 0) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.stop() @gen_test def test_unset_field(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': friends_list }) doc = yield self.c.users.find_one({'uid': 1}) res = yield users.get('test') self.assertEqual(res, 'xyz') yield users.delete(1, 'test') self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) self.assertEqual(sr.hgetall(users._key), {'haslog': '1'}) doc1 = yield self.c.users.find_one({'uid': 1}) doc.pop('test') self.assertEqual(doc, doc1) self.stop() @gen_test def test_static_find(self): sr = self.sr test_resources = [{ 'file_id': str(i), 'file_name': 'file' + str(i), 'main_type': i % 2, 'mtime': i, 'download_num': i } for i in range(15)] test_resources_copy = copy.deepcopy(test_resources) for t in test_resources_copy: self.c.all_resources.insert(t) res = yield self.redis_delegator.all_resources.find('0') self.assertEqual(res, test_resources[0]) self.assertTrue(sr.exists('all_resources:0')) res = yield self.redis_delegator.all_resources.find( [str(_) for _ in range(6)]) res = sorted(res, key=lambda x: x['file_id']) self.assertEqual(res, test_resources[:6]) res = yield self.redis_delegator.all_resources.find( [str(_) for _ in range(7, 12)], {'main_type': 1}) res = sorted(res, key=lambda x: int(x['file_id'])) self.assertEqual(res, test_resources[7:12:2]) @gen_test def test_find_update(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] doc = { 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) } yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) }) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertFalse(sr.zrank(LRU_QUEUE, users.friends.key_name) is None) update_doc = {'haslog': None} yield users.update(1, update_doc) #doc.update(update_doc) doc.pop('haslog') res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) self.stop() @gen_test def test_write_back(self): users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] doc = { 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) } yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) }) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) res = yield users.find(1) self.assertEqual(res, doc) #self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) #self.assertFalse(sr.zrank(LRU_QUEUE, users._key) is None) yield users.write_back(1) #self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) #self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) yield users.write_back(1, 'friends') yield users.write_back(1) doc1 = yield self.c.users.find_one({'uid': 1}, {'uid': 0, '_id': 0}) self.assertEqual(doc, doc1) self.stop() @gen_test(timeout=10) def test_try_write_back(self): def side_effect(*args, **kwargs): kwargs['lock_timeout'] = 1 return acquire_lock_with_timeout(*args, **kwargs) with mock.patch('redis_async_lru_scheduler.acquire_lock_with_timeout', side_effect=side_effect) as whate_ever: #with mock.patch('redis_async_lru_scheduler.LOCK_TIMEOUT', 1) as whate_ever: LOCK_TIMEOUT = 1 users = self.redis_delegator.users(1) sr = self.sr friends_list = [{ 'uid': 1, 'isStar': 5 }, { 'uid': 2, 'isStar': 0 }, { 'uid': 3, 'isStar': 1 }, { 'uid': 4, 'isStar': 0 }] doc = { 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) } yield self.c.users.insert({ 'uid': 1, 'haslog': 1, 'test': 'xyz', 'friends': sorted(friends_list, key=lambda x: x['isStar']) }) #doc = yield self.c.users.find_one({'uid': 1}) res = yield users.find(1) self.assertEqual(res, doc) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) update_doc = {'test': '123', 'abc': 'aaa'} yield users.update(1, update_doc) doc.update(update_doc) self.assertTrue(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is not None) time.sleep(LOCK_TIMEOUT + 1) res = yield self.redis_delegator.try_write_back(self.ar, 'users:1') self.assertTrue(res) self.assertFalse(sr.sismember(KEYS_MODIFIED_SET, users._key)) self.assertTrue(sr.zrank(LRU_QUEUE, users._key) is None) doc1 = yield self.c.users.find_one({'uid': 1}, { 'uid': 0, '_id': 0 }) self.assertEqual(doc, doc1) update_doc = {'haslog': 3, 'friends': [{'uid': 2, 'isStar': 11}]} yield users.update(1, update_doc) doc.update(update_doc) self.assertTrue( sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertTrue( sr.zrank(LRU_QUEUE, users.friends.key_name) is not None) time.sleep(LOCK_TIMEOUT + 1) res = yield self.redis_delegator.try_write_back( self.ar, 'users:1.friends') self.assertTrue(res) res = yield self.redis_delegator.try_write_back(self.ar, 'users:1') self.assertFalse( sr.sismember(KEYS_MODIFIED_SET, users.friends.key_name)) self.assertTrue( sr.zrank(LRU_QUEUE, users.friends.key_name) is None) doc1 = yield self.c.users.find_one({'uid': 1}, { 'uid': 0, '_id': 0 }) self.assertEqual(doc, doc1) self.stop()