class Test(unittest.TestCase):
def setUp(self):
self.redis = mock.MagicMock()
self.mongo = mock.MagicMock()
self.cache_controller = CacheController(self.redis, self.mongo, loop_threshold=5, sync_interval=1, logger=mock.MagicMock())
def test_is_sync_redis2mongo_1000(self):
self.redis.cli.scard = mock.Mock()
self.redis.cli.scard.return_value = 1000
self.assertEqual(True, self.cache_controller.is_sync_redis2mongo())
def test_is_sync_redis2mongo_0(self):
self.redis.cli.scard = mock.Mock()
self.redis.cli.scard.return_value = 0
self.assertEqual(False, self.cache_controller.is_sync_redis2mongo())
def test_is_sync_mongo2redis(self):
self.assertEqual(False, self.cache_controller.is_sync_mongo2redis())
def test_run_all(self):
self.redis.cli.scard = mock.Mock(side_effect=[1000, 0, 0, 0, 0, 0, 0, 0, 0, 0])
self.redis.cli.zcard.return_value = mock.Mock(side_effect=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
htmls = [Html(url='http://www.crosswarp.com/'),
Html(url='http://cs.crosswarp.com/'),
Html(url='http://cr.crosswarp.com/')]
self.redis.get_parsed_htmls.return_value = htmls
self.cache_controller.run()
self.assertTrue(self.redis.get_parsed_htmls.call_count > 0)
self.assertTrue(self.mongo.insert_htmls.call_count > 0)
def test_run_no_parsed_htmls(self):
self.redis.cli.scard.return_value = 0
self.redis.cli.zcard.return_value = 0
self.redis.get_parsed_htmls.return_value = []
self.cache_controller.run()
self.assertTrue(self.redis.get_parsed_htmls.call_count == 0)
self.assertTrue(self.mongo.update_crawled_html.call_count == 0)
def build_cache_controller_thread(datasource_cache, datasource_backend, request, logger, exclude_thread_list):
thread = CacheController(datasource_cache, datasource_backend, sync_count=request['sync_count'],
sync_interval=request['sync_interval'], logger=logger)
exclude_thread_list.append(thread)
thread.start()
return thread
def setUp(self):
self.redis = mock.MagicMock()
self.mongo = mock.MagicMock()
self.cache_controller = CacheController(self.redis, self.mongo, loop_threshold=5, sync_interval=1, logger=mock.MagicMock())