class TestDepthMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider('scrapytest.org') self.stats = StatsCollector(crawler) self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) def test_process_spider_output(self): req = Request('http://scrapytest.org') resp = Response('http://scrapytest.org') resp.request = req result = [Request('http://scrapytest.org')] out = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out, result) rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) self.assertEquals(rdc, 1) req.meta['depth'] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out2, []) rdm = self.stats.get_value('request_depth_max', spider=self.spider) self.assertEquals(rdm, 1) def tearDown(self): self.stats.close_spider(self.spider, '')
class TestDepthMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('scrapytest.org') self.stats = StatsCollector() self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) def test_process_spider_output(self): req = Request('http://scrapytest.org') resp = Response('http://scrapytest.org') resp.request = req result = [Request('http://scrapytest.org')] out = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out, result) rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) self.assertEquals(rdc, 1) req.meta['depth'] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out2, []) rdm = self.stats.get_value('request_depth_max', spider=self.spider) self.assertEquals(rdm, 1) def tearDown(self): self.stats.close_spider(self.spider, '')
def test_collector(self): stats = StatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.get_value('anything', 'default'), 'default') stats.set_value('test', 'value') self.assertEqual(stats.get_stats(), {'test': 'value'}) stats.set_value('test2', 23) self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23}) self.assertEqual(stats.get_value('test2'), 23) stats.inc_value('test2') self.assertEqual(stats.get_value('test2'), 24) stats.inc_value('test2', 6) self.assertEqual(stats.get_value('test2'), 30) stats.max_value('test2', 6) self.assertEqual(stats.get_value('test2'), 30) stats.max_value('test2', 40) self.assertEqual(stats.get_value('test2'), 40) stats.max_value('test3', 1) self.assertEqual(stats.get_value('test3'), 1) stats.min_value('test2', 60) self.assertEqual(stats.get_value('test2'), 40) stats.min_value('test2', 35) self.assertEqual(stats.get_value('test2'), 35) stats.min_value('test4', 7) self.assertEqual(stats.get_value('test4'), 7)
def test_collector(self): stats = StatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value("anything"), None) self.assertEqual(stats.get_value("anything", "default"), "default") stats.set_value("test", "value") self.assertEqual(stats.get_stats(), {"test": "value"}) stats.set_value("test2", 23) self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) self.assertEqual(stats.get_value("test2"), 23) stats.inc_value("test2") self.assertEqual(stats.get_value("test2"), 24) stats.inc_value("test2", 6) self.assertEqual(stats.get_value("test2"), 30) stats.max_value("test2", 6) self.assertEqual(stats.get_value("test2"), 30) stats.max_value("test2", 40) self.assertEqual(stats.get_value("test2"), 40) stats.max_value("test3", 1) self.assertEqual(stats.get_value("test3"), 1) stats.min_value("test2", 60) self.assertEqual(stats.get_value("test2"), 40) stats.min_value("test2", 35) self.assertEqual(stats.get_value("test2"), 35) stats.min_value("test4", 7) self.assertEqual(stats.get_value("test4"), 7)