def test_process_max_distance(self, spider, middleware):
request = Request('http://domain.org', meta={'distance': 5})
response = Response('http://domain.org', request=request)
result = [Request('http://domain.com')]
out = list(middleware.process_spider_output(response, result, spider))
self.assertEqual(out, [])
def test_non_request_ignored(self, spider, middleware):
request = Request('http://domain.org')
response = Response('http://domain.org', request=request)
result = [None, Request('http://domain.org')]
out = list(middleware.process_spider_output(response, result, spider))
self.assertEqual(out, result)
self.assertEqual(out[1].meta['distance'], 0)
def test_disabled(self, spider, middleware):
request = Request('http://domain.org')
response = Response('http://domain.org', request=request)
result = [Request('http://domain.com')]
out = list(middleware.process_spider_output(response, result, spider))
self.assertEqual(out, result)
self.assertEqual(out[0].meta['distance'], 1)
def test_stats(self, spider, middleware):
request = Request('http://domain.org')
response = Response('http://domain.org', request=request)
result = [Request('http://domain.org'), Request('http://domain.com')]
out = list(middleware.process_spider_output(response, result, spider))
self.assertEqual(out, result)
self.assertEqual(out[0].meta['distance'], 0)
self.assertEqual(out[1].meta['distance'], 1)
stats = spider.crawler.stats
self.assertEqual(
stats.get_value('request_distance_count/0', spider=spider), 1)
self.assertEqual(
stats.get_value('request_distance_count/1', spider=spider), 1)
self.assertEqual(
stats.get_value('request_distance_max', spider=spider), 1)