def test_log_extra_retry_success(self): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider', )) as log: get_retry_request( request, spider=spider, ) log.check_present(spider)
def test_custom_stats_key(self): request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" stats_key = "custom_retry" get_retry_request( request, spider=spider, reason=expected_reason, stats_base_key=stats_key, ) for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): self.assertEqual(spider.crawler.stats.get_value(stat), 1)
def test_two_retries(self): spider = self.get_spider() request = Request('https://example.com') new_request = request max_retry_times = 2 for index in range(max_retry_times): with LogCapture() as log: new_request = get_retry_request( new_request, spider=spider, max_retry_times=max_retry_times, ) self.assertIsInstance(new_request, Request) self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) expected_retry_times = index + 1 self.assertEqual(new_request.meta['retry_times'], expected_retry_times) self.assertEqual(new_request.priority, -expected_retry_times) expected_reason = "unspecified" for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): value = spider.crawler.stats.get_value(stat) self.assertEqual(value, expected_retry_times) log.check_present(( "scrapy.downloadermiddlewares.retry", "DEBUG", f"Retrying {request} (failed {expected_retry_times} times): " f"{expected_reason}", )) with LogCapture() as log: new_request = get_retry_request( new_request, spider=spider, max_retry_times=max_retry_times, ) self.assertEqual(new_request, None) self.assertEqual(spider.crawler.stats.get_value('retry/max_reached'), 1) failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present(( "scrapy.downloadermiddlewares.retry", "ERROR", f"Gave up retrying {request} (failed {failure_count} times): " f"{expected_reason}", ))
def test_custom_logger(self): logger = logging.getLogger("custom-logger") request = Request("https://example.com") spider = self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( request, spider=spider, reason=expected_reason, logger=logger, ) log.check_present(( "custom-logger", "DEBUG", f"Retrying {request} (failed 1 times): {expected_reason}", ))
def test_priority_adjust_setting(self): priority_adjust = 1 spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) request = Request('https://example.com') new_request = get_retry_request( request, spider=spider, ) self.assertEqual(new_request.priority, priority_adjust)
def test_max_retry_times_setting(self): max_retry_times = 0 spider = self.get_spider({'RETRY_TIMES': max_retry_times}) request = Request('https://example.com') new_request = get_retry_request( request, spider=spider, ) self.assertEqual(new_request, None)
def test_reason_string(self): request = Request('https://example.com') spider = self.get_spider() expected_reason = 'because' with LogCapture() as log: get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): self.assertEqual(spider.crawler.stats.get_value(stat), 1) log.check_present(( "scrapy.downloadermiddlewares.retry", "DEBUG", f"Retrying {request} (failed {expected_retry_times} times): " f"{expected_reason}", ))
def test_max_retry_times_meta(self): max_retry_times = 0 spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) meta = {'max_retry_times': max_retry_times} request = Request('https://example.com', meta=meta) new_request = get_retry_request( request, spider=spider, ) self.assertEqual(new_request, None)
def test_reason_custom_exception_class(self): request = Request('https://example.com') spider = self.get_spider() expected_reason = IgnoreRequest expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: get_retry_request( request, spider=spider, reason=expected_reason, ) expected_retry_times = 1 stat = spider.crawler.stats.get_value( f'retry/reason_count/{expected_reason_string}') self.assertEqual(stat, 1) log.check_present(( "scrapy.downloadermiddlewares.retry", "DEBUG", f"Retrying {request} (failed {expected_retry_times} times): " f"{expected_reason}", ))
def test_max_retries_reached(self): request = Request('https://example.com') spider = self.get_spider() max_retry_times = 0 with LogCapture() as log: new_request = get_retry_request( request, spider=spider, max_retry_times=max_retry_times, ) self.assertEqual(new_request, None) self.assertEqual(spider.crawler.stats.get_value('retry/max_reached'), 1) failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present(( "scrapy.downloadermiddlewares.retry", "ERROR", f"Gave up retrying {request} (failed {failure_count} times): " f"{expected_reason}", ))
def test_basic_usage(self): request = Request('https://example.com') spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, spider=spider, ) self.assertIsInstance(new_request, Request) self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) expected_retry_times = 1 self.assertEqual(new_request.meta['retry_times'], expected_retry_times) self.assertEqual(new_request.priority, -1) expected_reason = "unspecified" for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): self.assertEqual(spider.crawler.stats.get_value(stat), 1) log.check_present(( "scrapy.downloadermiddlewares.retry", "DEBUG", f"Retrying {request} (failed {expected_retry_times} times): " f"{expected_reason}", ))
def test_no_spider(self): request = Request('https://example.com') with self.assertRaises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa