def test_log_extra_retry_success(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture(attributes=('spider', )) as log:
get_retry_request(
request,
spider=spider,
)
log.check_present(spider)
def test_custom_stats_key(self):
request = Request("https://example.com")
spider = self.get_spider()
expected_reason = "because"
stats_key = "custom_retry"
get_retry_request(
request,
spider=spider,
reason=expected_reason,
stats_base_key=stats_key,
)
for stat in (f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}"):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
def test_two_retries(self):
spider = self.get_spider()
request = Request('https://example.com')
new_request = request
max_retry_times = 2
for index in range(max_retry_times):
with LogCapture() as log:
new_request = get_retry_request(
new_request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertIsInstance(new_request, Request)
self.assertNotEqual(new_request, request)
self.assertEqual(new_request.dont_filter, True)
expected_retry_times = index + 1
self.assertEqual(new_request.meta['retry_times'],
expected_retry_times)
self.assertEqual(new_request.priority, -expected_retry_times)
expected_reason = "unspecified"
for stat in ('retry/count',
f'retry/reason_count/{expected_reason}'):
value = spider.crawler.stats.get_value(stat)
self.assertEqual(value, expected_retry_times)
log.check_present((
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
))
with LogCapture() as log:
new_request = get_retry_request(
new_request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertEqual(new_request, None)
self.assertEqual(spider.crawler.stats.get_value('retry/max_reached'),
1)
failure_count = max_retry_times + 1
expected_reason = "unspecified"
log.check_present((
"scrapy.downloadermiddlewares.retry",
"ERROR",
f"Gave up retrying {request} (failed {failure_count} times): "
f"{expected_reason}",
))
def test_custom_logger(self):
logger = logging.getLogger("custom-logger")
request = Request("https://example.com")
spider = self.get_spider()
expected_reason = "because"
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
logger=logger,
)
log.check_present((
"custom-logger",
"DEBUG",
f"Retrying {request} (failed 1 times): {expected_reason}",
))
def test_priority_adjust_setting(self):
priority_adjust = 1
spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust})
request = Request('https://example.com')
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request.priority, priority_adjust)
def test_max_retry_times_setting(self):
max_retry_times = 0
spider = self.get_spider({'RETRY_TIMES': max_retry_times})
request = Request('https://example.com')
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request, None)
def test_reason_string(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = 'because'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
log.check_present((
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
))
def test_max_retry_times_meta(self):
max_retry_times = 0
spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
meta = {'max_retry_times': max_retry_times}
request = Request('https://example.com', meta=meta)
new_request = get_retry_request(
request,
spider=spider,
)
self.assertEqual(new_request, None)
def test_reason_custom_exception_class(self):
request = Request('https://example.com')
spider = self.get_spider()
expected_reason = IgnoreRequest
expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
reason=expected_reason,
)
expected_retry_times = 1
stat = spider.crawler.stats.get_value(
f'retry/reason_count/{expected_reason_string}')
self.assertEqual(stat, 1)
log.check_present((
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
))
def test_max_retries_reached(self):
request = Request('https://example.com')
spider = self.get_spider()
max_retry_times = 0
with LogCapture() as log:
new_request = get_retry_request(
request,
spider=spider,
max_retry_times=max_retry_times,
)
self.assertEqual(new_request, None)
self.assertEqual(spider.crawler.stats.get_value('retry/max_reached'),
1)
failure_count = max_retry_times + 1
expected_reason = "unspecified"
log.check_present((
"scrapy.downloadermiddlewares.retry",
"ERROR",
f"Gave up retrying {request} (failed {failure_count} times): "
f"{expected_reason}",
))
def test_basic_usage(self):
request = Request('https://example.com')
spider = self.get_spider()
with LogCapture() as log:
new_request = get_retry_request(
request,
spider=spider,
)
self.assertIsInstance(new_request, Request)
self.assertNotEqual(new_request, request)
self.assertEqual(new_request.dont_filter, True)
expected_retry_times = 1
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
self.assertEqual(new_request.priority, -1)
expected_reason = "unspecified"
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
log.check_present((
"scrapy.downloadermiddlewares.retry",
"DEBUG",
f"Retrying {request} (failed {expected_retry_times} times): "
f"{expected_reason}",
))
def test_no_spider(self):
request = Request('https://example.com')
with self.assertRaises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa