def _handle_redirect(self, origin_crawl_request: CrawlRequest,
origin_request: Request, response: Response):
crawl_response = CrawlResponse(origin_crawl_request, response.status,
response.headers, None)
redirected_request = CrawlRequest(
origin_request.url).merge(origin_crawl_request)
self.crawl(redirected_request)
origin_crawl_request.redirect_func(crawl_response, redirected_request) if origin_crawl_request.redirect_func \
else self.on_request_redirect(crawl_response, redirected_request)
def _handle_response(self, request: CrawlRequest, response: Response):
crawl_response = CrawlResponse(request, response.status,
response.headers,
syncer.sync(response.text()))
if 200 <= response.status < 300:
request.success_func(
crawl_response
) if request.success_func else self.on_response_success(
crawl_response)
else:
request.error_func(
crawl_response
) if request.error_func else self.on_response_error(crawl_response)
def test_str_should_return_string_representation() -> None:
assert str(CrawlResponse(
request, 200,
{})) == 'CrawlResponse(url=https://example.com, status=200)'
def test_text_should_return_text_content_when_text_content_is_set() -> None:
text_content = 'Test'
assert CrawlResponse(request, 200, {},
text=text_content).text == text_content
def test_text_should_return_none_when_text_content_is_not_set() -> None:
assert CrawlResponse(request, 200, {}).text is None
def test_headers_should_return_response_headers() -> None:
headers = {'Content-Type': 'text/html; charset=utf-8'}
assert CrawlResponse(request, 200, headers).headers == headers
def test_status_should_return_response_status() -> None:
assert CrawlResponse(request, 200, {}).status == 200
def test_request_should_return_request() -> None:
assert CrawlResponse(request, 200, {}).request is request