def test_post_request():
mw = _get_mw()
for body in [b'', b'foo=bar']:
req1 = scrapy.Request("http://example.com",
method="POST",
body=body,
meta={'prerender': {
'endpoint': 'render'
}})
req = mw.process_request(req1, None)
assert json.loads(to_native_str(req.body)) == {
'url': 'http://example.com',
'http_method': 'POST',
'body': to_native_str(body),
}
def test_url_with_fragment():
mw = _get_mw()
url = "http://example.com#id1"
req = scrapy.Request("http://example.com",
meta={'prerender': {
'args': {
'url': url
}
}})
req = mw.process_request(req, None)
assert json.loads(to_native_str(req.body)) == {'url': url}
def test_override_prerender_url():
mw = _get_mw()
req1 = scrapy.Request("http://example.com",
meta={
'prerender': {
'endpoint': 'render.png',
'prerender_url':
'http://prerender.example.com'
}
})
req = mw.process_request(req1, None)
assert req.url == 'http://prerender.example.com/render.png'
assert json.loads(to_native_str(req.body)) == {'url': req1.url}
def test_float_wait_arg():
mw = _get_mw()
req1 = scrapy.Request(
"http://example.com",
meta={'prerender': {
'endpoint': 'render',
'args': {
'wait': 0.5
}
}})
req = mw.process_request(req1, None)
assert json.loads(to_native_str(req.body)) == {
'url': req1.url,
'wait': 0.5
}
def test_prerender_request_no_url():
mw = _get_mw()
lua_source = "function main(prerender) return {result='ok'} end"
req1 = PrerenderRequest(meta={
'prerender': {
'args': {
'lua_source': lua_source
},
'endpoint': 'execute',
}
})
req = mw.process_request(req1, None)
assert req.url == 'http://127.0.0.1:8050/execute'
assert json.loads(to_native_str(req.body)) == {
'url': 'about:blank',
'lua_source': lua_source
}
def test_prerender_request():
mw = _get_mw()
cookie_mw = _get_cookie_mw()
req = PrerenderRequest("http://example.com?foo=bar&url=1&wait=100")
assert repr(req) == "<GET http://example.com?foo=bar&url=1&wait=100>"
# check request preprocessing
req2 = cookie_mw.process_request(req, None) or req
req2 = mw.process_request(req2, None) or req2
assert req2 is not None
assert req2 is not req
assert req2.url == "http://127.0.0.1:8050/render"
assert req2.headers == {b'Content-Type': [b'application/json']}
assert req2.method == 'POST'
assert isinstance(req2, PrerenderRequest)
assert repr(
req2
) == "<GET http://example.com?foo=bar&url=1&wait=100 via http://127.0.0.1:8050/render>"
expected_body = {'url': req.url}
assert json.loads(to_native_str(req2.body)) == expected_body
# check response post-processing
response = TextResponse(
"http://127.0.0.1:8050/render",
# Scrapy doesn't pass request to constructor
# request=req2,
headers={b'Content-Type': b'text/html'},
body=b"<html><body>Hello</body></html>")
response2 = mw.process_response(req2, response, None)
response2 = cookie_mw.process_response(req2, response2, None)
assert isinstance(response2, scrapy_prerender.PrerenderTextResponse)
assert response2 is not response
assert response2.real_url == req2.url
assert response2.url == req.url
assert response2.body == b"<html><body>Hello</body></html>"
assert response2.css("body").extract_first() == "<body>Hello</body>"
assert response2.headers == {b'Content-Type': [b'text/html']}
# check .replace method
response3 = response2.replace(status=404)
assert response3.status == 404
assert isinstance(response3, scrapy_prerender.PrerenderTextResponse)
for attr in ['url', 'real_url', 'headers', 'body']:
assert getattr(response3, attr) == getattr(response2, attr)
def test_prerender_request_url_with_fragment():
mw = _get_mw()
url = "http://example.com#id1"
req = PrerenderRequest(url)
req = mw.process_request(req, None)
assert json.loads(to_native_str(req.body)) == {'url': url}