def test_not_foundbasic(self):
url_404 = "http://httpbin.org/status/404"
with respx.mock:
respx.get(url_404).mock(httpx.Response(404))
result = self.perform_request(url_404)
self.assertEqual(result.status_code, 404)
span = self.assert_span()
self.assertEqual(
span.attributes.get(SpanAttributes.HTTP_STATUS_CODE), 404)
self.assertIs(
span.status.status_code,
trace.StatusCode.ERROR,
)
def test_api_inheritance_basic(respx_mock, backend) -> None:
"""
"""
client = InheritedClient(backend=backend)
expected = dict(id=1, name='thing1')
req = respx_mock.get("https://example.com/stuff/thing1")\
.mock(return_value=httpx.Response(200, content=json.dumps(expected)))
res = client.get('thing1')
assert req.called is True
assert res == expected
client = InheritedClient('https://example2.com', backend=backend)
req = respx_mock.get('https://example2.com/stuff/thing1')\
.mock(return_value=httpx.Response(200, content=json.dumps(expected)))
res = client.get('thing1')
assert req.called is True
assert res == expected
def app(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/v1/public/search"
assert dict(httpx.QueryParams(request.url.query)) == {
"page": "1",
"per_page": "10000",
"in.isPublicProfile": "true",
"in.isCovidVaccineSupported": "true",
"or.covidOnlineBookingAvailabilities.vaccineAstraZeneca1": "true",
"or.covidOnlineBookingAvailabilities.vaccineJanssen1": "true",
"or.covidOnlineBookingAvailabilities.vaccinePfizer1": "true",
"or.covidOnlineBookingAvailabilities.vaccineModerna1": "true",
}
path = Path("tests/fixtures/ordoclic/search.json")
return httpx.Response(200, json=json.loads(path.read_text()))
def test_cannot_read_after_response_closed():
is_closed = False
def close_func():
nonlocal is_closed
is_closed = True
stream = IteratorStream(iterator=streaming_body(), close_func=close_func)
response = httpx.Response(200, stream=stream, request=REQUEST)
response.close()
assert is_closed
with pytest.raises(httpx.ResponseClosed):
response.read()
def test_needs_refresh_with_valid(client):
with mock.patch.object(httpx.Client, "get") as mock_get:
with mock.patch.object(Tokens, "write") as mock_write:
mock_write.return_value = None
mock_get.return_value = httpx.Response(
status_code=200,
request=mock.Mock(),
json={"expires_in": 10000},
)
client.authorise(token_path=TOKENS_PATH)
needs_refresh = client.needs_refresh()
mock_get.assert_called_once()
assert not needs_refresh
def test_response_no_charset_with_cp_1252_content():
"""
A response with Windows 1252 encoded content should decode correctly,
even with no charset specified.
"""
content = "Euro Currency: € abcdefghijklmnopqrstuzwxyz".encode("cp1252")
headers = {"Content-Type": "text/plain"}
response = httpx.Response(
200,
content=content,
headers=headers,
)
assert response.text == "Euro Currency: € abcdefghijklmnopqrstuzwxyz"
assert response.charset_encoding is None
assert response.apparent_encoding is not None
def test_response_no_charset_with_ascii_content():
"""
A response with ascii encoded content should decode correctly,
even with no charset specified.
"""
content = b"Hello, world!"
headers = {"Content-Type": "text/plain"}
response = httpx.Response(
200,
content=content,
headers=headers,
)
assert response.status_code == 200
assert response.encoding is None
assert response.text == "Hello, world!"
async def test_async_streaming_response():
stream = AsyncIteratorStream(aiterator=async_streaming_body())
response = httpx.Response(
200,
stream=stream,
)
assert response.status_code == 200
assert not response.is_closed
content = await response.aread()
assert content == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
def test_requests_500_error(self):
respx.get(self.URL).mock(httpx.Response(500))
self.perform_request(self.URL)
span = self.assert_span()
self.assertEqual(
span.attributes,
{
SpanAttributes.HTTP_METHOD: "GET",
SpanAttributes.HTTP_URL: self.URL,
SpanAttributes.HTTP_STATUS_CODE: 500,
},
)
self.assertEqual(span.status.status_code, StatusCode.ERROR)
async def test_async_try_login_post_wrong_credentials():
target_url = "http://perdu.com/"
respx.get(target_url).mock(return_value=httpx.Response(
200,
text=
"<html><head><title>Vous Etes Perdu ?</title></head><body><h1>Perdu sur l'Internet ?</h1> \
<h2>Pas de panique, on va vous aider</h2> \
<strong><pre> * <----- vous êtes ici</pre></strong><a href='http://perdu.com/foobar/'></a> \
<a href='http://perdu.com/foobar/signout'></a> \
<form name='loginform' method='post' action='userinfo.php'> \
<table cellpadding='4' cellspacing='1'> \
<tbody><tr><td>Username : </td><td><input name='uname' type='text' size='20'></td></tr> \
<tr><td>Password : </td><td><input name='pass' type='password' size='20'></td></tr> \
<tr><td colspan='2' align='right'><input type='submit' value='login' style='width:75px;'></td></tr> \
</tbody></table> \
</form> \
<div><a href='http://perdu.com/a/b/signout'></a></div></body></html>"
))
respx.post(target_url + 'userinfo.php').mock(return_value=httpx.Response(
401,
text=
"<html><head><title>Vous Etes Perdu ?</title></head><body><h1>Perdu sur l'Internet ?</h1> \
<h2>Pas de panique, on va vous aider</h2> \
<strong><pre> * <----- vous êtes ici</pre></strong><a href='http://perdu.com/foobar/'></a> \
<div></div></body></html>"))
crawler = AsyncCrawler(Request(target_url), timeout=1)
crawler._auth_credentials = ["username", "password"]
is_logged_in, form, disconnect_urls = await crawler._async_try_login_post(
"username", "password", target_url)
assert form == {'login_field': 'uname', 'password_field': 'pass'}
assert len(disconnect_urls) == 0
assert is_logged_in is False
def test_csp_detection():
url = "http://perdu.com/"
respx.get(url).mock(return_value=httpx.Response(
200, headers={"Content-Type": "text/html"}))
resp = httpx.get(url)
page = Page(resp)
assert not has_csp(page)
url = "http://perdu.com/http_csp"
respx.get(url).mock(
return_value=httpx.Response(200,
headers={
"Content-Type": "text/html",
"Content-Security-Policy": "blahblah;"
}))
resp = httpx.get(url)
page = Page(resp)
assert has_csp(page)
url = "http://perdu.com/meta_csp"
respx.get(url).mock(
return_value=httpx.Response(200,
headers={"Content-Type": "text/html"},
text="""<html>
<head>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src https://*; child-src 'none';">
</head>
<body>Hello there</body>
</html>"""))
resp = httpx.get(url)
page = Page(resp)
assert has_csp(page)
def test_response_default_text_encoding():
"""
A media type of 'text/*' with no charset should default to ISO-8859-1.
See: https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1
"""
content = b"Hello, world!"
headers = {"Content-Type": "text/plain"}
response = httpx.Response(
200,
content=content,
headers=headers,
)
assert response.status_code == 200
assert response.encoding == "iso-8859-1"
assert response.text == "Hello, world!"
async def test_should_follow_unprocessed_url(self):
url = 'http://foo.com'
request = httpx.Request('GET', url)
httpx_response = httpx.Response(200, request=request)
queue = Queue(2)
response = StaticResponse(reachable_urls={'http://bar.com'},
followed_urls=set(),
queue=queue,
httpx_response=httpx_response)
await response.follow(url)
queue_length = queue.length
await queue.close()
assert {'http://bar.com'} == response._reachable_urls
assert {url} == response._followed_urls
assert 1 == queue_length
async def test_cookies():
respx.get("http://perdu.com/").mock(return_value=httpx.Response(
200, text="Hello there!", headers={"Set-Cookie": "foo=bar; Path=/"}))
def print_headers_callback(request):
return httpx.Response(200, json=dict(request.headers))
respx.get("http://perdu.com/cookies").mock(
side_effect=print_headers_callback)
crawler = AsyncCrawler(Request("http://perdu.com/"))
response = await crawler.async_get(Request("http://perdu.com/"))
assert "foo=bar" in response.headers["set-cookie"]
response = await crawler.async_get(Request("http://perdu.com/cookies"))
assert "foo=bar" in response.content
await crawler.close()
def test_response_no_charset_with_iso_8859_1_content():
"""
A response with ISO 8859-1 encoded content should decode correctly,
even with no charset specified.
"""
content = "Accented: Österreich abcdefghijklmnopqrstuzwxyz".encode(
"iso-8859-1")
headers = {"Content-Type": "text/plain"}
response = httpx.Response(
200,
content=content,
headers=headers,
)
assert response.text == "Accented: Österreich abcdefghijklmnopqrstuzwxyz"
assert response.charset_encoding is None
assert response.apparent_encoding is not None
def test_deflate():
"""
Deflate encoding may use either 'zlib' or 'deflate' in the wild.
https://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib#answer-22311297
"""
body = b"test 123"
compressor = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS)
compressed_body = compressor.compress(body) + compressor.flush()
headers = [(b"Content-Encoding", b"deflate")]
response = httpx.Response(200,
headers=headers,
content=compressed_body,
request=REQUEST)
assert response.content == body
async def test_cannot_aread_after_response_closed():
is_closed = False
async def close_func():
nonlocal is_closed
is_closed = True
stream = AsyncIteratorStream(aiterator=async_streaming_body(),
close_func=close_func)
response = httpx.Response(200, stream=stream, request=REQUEST)
await response.aclose()
assert is_closed
with pytest.raises(httpx.ResponseClosed):
await response.aread()
async def keys_query(self,
user_set: Iterable[str],
token: Optional[str] = None) -> MatrixResponse:
"""Query the current devices and identity keys for the given users.
Returns the HTTP method, HTTP path and data for the request.
Args:
user_set (Set[str]): The users for which the keys should be
downloaded.
token (Optional[str]): If the client is fetching keys as a result
of a device update received in a sync request, this should be
the 'since' token of that sync request, or any later sync
token.
* Matrix Spec
"""
return MatrixResponse(httpx.Response(status_code=404, json={}))
async def test_streaming():
body = b"test 123"
compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
async def compress(body):
yield compressor.compress(body)
yield compressor.flush()
headers = [(b"Content-Encoding", b"gzip")]
response = httpx.Response(
200,
headers=headers,
content=compress(body),
)
assert not hasattr(response, "body")
assert await response.aread() == body
async def test_aread():
response = httpx.Response(
200,
content=b"Hello, world!",
)
assert response.status_code == 200
assert response.text == "Hello, world!"
assert response.encoding == "ascii"
assert response.is_closed
content = await response.aread()
assert content == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
async def test_text_decoder_known_encoding():
async def iterator():
yield b"\x83g"
yield b"\x83"
yield b"\x89\x83x\x83\x8b"
stream = AsyncIteratorStream(aiterator=iterator())
response = httpx.Response(
200,
headers=[(b"Content-Type", b"text/html; charset=shift-jis")],
stream=stream,
request=REQUEST,
)
await response.aread()
assert "".join(response.text) == "トラベル"
def test_response_picklable():
response = httpx.Response(
200,
content=b"Hello, world!",
request=httpx.Request("GET", "https://example.org"),
)
pickle_response = pickle.loads(pickle.dumps(response))
assert pickle_response.is_closed is True
assert pickle_response.is_stream_consumed is True
assert pickle_response.next_request is None
assert pickle_response.stream is not None
assert pickle_response.content == b"Hello, world!"
assert pickle_response.status_code == 200
assert pickle_response.request.url == response.request.url
assert pickle_response.extensions == {}
assert pickle_response.history == []
def test_read():
response = httpx.Response(
200,
content=b"Hello, world!",
)
assert response.status_code == 200
assert response.text == "Hello, world!"
assert response.encoding is None
assert response.is_closed
content = response.read()
assert content == b"Hello, world!"
assert response.content == b"Hello, world!"
assert response.is_closed
async def post(
self,
url: str,
**kwargs
) -> httpx.Response:
self.calls.append(('POST', url, kwargs))
return httpx.Response(200, request=httpx.Request(
'POST',
url,
params=kwargs.get('params'),
headers=kwargs.get('headers'),
cookies=kwargs.get('cookies'),
data=kwargs.get('data'),
files=kwargs.get('files'),
json=kwargs.get('json'),
))
async def test_streaming():
body = b"test 123"
compressor = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
async def compress(body):
yield compressor.compress(body)
yield compressor.flush()
headers = [(b"Content-Encoding", b"gzip")]
stream = AsyncIteratorStream(aiterator=compress(body))
response = httpx.Response(200,
headers=headers,
stream=stream,
request=REQUEST)
assert not hasattr(response, "body")
assert await response.aread() == body
def test_js_parser():
with open("tests/data/js_links.html") as data_body:
url = "http://perdu.com/"
respx.get(url).mock(return_value=httpx.Response(200, text=data_body.read()))
resp = httpx.get(url)
page = Page(resp)
assert set(page.extra_urls) == {
"http://perdu.com/onload.html",
"http://perdu.com/popup.html",
"http://perdu.com/redir.html",
"http://perdu.com/concat.html",
"http://perdu.com/concat.html?var=value",
"http://perdu.com/link.html",
}
async def test_authorise_without_tokens_legacy(client, tokens_dict):
with mock.patch.object(builtins, "input") as mock_input:
with mock.patch.object(httpx.AsyncClient, "post") as mock_post:
with mock.patch.object(Tokens, "awrite") as mock_write:
mock_write.return_value = None
mock_post.return_value = httpx.Response(
status_code=200, json=tokens_dict, request=mock.Mock()
)
client.legacy_auth = True
tokens = await client.authorise(token_path=TOKENS_PATH, force=True)
mock_post.assert_awaited_once()
mock_write.assert_awaited_once()
mock_input.assert_called_once()
assert isinstance(tokens, Tokens)
def test_delete__makes_request_and_sends_terminal_message(
respx_mock,
make_test_app,
dummy_domain,
cli_runner,
):
job_script_id = 13
delete_route = respx_mock.delete(f"{dummy_domain}/job-scripts/{job_script_id}").mock(
return_value=httpx.Response(httpx.codes.NO_CONTENT),
)
test_app = make_test_app("delete", delete)
result = cli_runner.invoke(test_app, shlex.split(f"delete --id={job_script_id}"))
assert result.exit_code == 0, f"delete failed: {result.stdout}"
assert delete_route.called
assert "JOB SCRIPT DELETE SUCCEEDED"
def test_zlib():
"""
Deflate encoding may use either 'zlib' or 'deflate' in the wild.
https://stackoverflow.com/questions/1838699/how-can-i-decompress-a-gzip-stream-with-zlib#answer-22311297
"""
body = b"test 123"
compressed_body = zlib.compress(body)
headers = [(b"Content-Encoding", b"deflate")]
response = httpx.Response(
200,
headers=headers,
content=compressed_body,
)
assert response.content == body
async def room_get_state_event(
self,
room_id: str,
event_event_type: str,
state_key: str = "",
) -> MatrixResponse:
"""Fetch a state event.
Returns the HTTP method and HTTP path for the request.
Args:
room_id (str): The room id of the room where the state is fetched
from.
event_type (str): The type of the event that will be fetched.
state_key: The key of the state to look up. Defaults to an empty
string.
"""
return MatrixResponse(httpx.Response(status_code=404, json={}))