Пример #1
0
    def test_decode_brotli(self):
        data = brotli.compress(b"foo")

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "br"})
        r.preload_content()
        assert r.data == b"foo"
Пример #2
0
    def test_deflate_streaming_tell_intermediate_point(self):
        # Ensure that ``tell()`` returns the correct number of bytes when
        # part-way through streaming compressed content.
        NUMBER_OF_READS = 10

        class MockCompressedDataReading(BytesIO):
            """
            A BytesIO-like reader returning ``payload`` in ``NUMBER_OF_READS``
            calls to ``read``.
            """
            def __init__(self, payload, payload_part_size):
                self.payloads = [
                    payload[i * payload_part_size:(i + 1) * payload_part_size]
                    for i in range(NUMBER_OF_READS + 1)
                ]
                self.consumed = 0

                assert b"".join(self.payloads) == payload

            def read(self, _):
                # Amount is unused.
                if len(self.payloads) > 0:
                    payload = self.payloads.pop(0)
                    self.consumed += len(payload)
                    return payload
                return b""

            def __iter__(self):
                return self

            def __next__(self):
                if not self.payloads:
                    raise StopIteration()
                return self.read(None)

            next = __next__

        uncompressed_data = zlib.decompress(ZLIB_PAYLOAD)

        payload_part_size = len(ZLIB_PAYLOAD) // NUMBER_OF_READS
        fp = MockCompressedDataReading(ZLIB_PAYLOAD, payload_part_size)
        resp = HTTPResponse(fp, headers={"content-encoding": "deflate"})
        parts = []
        stream = resp.stream(1)

        for part in stream:
            parts.append(part)
            assert resp.tell() == fp.consumed

        end_of_stream = resp.tell()

        with pytest.raises(StopIteration):
            next(stream)

        # Check that the payload is equal to the uncompressed data
        payload = b"".join(parts)
        assert uncompressed_data == payload

        # Check that the end of the stream is in the correct place
        assert len(ZLIB_PAYLOAD) == end_of_stream
Пример #3
0
    def test_close_midstream(self):
        # A mock fp object that wraps a list and allows closing.
        class MockFP(object):
            self.list = None

            def close(self):
                self.list = None

            def __iter__(self):
                return self

            def __next__(self):
                if not self.list:
                    raise StopIteration()
                return self.list.pop(0)

            next = __next__

        data = [b"fo", b"o"]
        fp = MockFP()
        fp.list = data
        resp = HTTPResponse(fp)
        stream = resp.stream()

        assert next(stream) == b"fo"
        resp.close()
        with pytest.raises(StopIteration):
            next(stream)
Пример #4
0
    def test_empty_stream(self):
        fp = BytesIO(b"")
        resp = HTTPResponse(fp)
        stream = resp.stream(decode_content=False)

        with pytest.raises(StopIteration):
            next(stream)
Пример #5
0
 def test_retries(self):
     fp = BytesIO(b"")
     resp = HTTPResponse(fp)
     assert resp.retries is None
     retry = Retry()
     resp = HTTPResponse(fp, retries=retry)
     assert resp.retries == retry
Пример #6
0
    def test_io_bufferedreader(self):
        fp = BytesIO(b"foo")
        resp = HTTPResponse(fp)
        br = BufferedReader(resp)

        assert br.read() == b"foo"

        br.close()
        assert resp.closed

        # HTTPResponse.read() by default closes the response
        # https://github.com/urllib3/urllib3/issues/1305
        fp = BytesIO(b"hello\nworld")
        resp = HTTPResponse(fp)
        with pytest.raises(ValueError) as ctx:
            list(BufferedReader(resp))
        assert str(ctx.value) == "readline of closed file"

        b = b"!tenbytes!"
        fp = BytesIO(b)
        resp = HTTPResponse(fp)
        br = BufferedReader(resp, 5)

        # This is necessary to make sure the "no bytes left" part of `readinto`
        # gets tested.
        assert len(br.read(5)) == 5
        assert len(br.read(5)) == 5
        assert len(br.read(5)) == 0
Пример #7
0
    def test_multi_decoding_deflate_deflate(self):
        data = zlib.compress(zlib.compress(b"foo"))

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "deflate, deflate"})
        r.preload_content()

        assert r.data == b"foo"
Пример #8
0
    def test_reference_read(self):
        fp = BytesIO(b"foo")
        r = HTTPResponse(fp)

        assert r.read(1) == b"f"
        assert r.read(2) == b"oo"
        assert r.read() == b""
        assert r.read() == b""
Пример #9
0
    def test_preload(self):
        fp = BytesIO(b"foo")

        r = HTTPResponse(fp)
        r.preload_content()

        assert fp.tell() == len(b"foo")
        assert r.data == b"foo"
Пример #10
0
    def test_decode_deflate_case_insensitve(self):
        data = zlib.compress(b"foo")

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "DeFlAtE"})
        r.preload_content()

        assert r.data == b"foo"
Пример #11
0
    def test_streaming(self):
        fp = [b"fo", b"o"]
        resp = HTTPResponse(fp)
        stream = resp.stream(decode_content=False)

        assert next(stream) == b"fo"
        assert next(stream) == b"o"
        with pytest.raises(StopIteration):
            next(stream)
Пример #12
0
    def test_double_streaming(self):
        fp = [b"fo", b"o"]
        resp = HTTPResponse(fp)

        stream = list(resp.stream(decode_content=False))
        assert stream == fp

        stream = list(resp.stream(decode_content=False))
        assert stream == []
Пример #13
0
    def test_deflate_streaming(self):
        data = zlib.compress(b"foo")

        fp = BytesIO(data)
        resp = HTTPResponse(fp, headers={"content-encoding": "deflate"})
        stream = resp.stream()

        assert next(stream) == b"foo"
        with pytest.raises(StopIteration):
            next(stream)
Пример #14
0
    def test_multi_decoding_deflate_gzip(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        data = compress.compress(zlib.compress(b"foo"))
        data += compress.flush()

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "deflate, gzip"})
        r.preload_content()

        assert r.data == b"foo"
Пример #15
0
    def test_decode_gzip_multi_member(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()
        data = data * 3

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "gzip"})
        r.preload_content()

        assert r.data == b"foofoofoo"
Пример #16
0
    def test_chunked_decoding_gzip_swallow_garbage(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()
        data = data * 3 + b"foo"

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "gzip"})
        r.preload_content()

        assert r.data == b"foofoofoo"
Пример #17
0
    def test_chunked_decoding_gzip(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "gzip"})

        assert r.read(1) == b"f"
        assert r.read(2) == b"oo"
        assert r.read() == b""
        assert r.read() == b""
Пример #18
0
    def test_deflate2_streaming(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()

        fp = BytesIO(data)
        resp = HTTPResponse(fp, headers={"content-encoding": "deflate"})
        stream = resp.stream()

        assert next(stream) == b"foo"
        with pytest.raises(StopIteration):
            next(stream)
Пример #19
0
    def test_chunked_decoding_deflate(self):
        data = zlib.compress(b"foo")

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "deflate"})

        assert r.read(1) == b"f"
        # Buffer in case we need to switch to the raw stream
        assert r._decoder._data is None
        assert r.read(2) == b"oo"
        assert r.read() == b""
        assert r.read() == b""
Пример #20
0
    def test_chunked_decoding_brotli(self):
        data = brotli.compress(b"foobarbaz")

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "br"})

        ret = b""
        for _ in range(100):
            ret += r.read(1)
            if r.closed:
                break
        assert ret == b"foobarbaz"
Пример #21
0
    def test_chunked_decoding_deflate2(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "deflate"})

        assert r.read(1) == b"f"
        # Once we've decoded data, we just stream to the decoder; no buffering
        assert r._decoder._data is None
        assert r.read(2) == b"oo"
        assert r.read() == b""
        assert r.read() == b""
Пример #22
0
    def test_history(self):
        retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"]))
        assert retry.history == tuple()
        connection_error = ConnectTimeoutError("conntimeout")
        retry = retry.increment("GET", "/test1", None, connection_error)
        history = (RequestHistory("GET", "/test1", connection_error, None,
                                  None), )
        assert retry.history == history

        read_error = ReadTimeoutError(None, "/test2", "read timed out")
        retry = retry.increment("POST", "/test2", None, read_error)
        history = (
            RequestHistory("GET", "/test1", connection_error, None, None),
            RequestHistory("POST", "/test2", read_error, None, None),
        )
        assert retry.history == history

        response = HTTPResponse(status=500)
        retry = retry.increment("GET", "/test3", response, None)
        history = (
            RequestHistory("GET", "/test1", connection_error, None, None),
            RequestHistory("POST", "/test2", read_error, None, None),
            RequestHistory("GET", "/test3", None, 500, None),
        )
        assert retry.history == history
Пример #23
0
    def test_error_message(self):
        retry = Retry(total=0)
        with pytest.raises(MaxRetryError) as e:
            retry = retry.increment(method="GET",
                                    error=ReadTimeoutError(
                                        None, "/", "read timed out"))
        assert "Caused by redirect" not in str(e.value)
        assert str(e.value.reason) == "None: read timed out"

        retry = Retry(total=1)
        with pytest.raises(MaxRetryError) as e:
            retry = retry.increment("POST", "/")
            retry = retry.increment("POST", "/")
        assert "Caused by redirect" not in str(e.value)
        assert isinstance(e.value.reason, ResponseError)
        assert str(e.value.reason) == ResponseError.GENERIC_ERROR

        retry = Retry(total=1)
        response = HTTPResponse(status=500)
        with pytest.raises(MaxRetryError) as e:
            retry = retry.increment("POST", "/", response=response)
            retry = retry.increment("POST", "/", response=response)
        assert "Caused by redirect" not in str(e.value)
        msg = ResponseError.SPECIFIC_ERROR.format(status_code=500)
        assert str(e.value.reason) == msg

        retry = Retry(connect=1)
        with pytest.raises(MaxRetryError) as e:
            retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
            retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
        assert "Caused by redirect" not in str(e.value)
        assert str(e.value.reason) == "conntimeout"
Пример #24
0
    def test_no_preload(self):
        fp = BytesIO(b"foo")

        r = HTTPResponse(fp)

        assert fp.tell() == 0
        assert r.data == b"foo"
        assert fp.tell() == len(b"foo")
Пример #25
0
    def test_streaming_tell(self):
        fp = [b"fo", b"o"]
        resp = HTTPResponse(fp)
        stream = resp.stream(decode_content=False)

        position = 0

        position += len(next(stream))
        assert 2 == position
        assert 2 == resp.tell()

        position += len(next(stream))
        assert 3 == position
        assert 3 == resp.tell()

        with pytest.raises(StopIteration):
            next(stream)
Пример #26
0
    def test_decode_gzip_swallow_garbage(self):
        # When data comes from multiple calls to read(), data after
        # the first zlib error (here triggered by garbage) should be
        # ignored.
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        data = compress.compress(b"foo")
        data += compress.flush()
        data = data * 3 + b"foo"

        fp = BytesIO(data)
        r = HTTPResponse(fp, headers={"content-encoding": "gzip"})
        ret = b""
        for _ in range(100):
            ret += r.read(1)
            if r.closed:
                break

        assert ret == b"foofoofoo"
Пример #27
0
    def test_gzipped_streaming_tell(self):
        compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
        uncompressed_data = b"foo"
        data = compress.compress(uncompressed_data)
        data += compress.flush()

        fp = BytesIO(data)
        resp = HTTPResponse(fp, headers={"content-encoding": "gzip"})
        stream = resp.stream()

        # Read everything
        payload = next(stream)
        assert payload == uncompressed_data

        assert len(data) == resp.tell()

        with pytest.raises(StopIteration):
            next(stream)
Пример #28
0
 def test_status_counter(self):
     resp = HTTPResponse(status=400)
     retry = Retry(status=2)
     retry = retry.increment(response=resp)
     retry = retry.increment(response=resp)
     with pytest.raises(MaxRetryError) as e:
         retry.increment(response=resp)
     assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format(
         status_code=400)
Пример #29
0
 def test_backoff_reset_after_redirect(self):
     retry = Retry(total=100, redirect=5, backoff_factor=0.2)
     retry = retry.increment(method="GET")
     retry = retry.increment(method="GET")
     assert retry.get_backoff_time() == 0.4
     redirect_response = HTTPResponse(status=302,
                                      headers={"location": "test"})
     retry = retry.increment(method="GET", response=redirect_response)
     assert retry.get_backoff_time() == 0
     retry = retry.increment(method="GET")
     retry = retry.increment(method="GET")
     assert retry.get_backoff_time() == 0.4
Пример #30
0
    def test_extract(self):
        request = urllib.request.Request("http://google.com")
        cookiejar = http_cookiejar.CookieJar()
        response = HTTPResponse()

        cookies = [
            "sessionhash=abcabcabcabcab; path=/; HttpOnly",
            "lastvisit=1348253375; expires=Sat, 21-Sep-2050 18:49:35 GMT; path=/",
        ]
        for c in cookies:
            response.headers.add("set-cookie", c)
        cookiejar.extract_cookies(response, request)
        assert len(cookiejar) == len(cookies)