def test_bearer_policy_optionally_enforces_https(): """HTTPS enforcement should be controlled by a keyword argument, and enabled by default""" def assert_option_popped(request, **kwargs): assert "enforce_https" not in kwargs, "BearerTokenCredentialPolicy didn't pop the 'enforce_https' option" credential = Mock(get_token=lambda *_, **__: AccessToken("***", 42)) pipeline = Pipeline( transport=Mock(send=assert_option_popped), policies=[BearerTokenCredentialPolicy(credential, "scope")] ) # by default and when enforce_https=True, the policy should raise when given an insecure request with pytest.raises(ServiceRequestError): pipeline.run(HttpRequest("GET", "http://not.secure")) with pytest.raises(ServiceRequestError): pipeline.run(HttpRequest("GET", "http://not.secure"), enforce_https=True) # when enforce_https=False, an insecure request should pass pipeline.run(HttpRequest("GET", "http://not.secure"), enforce_https=False) # https requests should always pass pipeline.run(HttpRequest("GET", "https://secure"), enforce_https=False) pipeline.run(HttpRequest("GET", "https://secure"), enforce_https=True) pipeline.run(HttpRequest("GET", "https://secure"))
def _batch_send( self, entities, # type: List[TableEntity] *reqs, # type: List[HttpRequest] **kwargs): # (...) -> List[HttpResponse] """Given a series of request, do a Storage batch call.""" # Pop it here, so requests doesn't feel bad about additional kwarg policies = [StorageHeadersPolicy()] changeset = HttpRequest("POST", None) changeset.set_multipart_mixed(*reqs, policies=policies, boundary="changeset_{}".format(uuid4())) request = self._client._client.post( # pylint: disable=protected-access url="https://{}/$batch".format(self._primary_hostname), headers={ "x-ms-version": self.api_version, "DataServiceVersion": "3.0", "MaxDataServiceVersion": "3.0;NetFx", 'Content-Type': 'application/json', 'Accept': 'application/json' }, ) request.set_multipart_mixed( changeset, policies=policies, enforce_https=False, boundary="batch_{}".format(uuid4()), ) pipeline_response = self._client._client._pipeline.run( request, **kwargs) # pylint: disable=protected-access response = pipeline_response.http_response if response.status_code == 403: raise ClientAuthenticationError( message="There was an error authenticating with the service", response=response, ) if response.status_code == 404: raise ResourceNotFoundError( message="The resource could not be found", response=response) if response.status_code == 413: raise RequestTooLargeError(message="The request was too large", response=response) if response.status_code != 202: raise BatchErrorException( message="There is a failure in the batch operation.", response=response, parts=None, ) parts = list(response.parts()) if any(p for p in parts if not 200 <= p.status_code < 300): if any(p for p in parts if p.status_code == 404): raise ResourceNotFoundError( message="The resource could not be found", response=response) if any(p for p in parts if p.status_code == 413): raise RequestTooLargeError(message="The request was too large", response=response) raise BatchErrorException( message="There is a failure in the batch operation.", response=response, parts=parts, ) return list( zip(entities, (extract_batch_part_metadata(p) for p in parts)))
async def test_recursive_multipart_receive(): req0 = HttpRequest("DELETE", "/container0/blob0") internal_req0 = HttpRequest("DELETE", "/container0/blob0") req0.set_multipart_mixed(internal_req0) request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed(req0) class MockResponse(AsyncHttpResponse): def __init__(self, request, body, content_type): super(MockResponse, self).__init__(request, None) self._body = body self.content_type = content_type def body(self): return self._body internal_body_as_str = ( "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" "Content-Type: application/http\r\n" "Content-ID: 0\r\n" "\r\n" "HTTP/1.1 400 Accepted\r\n" "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" "x-ms-version: 2018-11-09\r\n" "\r\n" "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--") body_as_str = ( "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6\r\n" "Content-Type: application/http\r\n" "Content-ID: 0\r\n" "\r\n" "HTTP/1.1 202 Accepted\r\n" "Content-Type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" "\r\n" "{}" "--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6--" ).format(internal_body_as_str) response = MockResponse( request, body_as_str.encode('ascii'), "multipart/mixed; boundary=batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6" ) parts = [] async for part in response.parts(): parts.append(part) assert len(parts) == 1 res0 = parts[0] assert res0.status_code == 202 internal_parts = [] async for part in res0.parts(): internal_parts.append(part) assert len(internal_parts) == 1 internal_response0 = internal_parts[0] assert internal_response0.status_code == 400
async def do(): request = HttpRequest("GET", "https://bing.com/") policies = [UserAgentPolicy("myusergant"), AsyncRedirectPolicy()] async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: return await pipeline.run(request)
async def test_multipart_send_with_multiple_changesets(): transport = MockAsyncHttpTransport() changeset1 = HttpRequest(None, None) changeset1.set_multipart_mixed( HttpRequest("DELETE", "/container0/blob0"), HttpRequest("DELETE", "/container1/blob1"), boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525") changeset2 = HttpRequest(None, None) changeset2.set_multipart_mixed( HttpRequest("DELETE", "/container2/blob2"), HttpRequest("DELETE", "/container3/blob3"), boundary="changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314") request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed( changeset1, changeset2, boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525", ) async with AsyncPipeline(transport) as pipeline: await pipeline.run(request) assert request.body == ( b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: multipart/mixed; boundary=changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 0\r\n' b'\r\n' b'DELETE /container0/blob0 HTTP/1.1\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 1\r\n' b'\r\n' b'DELETE /container1/blob1 HTTP/1.1\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: multipart/mixed; boundary=changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 2\r\n' b'\r\n' b'DELETE /container2/blob2 HTTP/1.1\r\n' b'\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 3\r\n' b'\r\n' b'DELETE /container3/blob3 HTTP/1.1\r\n' b'\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314--\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_request_xml(self): request = HttpRequest("GET", "/") data = ET.Element("root") request.set_xml_body(data) assert request.data == b"<?xml version='1.0' encoding='utf8'?>\n<root />"
def test_repr(self): request = HttpRequest("GET", "hello.com") assert repr(request) == "<HttpRequest [GET], url: 'hello.com'>"
def test_request_url_with_params_with_none_in_list(): request = HttpRequest("GET", "/") request.url = "a/b/c?t=y" with pytest.raises(ValueError): request.format_parameters({"g": ["h",None]})
def _get_request(url, scope, identity_config): # type: (str, str, dict) -> HttpRequest request = HttpRequest("GET", url) request.format_parameters(dict({"api-version": "2019-07-01-preview", "resource": scope}, **identity_config)) return request
def test_multipart_send_with_multiple_changesets(): transport = mock.MagicMock(spec=HttpTransport) header_policy = HeadersPolicy( {'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT'}) changeset1 = HttpRequest(None, None) changeset1.set_multipart_mixed( HttpRequest("DELETE", "/container0/blob0"), HttpRequest("DELETE", "/container1/blob1"), policies=[header_policy], boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525") changeset2 = HttpRequest(None, None) changeset2.set_multipart_mixed( HttpRequest("DELETE", "/container2/blob2"), HttpRequest("DELETE", "/container3/blob3"), policies=[header_policy], boundary="changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314") request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed( changeset1, changeset2, policies=[header_policy], boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525", ) with Pipeline(transport) as pipeline: pipeline.run(request) assert request.body == ( b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: multipart/mixed; boundary=changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 0\r\n' b'\r\n' b'DELETE /container0/blob0 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 1\r\n' b'\r\n' b'DELETE /container1/blob1 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: multipart/mixed; boundary=changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 2\r\n' b'\r\n' b'DELETE /container2/blob2 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 3\r\n' b'\r\n' b'DELETE /container3/blob3 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314--\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_only_default(): request = HttpRequest("GET", "") response = HttpResponse(request, None) error_map = ErrorMap(default_error=ResourceExistsError) with pytest.raises(ResourceExistsError): map_error(401, response, error_map)
def test_error_map_no_default(): request = HttpRequest("GET", "") response = HttpResponse(request, None) error_map = ErrorMap({404: ResourceNotFoundError}) with pytest.raises(ResourceNotFoundError): map_error(404, response, error_map)
def _make_request(self): return PipelineRequest( HttpRequest("CredentialWrapper", "https://fakeurl"), PipelineContext(None))
async def test_bearer_policy_enforces_tls(): credential = Mock() pipeline = AsyncPipeline(transport=Mock(), policies=[AsyncBearerTokenCredentialPolicy(credential, "scope")]) with pytest.raises(ServiceRequestError): await pipeline.run(HttpRequest("GET", "http://not.secure"))
def test_distributed_tracing_policy_with_user_agent(): """Test policy working with user agent.""" with ContextHelper(environ={"AZURE_HTTP_USER_AGENT": "mytools"}): exporter = MockExporter() trace = tracer_module.Tracer(sampler=AlwaysOnSampler(), exporter=exporter) with trace.span("parent"): policy = DistributedTracingPolicy() request = HttpRequest("GET", "http://127.0.0.1") request.headers["x-ms-client-request-id"] = "some client request id" pipeline_request = PipelineRequest(request, PipelineContext(None)) user_agent = UserAgentPolicy() user_agent.on_request(pipeline_request) policy.on_request(pipeline_request) response = HttpResponse(request, None) response.headers = request.headers response.status_code = 202 response.headers["x-ms-request-id"] = "some request id" pipeline_response = PipelineResponse(request, response, PipelineContext(None)) ctx = trace.span_context header = trace.propagator.to_headers(ctx) assert request.headers.get("traceparent") == header.get("traceparent") policy.on_response(pipeline_request, pipeline_response) time.sleep(0.001) policy.on_request(pipeline_request) try: raise ValueError("Transport trouble") except: policy.on_exception(pipeline_request) user_agent.on_response(pipeline_request, pipeline_response) trace.finish() exporter.build_tree() parent = exporter.root network_span = parent.children[0] assert network_span.span_data.name == "/" assert network_span.span_data.attributes.get("http.method") == "GET" assert network_span.span_data.attributes.get("component") == "http" assert network_span.span_data.attributes.get("http.url") == "http://127.0.0.1" assert network_span.span_data.attributes.get("http.user_agent").endswith("mytools") assert network_span.span_data.attributes.get("x-ms-request-id") == "some request id" assert network_span.span_data.attributes.get("x-ms-client-request-id") == "some client request id" assert network_span.span_data.attributes.get("http.status_code") == 202 network_span = parent.children[1] assert network_span.span_data.name == "/" assert network_span.span_data.attributes.get("http.method") == "GET" assert network_span.span_data.attributes.get("component") == "http" assert network_span.span_data.attributes.get("http.url") == "http://127.0.0.1" assert network_span.span_data.attributes.get("http.user_agent").endswith("mytools") assert network_span.span_data.attributes.get("x-ms-client-request-id") == "some client request id" assert network_span.span_data.attributes.get("x-ms-request-id") is None assert network_span.span_data.attributes.get("http.status_code") == 504 # Exception should propagate status for Opencensus assert network_span.span_data.status.message == 'Transport trouble'
async def test_send_data(): async with AsyncioRequestsTransport() as transport: req = HttpRequest('PUT', 'http://httpbin.org/anything', data=b"azerty") response = await transport.send(req) assert json.loads(response.text())['data'] == "azerty"
async def test_policy_updates_cache(): """ It's possible for the challenge returned for a request to change, e.g. when a vault is moved to a new tenant. When the policy receives a 401, it should update the cached challenge for the requested URL, if one exists. """ url = get_random_url() first_scope = "https://first-scope" first_token = "first-scope-token" second_scope = "https://second-scope" second_token = "second-scope-token" challenge_fmt = 'Bearer authorization="https://login.authority.net/tenant", resource={}' # mocking a tenant change: # 1. first request -> respond with challenge # 2. second request should be authorized according to the challenge # 3. third request should match the second (using a cached access token) # 4. fourth request should also match the second -> respond with a new challenge # 5. fifth request should be authorized according to the new challenge # 6. sixth request should match the fifth transport = async_validating_transport( requests=( Request(url), Request(url, required_headers={ "Authorization": "Bearer {}".format(first_token) }), Request(url, required_headers={ "Authorization": "Bearer {}".format(first_token) }), Request(url, required_headers={ "Authorization": "Bearer {}".format(first_token) }), Request(url, required_headers={ "Authorization": "Bearer {}".format(second_token) }), Request(url, required_headers={ "Authorization": "Bearer {}".format(second_token) }), ), responses=( mock_response(status_code=401, headers={ "WWW-Authenticate": challenge_fmt.format(first_scope) }), mock_response(status_code=200), mock_response(status_code=200), mock_response(status_code=401, headers={ "WWW-Authenticate": challenge_fmt.format(second_scope) }), mock_response(status_code=200), mock_response(status_code=200), ), ) token = AccessToken(first_token, time.time() + 3600) async def get_token(*_, **__): return token credential = Mock(get_token=Mock(wraps=get_token)) pipeline = AsyncPipeline( policies=[AsyncChallengeAuthPolicy(credential=credential)], transport=transport) # policy should complete and cache the first challenge and access token for _ in range(2): await pipeline.run(HttpRequest("GET", url)) assert credential.get_token.call_count == 1 # The next request will receive a new challenge. The policy should handle it and update caches. token = AccessToken(second_token, time.time() + 3600) for _ in range(2): await pipeline.run(HttpRequest("GET", url)) assert credential.get_token.call_count == 2
def get_attestation_token(attestation_uri): request = HttpRequest("GET", "{}/generate-test-token".format(attestation_uri)) with Pipeline(transport=RequestsTransport()) as pipeline: response = pipeline.run(request) return json.loads(response.http_response.text())["token"]
def test_request_url_with_params_with_none(self): request = HttpRequest("GET", "/") request.url = "a/b/c?t=y" with pytest.raises(ValueError): request.format_parameters({"g": None})
async def test_connection_error_response(): class MockSession(object): def __init__(self): self.auto_decompress = True @property def auto_decompress(self): return self.auto_decompress class MockTransport(AsyncHttpTransport): def __init__(self): self._count = 0 self.session = MockSession async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send(self, request, **kwargs): request = HttpRequest('GET', 'http://127.0.0.1/') response = AsyncHttpResponse(request, None) response.status_code = 200 return response class MockContent(): def __init__(self): self._first = True async def read(self, block_size): if self._first: self._first = False raise ConnectionError return None class MockInternalResponse(): def __init__(self): self.headers = {} self.content = MockContent() async def close(self): pass class AsyncMock(mock.MagicMock): async def __call__(self, *args, **kwargs): return super(AsyncMock, self).__call__(*args, **kwargs) http_request = HttpRequest('GET', 'http://127.0.0.1/') pipeline = AsyncPipeline(MockTransport()) http_response = AsyncHttpResponse(http_request, None) http_response.internal_response = MockInternalResponse() stream = AioHttpStreamDownloadGenerator(pipeline, http_response, decompress=False) with mock.patch('asyncio.sleep', new_callable=AsyncMock): with pytest.raises(ConnectionError): await stream.__anext__()
# HttpXTransport does not support stream data: # File "lib\site-packages\httpx\_content_streams.py", line 404, in encode # raise TypeError(f"Unexpected type for 'data', {type(data)!r}") # TypeError: Unexpected type for 'data', <class '__main__.LargeStream'> # # start = time.perf_counter() # req = HttpRequest("PUT", url, data=LargeStream(size), headers=headers) # resp = pipelinex.run(req) # stop = time.perf_counter() # duration = stop - start # mbps = ((size / duration) * 8) / (1024 * 1024) # print(f'[PipelineX, stream] Put {size:,} bytes in {duration:.2f} seconds ({mbps:.2f} Mbps), Response={resp.http_response.status_code}') start = time.perf_counter() req = HttpRequest("PUT", url, data=array, headers=headers) resp = pipelinex.run(req) stop = time.perf_counter() duration = stop - start mbps = ((size / duration) * 8) / (1024 * 1024) print( f'[PipelineX, array] Put {size:,} bytes in {duration:.2f} seconds ({mbps:.2f} Mbps), Response={resp.http_response.status_code}' ) start = time.perf_counter() req = HttpRequest("PUT", url, data=LargeStream(size), headers=headers) resp = pipeline.run(req) stop = time.perf_counter() duration = stop - start mbps = ((size / duration) * 8) / (1024 * 1024) print(
async def send(self, request, **kwargs): request = HttpRequest('GET', 'http://127.0.0.1/') response = AsyncHttpResponse(request, None) response.status_code = 200 return response
async def _batch_send( self, entities, # type: List[TableEntity] *reqs: "HttpRequest", **kwargs ): """Given a series of request, do a Storage batch call.""" # Pop it here, so requests doesn't feel bad about additional kwarg raise_on_any_failure = kwargs.pop("raise_on_any_failure", True) policies = [StorageHeadersPolicy()] changeset = HttpRequest("POST", None) changeset.set_multipart_mixed( *reqs, policies=policies, boundary="changeset_{}".format(uuid4()) ) request = self._client._client.post( # pylint: disable=protected-access url="https://{}/$batch".format(self._primary_hostname), headers={ "x-ms-version": self.api_version, "DataServiceVersion": "3.0", "MaxDataServiceVersion": "3.0;NetFx", }, ) request.set_multipart_mixed( changeset, policies=policies, enforce_https=False, boundary="batch_{}".format(uuid4()), ) pipeline_response = await self._pipeline.run(request, **kwargs) response = pipeline_response.http_response if response.status_code == 403: raise ClientAuthenticationError( message="There was an error authenticating with the service", response=response, ) if response.status_code == 404: raise ResourceNotFoundError( message="The resource could not be found", response=response ) if response.status_code != 202: raise BatchErrorException( message="There is a failure in the batch operation.", response=response, parts=None, ) parts_iter = response.parts() parts = [] async for p in parts_iter: parts.append(p) transaction_result = BatchTransactionResult(reqs, parts, entities) if raise_on_any_failure: if any(p for p in parts if not 200 <= p.status_code < 300): if any(p for p in parts if p.status_code == 404): raise ResourceNotFoundError( message="The resource could not be found", response=response ) raise BatchErrorException( message="There is a failure in the batch operation.", response=response, parts=parts, ) return transaction_result
def test_raw_deserializer(): raw_deserializer = ContentDecodePolicy() context = PipelineContext(None, stream=False) universal_request = HttpRequest('GET', 'http://127.0.0.1/') request = PipelineRequest(universal_request, context) def build_response(body, content_type=None): class MockResponse(HttpResponse): def __init__(self, body, content_type): super(MockResponse, self).__init__(None, None) self._body = body self.content_type = content_type def body(self): return self._body return PipelineResponse(request, MockResponse(body, content_type), context) response = build_response(b"<groot/>", content_type="application/xml") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result.tag == "groot" response = build_response(b"\xef\xbb\xbf<utf8groot/>", content_type="application/xml") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result.tag == "utf8groot" # The basic deserializer works with unicode XML response = build_response(u'<groot language="français"/>'.encode('utf-8'), content_type="application/xml") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result.attrib["language"] == u"français" # Catch some weird situation where content_type is XML, but content is JSON response = build_response(b'{"ugly": true}', content_type="application/xml") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["ugly"] is True # Be sure I catch the correct exception if it's neither XML nor JSON response = build_response(b'gibberish', content_type="application/xml") with pytest.raises(DecodeError) as err: raw_deserializer.on_response(request, response) assert err.value.response is response.http_response response = build_response(b'{{gibberish}}', content_type="application/xml") with pytest.raises(DecodeError) as err: raw_deserializer.on_response(request, response) assert err.value.response is response.http_response # Simple JSON response = build_response(b'{"success": true}', content_type="application/json") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["success"] is True # Simple JSON with BOM response = build_response(b'\xef\xbb\xbf{"success": true}', content_type="application/json") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["success"] is True # Simple JSON with complex content_type response = build_response( b'{"success": true}', content_type="application/vnd.microsoft.appconfig.kv.v1+json") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["success"] is True # Simple JSON with complex content_type, v2 response = build_response( b'{"success": true}', content_type="text/vnd.microsoft.appconfig.kv.v1+json") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["success"] is True # For compat, if no content-type, decode JSON response = build_response(b'"data"') raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == "data" # Let text/plain let through response = build_response(b'I am groot', content_type="text/plain") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == "I am groot" # Let text/plain let through + BOM response = build_response(b'\xef\xbb\xbfI am groot', content_type="text/plain") raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == "I am groot" # Try with a mock of requests req_response = requests.Response() req_response.headers["content-type"] = "application/json" req_response._content = b'{"success": true}' req_response._content_consumed = True response = PipelineResponse(None, RequestsTransportResponse(None, req_response), PipelineContext(None, stream=False)) raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result["success"] is True # I can enable it per request request.context.options['response_encoding'] = 'utf-8' response = build_response(b'\xc3\xa9', content_type="text/plain") raw_deserializer.on_request(request) raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == u"é" assert response.context["response_encoding"] == "utf-8" del request.context['response_encoding'] # I can enable it globally raw_deserializer = ContentDecodePolicy(response_encoding="utf-8") response = build_response(b'\xc3\xa9', content_type="text/plain") raw_deserializer.on_request(request) raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == u"é" assert response.context["response_encoding"] == "utf-8" del request.context['response_encoding'] # Per request is more important request.context.options['response_encoding'] = 'utf-8-sig' response = build_response(b'\xc3\xa9', content_type="text/plain") raw_deserializer.on_request(request) raw_deserializer.on_response(request, response) result = response.context["deserialized_data"] assert result == u"é" assert response.context["response_encoding"] == "utf-8-sig" del request.context['response_encoding']
async def test_multipart_receive_with_combination_changeset_last(): changeset = HttpRequest(None, None) changeset.set_multipart_mixed(HttpRequest("DELETE", "/container1/blob1"), HttpRequest("DELETE", "/container2/blob2")) request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed(HttpRequest("DELETE", "/container0/blob0"), changeset) body_as_bytes = ( b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 2\r\n' b'\r\n' b'HTTP/1.1 200\r\n' b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n' b'x-ms-version: 2018-11-09\r\n' b'\r\n' b'\r\n' b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n' b'Content-Type: multipart/mixed; boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525"\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 0\r\n' b'\r\n' b'HTTP/1.1 202\r\n' b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n' b'x-ms-version: 2018-11-09\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 1\r\n' b'\r\n' b'HTTP/1.1 404\r\n' b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n' b'x-ms-version: 2018-11-09\r\n' b'\r\n' b'\r\n' b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n' b'\r\n' b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--\r\n') response = MockResponse( request, body_as_bytes, "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" ) parts = [] async for part in response.parts(): parts.append(part) assert len(parts) == 3 assert parts[0].status_code == 200 assert parts[1].status_code == 202 assert parts[2].status_code == 404
def test_http_logger(): class MockHandler(logging.Handler): def __init__(self): super(MockHandler, self).__init__() self.messages = [] def reset(self): self.messages = [] def emit(self, record): self.messages.append(record) mock_handler = MockHandler() logger = logging.getLogger("testlogger") logger.addHandler(mock_handler) logger.setLevel(logging.DEBUG) policy = HttpLoggingPolicy(logger=logger) universal_request = HttpRequest('GET', 'http://127.0.0.1/') http_response = HttpResponse(universal_request, None) http_response.status_code = 202 request = PipelineRequest(universal_request, PipelineContext(None)) # Basics policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) assert all(m.levelname == 'INFO' for m in mock_handler.messages) assert len(mock_handler.messages) == 5 assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' mock_handler.reset() # Let's make this request a failure, retried twice policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) assert all(m.levelname == 'INFO' for m in mock_handler.messages) assert len(mock_handler.messages) == 10 assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' mock_handler.reset() # Headers and query parameters policy.allowed_query_params = ['country'] universal_request.headers = { "Accept": "Caramel", "Hate": "Chocolat", } http_response.headers = { "Content-Type": "Caramel", "HateToo": "Chocolat", } universal_request.url = "http://127.0.0.1/?country=france&city=aix" policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) assert all(m.levelname == 'INFO' for m in mock_handler.messages) assert len(mock_handler.messages) == 9 assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/?country=france&city=REDACTED'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == "Request headers:" # Dict not ordered in Python, exact logging order doesn't matter assert set([ mock_handler.messages[3].message, mock_handler.messages[4].message ]) == set([" 'Accept': 'Caramel'", " 'Hate': 'REDACTED'"]) assert mock_handler.messages[5].message == "Response status: 202" assert mock_handler.messages[6].message == "Response headers:" # Dict not ordered in Python, exact logging order doesn't matter assert set([ mock_handler.messages[7].message, mock_handler.messages[8].message ]) == set([" 'Content-Type': 'Caramel'", " 'HateToo': 'REDACTED'"]) mock_handler.reset()
async def test_multipart_send(): # transport = mock.MagicMock(spec=AsyncHttpTransport) # MagicMock support async cxt manager only after 3.8 # https://github.com/python/cpython/pull/9296 class MockAsyncHttpTransport(AsyncHttpTransport): async def __aenter__(self): return self async def __aexit__(self, *args): pass async def open(self): pass async def close(self): pass async def send(self, request, **kwargs): pass transport = MockAsyncHttpTransport() class RequestPolicy(object): async def on_request(self, request): # type: (PipelineRequest) -> None request.http_request.headers[ 'x-ms-date'] = 'Thu, 14 Jun 2018 16:46:54 GMT' req0 = HttpRequest("DELETE", "/container0/blob0") req1 = HttpRequest("DELETE", "/container1/blob1") request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed( req0, req1, policies=[RequestPolicy()], boundary= "batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic ) async with AsyncPipeline(transport) as pipeline: await pipeline.run(request) assert request.body == ( b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 0\r\n' b'\r\n' b'DELETE /container0/blob0 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n' b'Content-Type: application/http\r\n' b'Content-Transfer-Encoding: binary\r\n' b'Content-ID: 1\r\n' b'\r\n' b'DELETE /container1/blob1 HTTP/1.1\r\n' b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n' b'\r\n' b'\r\n' b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_http_logger_operation_level(): class MockHandler(logging.Handler): def __init__(self): super(MockHandler, self).__init__() self.messages = [] def reset(self): self.messages = [] def emit(self, record): self.messages.append(record) mock_handler = MockHandler() logger = logging.getLogger("testlogger") logger.addHandler(mock_handler) logger.setLevel(logging.DEBUG) policy = HttpLoggingPolicy() kwargs = {'logger': logger} universal_request = HttpRequest('GET', 'http://127.0.0.1/') http_response = HttpResponse(universal_request, None) http_response.status_code = 202 request = PipelineRequest(universal_request, PipelineContext(None, **kwargs)) # Basics policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) assert all(m.levelname == 'INFO' for m in mock_handler.messages) assert len(mock_handler.messages) == 5 assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' mock_handler.reset() # Let's make this request a failure, retried twice request = PipelineRequest(universal_request, PipelineContext(None, **kwargs)) policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) policy.on_request(request) response = PipelineResponse(request, http_response, request.context) policy.on_response(request, response) assert all(m.levelname == 'INFO' for m in mock_handler.messages) assert len(mock_handler.messages) == 10 assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' assert mock_handler.messages[ 0].message == "Request URL: 'http://127.0.0.1/'" assert mock_handler.messages[1].message == "Request method: 'GET'" assert mock_handler.messages[2].message == 'Request headers:' assert mock_handler.messages[3].message == 'Response status: 202' assert mock_handler.messages[4].message == 'Response headers:' mock_handler.reset()
async def test_multipart_receive(): class MockResponse(AsyncHttpResponse): def __init__(self, request, body, content_type): super(MockResponse, self).__init__(request, None) self._body = body self.content_type = content_type def body(self): return self._body class ResponsePolicy(object): def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None response.http_response.headers['x-ms-fun'] = 'true' class AsyncResponsePolicy(object): async def on_response(self, request, response): # type: (PipelineRequest, PipelineResponse) -> None response.http_response.headers['x-ms-async-fun'] = 'true' req0 = HttpRequest("DELETE", "/container0/blob0") req1 = HttpRequest("DELETE", "/container1/blob1") request = HttpRequest("POST", "http://account.blob.core.windows.net/?comp=batch") request.set_multipart_mixed( req0, req1, policies=[ResponsePolicy(), AsyncResponsePolicy()]) body_as_str = ( "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" "Content-Type: application/http\r\n" "Content-ID: 0\r\n" "\r\n" "HTTP/1.1 202 Accepted\r\n" "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n" "x-ms-version: 2018-11-09\r\n" "\r\n" "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n" "Content-Type: application/http\r\n" "Content-ID: 2\r\n" "\r\n" "HTTP/1.1 404 The specified blob does not exist.\r\n" "x-ms-error-code: BlobNotFound\r\n" "x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\r\n" "x-ms-version: 2018-11-09\r\n" "Content-Length: 216\r\n" "Content-Type: application/xml\r\n" "\r\n" '<?xml version="1.0" encoding="utf-8"?>\r\n' "<Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.\r\n" "RequestId:778fdc83-801e-0000-62ff-0334671e2852\r\n" "Time:2018-06-14T16:46:54.6040685Z</Message></Error>\r\n" "--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--") response = MockResponse( request, body_as_str.encode('ascii'), "multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed" ) parts = [] async for part in response.parts(): parts.append(part) assert len(parts) == 2 res0 = parts[0] assert res0.status_code == 202 assert res0.headers['x-ms-fun'] == 'true' assert res0.headers['x-ms-async-fun'] == 'true' res1 = parts[1] assert res1.status_code == 404 assert res1.headers['x-ms-fun'] == 'true' assert res1.headers['x-ms-async-fun'] == 'true'
def test_no_log(mock_http_logger): universal_request = HttpRequest('GET', 'http://127.0.0.1/') request = PipelineRequest(universal_request, PipelineContext(None)) http_logger = NetworkTraceLoggingPolicy() response = PipelineResponse(request, HttpResponse(universal_request, None), request.context) # By default, no log handler for HTTP http_logger.on_request(request) mock_http_logger.debug.assert_not_called() http_logger.on_response(request, response) mock_http_logger.debug.assert_not_called() mock_http_logger.reset_mock() # I can enable it per request request.context.options['logging_enable'] = True http_logger.on_request(request) assert mock_http_logger.debug.call_count >= 1 mock_http_logger.reset_mock() request.context.options['logging_enable'] = True http_logger.on_response(request, response) assert mock_http_logger.debug.call_count >= 1 mock_http_logger.reset_mock() # I can enable it per request (bool value should be honored) request.context.options['logging_enable'] = False http_logger.on_request(request) mock_http_logger.debug.assert_not_called() request.context.options['logging_enable'] = False http_logger.on_response(request, response) mock_http_logger.debug.assert_not_called() mock_http_logger.reset_mock() # I can enable it globally request.context.options = {} http_logger.enable_http_logger = True http_logger.on_request(request) assert mock_http_logger.debug.call_count >= 1 http_logger.on_response(request, response) assert mock_http_logger.debug.call_count >= 1 mock_http_logger.reset_mock() # I can enable it globally and override it locally http_logger.enable_http_logger = True request.context.options['logging_enable'] = False http_logger.on_request(request) mock_http_logger.debug.assert_not_called() response.context['logging_enable'] = False http_logger.on_response(request, response) mock_http_logger.debug.assert_not_called() mock_http_logger.reset_mock() # Let's make this request a failure, retried twice request.context.options['logging_enable'] = True http_logger.on_request(request) http_logger.on_response(request, response) first_count = mock_http_logger.debug.call_count assert first_count >= 1 http_logger.on_request(request) http_logger.on_response(request, response) second_count = mock_http_logger.debug.call_count assert second_count == first_count * 2