def test_bearer_policy_optionally_enforces_https():
"""HTTPS enforcement should be controlled by a keyword argument, and enabled by default"""
def assert_option_popped(request, **kwargs):
assert "enforce_https" not in kwargs, "BearerTokenCredentialPolicy didn't pop the 'enforce_https' option"
credential = Mock(get_token=lambda *_, **__: AccessToken("***", 42))
pipeline = Pipeline(
transport=Mock(send=assert_option_popped), policies=[BearerTokenCredentialPolicy(credential, "scope")]
)
# by default and when enforce_https=True, the policy should raise when given an insecure request
with pytest.raises(ServiceRequestError):
pipeline.run(HttpRequest("GET", "http://not.secure"))
with pytest.raises(ServiceRequestError):
pipeline.run(HttpRequest("GET", "http://not.secure"), enforce_https=True)
# when enforce_https=False, an insecure request should pass
pipeline.run(HttpRequest("GET", "http://not.secure"), enforce_https=False)
# https requests should always pass
pipeline.run(HttpRequest("GET", "https://secure"), enforce_https=False)
pipeline.run(HttpRequest("GET", "https://secure"), enforce_https=True)
pipeline.run(HttpRequest("GET", "https://secure"))
def _batch_send(
self,
entities, # type: List[TableEntity]
*reqs, # type: List[HttpRequest]
**kwargs):
# (...) -> List[HttpResponse]
"""Given a series of request, do a Storage batch call."""
# Pop it here, so requests doesn't feel bad about additional kwarg
policies = [StorageHeadersPolicy()]
changeset = HttpRequest("POST", None)
changeset.set_multipart_mixed(*reqs,
policies=policies,
boundary="changeset_{}".format(uuid4()))
request = self._client._client.post( # pylint: disable=protected-access
url="https://{}/$batch".format(self._primary_hostname),
headers={
"x-ms-version": self.api_version,
"DataServiceVersion": "3.0",
"MaxDataServiceVersion": "3.0;NetFx",
'Content-Type': 'application/json',
'Accept': 'application/json'
},
)
request.set_multipart_mixed(
changeset,
policies=policies,
enforce_https=False,
boundary="batch_{}".format(uuid4()),
)
pipeline_response = self._client._client._pipeline.run(
request, **kwargs) # pylint: disable=protected-access
response = pipeline_response.http_response
if response.status_code == 403:
raise ClientAuthenticationError(
message="There was an error authenticating with the service",
response=response,
)
if response.status_code == 404:
raise ResourceNotFoundError(
message="The resource could not be found", response=response)
if response.status_code == 413:
raise RequestTooLargeError(message="The request was too large",
response=response)
if response.status_code != 202:
raise BatchErrorException(
message="There is a failure in the batch operation.",
response=response,
parts=None,
)
parts = list(response.parts())
if any(p for p in parts if not 200 <= p.status_code < 300):
if any(p for p in parts if p.status_code == 404):
raise ResourceNotFoundError(
message="The resource could not be found",
response=response)
if any(p for p in parts if p.status_code == 413):
raise RequestTooLargeError(message="The request was too large",
response=response)
raise BatchErrorException(
message="There is a failure in the batch operation.",
response=response,
parts=parts,
)
return list(
zip(entities, (extract_batch_part_metadata(p) for p in parts)))
async def test_recursive_multipart_receive():
req0 = HttpRequest("DELETE", "/container0/blob0")
internal_req0 = HttpRequest("DELETE", "/container0/blob0")
req0.set_multipart_mixed(internal_req0)
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(req0)
class MockResponse(AsyncHttpResponse):
def __init__(self, request, body, content_type):
super(MockResponse, self).__init__(request, None)
self._body = body
self.content_type = content_type
def body(self):
return self._body
internal_body_as_str = (
"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n"
"Content-Type: application/http\r\n"
"Content-ID: 0\r\n"
"\r\n"
"HTTP/1.1 400 Accepted\r\n"
"x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n"
"x-ms-version: 2018-11-09\r\n"
"\r\n"
"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--")
body_as_str = (
"--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6\r\n"
"Content-Type: application/http\r\n"
"Content-ID: 0\r\n"
"\r\n"
"HTTP/1.1 202 Accepted\r\n"
"Content-Type: multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n"
"\r\n"
"{}"
"--batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6--"
).format(internal_body_as_str)
response = MockResponse(
request, body_as_str.encode('ascii'),
"multipart/mixed; boundary=batchresponse_8d5f5bcd-2cb5-44bb-91b5-e9a722e68cb6"
)
parts = []
async for part in response.parts():
parts.append(part)
assert len(parts) == 1
res0 = parts[0]
assert res0.status_code == 202
internal_parts = []
async for part in res0.parts():
internal_parts.append(part)
assert len(internal_parts) == 1
internal_response0 = internal_parts[0]
assert internal_response0.status_code == 400
async def do():
request = HttpRequest("GET", "https://bing.com/")
policies = [UserAgentPolicy("myusergant"), AsyncRedirectPolicy()]
async with AsyncPipeline(TrioRequestsTransport(),
policies=policies) as pipeline:
return await pipeline.run(request)
async def test_multipart_send_with_multiple_changesets():
transport = MockAsyncHttpTransport()
changeset1 = HttpRequest(None, None)
changeset1.set_multipart_mixed(
HttpRequest("DELETE", "/container0/blob0"),
HttpRequest("DELETE", "/container1/blob1"),
boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525")
changeset2 = HttpRequest(None, None)
changeset2.set_multipart_mixed(
HttpRequest("DELETE", "/container2/blob2"),
HttpRequest("DELETE", "/container3/blob3"),
boundary="changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314")
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(
changeset1,
changeset2,
boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525",
)
async with AsyncPipeline(transport) as pipeline:
await pipeline.run(request)
assert request.body == (
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: multipart/mixed; boundary=changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 0\r\n'
b'\r\n'
b'DELETE /container0/blob0 HTTP/1.1\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 1\r\n'
b'\r\n'
b'DELETE /container1/blob1 HTTP/1.1\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: multipart/mixed; boundary=changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 2\r\n'
b'\r\n'
b'DELETE /container2/blob2 HTTP/1.1\r\n'
b'\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 3\r\n'
b'\r\n'
b'DELETE /container3/blob3 HTTP/1.1\r\n'
b'\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314--\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_request_xml(self):
request = HttpRequest("GET", "/")
data = ET.Element("root")
request.set_xml_body(data)
assert request.data == b"<?xml version='1.0' encoding='utf8'?>\n<root />"
def test_repr(self):
request = HttpRequest("GET", "hello.com")
assert repr(request) == "<HttpRequest [GET], url: 'hello.com'>"
def test_request_url_with_params_with_none_in_list():
request = HttpRequest("GET", "/")
request.url = "a/b/c?t=y"
with pytest.raises(ValueError):
request.format_parameters({"g": ["h",None]})
def _get_request(url, scope, identity_config):
# type: (str, str, dict) -> HttpRequest
request = HttpRequest("GET", url)
request.format_parameters(dict({"api-version": "2019-07-01-preview", "resource": scope}, **identity_config))
return request
def test_multipart_send_with_multiple_changesets():
transport = mock.MagicMock(spec=HttpTransport)
header_policy = HeadersPolicy(
{'x-ms-date': 'Thu, 14 Jun 2018 16:46:54 GMT'})
changeset1 = HttpRequest(None, None)
changeset1.set_multipart_mixed(
HttpRequest("DELETE", "/container0/blob0"),
HttpRequest("DELETE", "/container1/blob1"),
policies=[header_policy],
boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525")
changeset2 = HttpRequest(None, None)
changeset2.set_multipart_mixed(
HttpRequest("DELETE", "/container2/blob2"),
HttpRequest("DELETE", "/container3/blob3"),
policies=[header_policy],
boundary="changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314")
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(
changeset1,
changeset2,
policies=[header_policy],
boundary="batch_357de4f7-6d0b-4e02-8cd2-6361411a9525",
)
with Pipeline(transport) as pipeline:
pipeline.run(request)
assert request.body == (
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: multipart/mixed; boundary=changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 0\r\n'
b'\r\n'
b'DELETE /container0/blob0 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 1\r\n'
b'\r\n'
b'DELETE /container1/blob1 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: multipart/mixed; boundary=changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 2\r\n'
b'\r\n'
b'DELETE /container2/blob2 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 3\r\n'
b'\r\n'
b'DELETE /container3/blob3 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--changeset_8b9e487e-a353-4dcb-a6f4-0688191e0314--\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_only_default():
request = HttpRequest("GET", "")
response = HttpResponse(request, None)
error_map = ErrorMap(default_error=ResourceExistsError)
with pytest.raises(ResourceExistsError):
map_error(401, response, error_map)
def test_error_map_no_default():
request = HttpRequest("GET", "")
response = HttpResponse(request, None)
error_map = ErrorMap({404: ResourceNotFoundError})
with pytest.raises(ResourceNotFoundError):
map_error(404, response, error_map)
def _make_request(self):
return PipelineRequest(
HttpRequest("CredentialWrapper", "https://fakeurl"),
PipelineContext(None))
async def test_bearer_policy_enforces_tls():
credential = Mock()
pipeline = AsyncPipeline(transport=Mock(), policies=[AsyncBearerTokenCredentialPolicy(credential, "scope")])
with pytest.raises(ServiceRequestError):
await pipeline.run(HttpRequest("GET", "http://not.secure"))
def test_distributed_tracing_policy_with_user_agent():
"""Test policy working with user agent."""
with ContextHelper(environ={"AZURE_HTTP_USER_AGENT": "mytools"}):
exporter = MockExporter()
trace = tracer_module.Tracer(sampler=AlwaysOnSampler(), exporter=exporter)
with trace.span("parent"):
policy = DistributedTracingPolicy()
request = HttpRequest("GET", "http://127.0.0.1")
request.headers["x-ms-client-request-id"] = "some client request id"
pipeline_request = PipelineRequest(request, PipelineContext(None))
user_agent = UserAgentPolicy()
user_agent.on_request(pipeline_request)
policy.on_request(pipeline_request)
response = HttpResponse(request, None)
response.headers = request.headers
response.status_code = 202
response.headers["x-ms-request-id"] = "some request id"
pipeline_response = PipelineResponse(request, response, PipelineContext(None))
ctx = trace.span_context
header = trace.propagator.to_headers(ctx)
assert request.headers.get("traceparent") == header.get("traceparent")
policy.on_response(pipeline_request, pipeline_response)
time.sleep(0.001)
policy.on_request(pipeline_request)
try:
raise ValueError("Transport trouble")
except:
policy.on_exception(pipeline_request)
user_agent.on_response(pipeline_request, pipeline_response)
trace.finish()
exporter.build_tree()
parent = exporter.root
network_span = parent.children[0]
assert network_span.span_data.name == "/"
assert network_span.span_data.attributes.get("http.method") == "GET"
assert network_span.span_data.attributes.get("component") == "http"
assert network_span.span_data.attributes.get("http.url") == "http://127.0.0.1"
assert network_span.span_data.attributes.get("http.user_agent").endswith("mytools")
assert network_span.span_data.attributes.get("x-ms-request-id") == "some request id"
assert network_span.span_data.attributes.get("x-ms-client-request-id") == "some client request id"
assert network_span.span_data.attributes.get("http.status_code") == 202
network_span = parent.children[1]
assert network_span.span_data.name == "/"
assert network_span.span_data.attributes.get("http.method") == "GET"
assert network_span.span_data.attributes.get("component") == "http"
assert network_span.span_data.attributes.get("http.url") == "http://127.0.0.1"
assert network_span.span_data.attributes.get("http.user_agent").endswith("mytools")
assert network_span.span_data.attributes.get("x-ms-client-request-id") == "some client request id"
assert network_span.span_data.attributes.get("x-ms-request-id") is None
assert network_span.span_data.attributes.get("http.status_code") == 504
# Exception should propagate status for Opencensus
assert network_span.span_data.status.message == 'Transport trouble'
async def test_send_data():
async with AsyncioRequestsTransport() as transport:
req = HttpRequest('PUT', 'http://httpbin.org/anything', data=b"azerty")
response = await transport.send(req)
assert json.loads(response.text())['data'] == "azerty"
async def test_policy_updates_cache():
"""
It's possible for the challenge returned for a request to change, e.g. when a vault is moved to a new tenant.
When the policy receives a 401, it should update the cached challenge for the requested URL, if one exists.
"""
url = get_random_url()
first_scope = "https://first-scope"
first_token = "first-scope-token"
second_scope = "https://second-scope"
second_token = "second-scope-token"
challenge_fmt = 'Bearer authorization="https://login.authority.net/tenant", resource={}'
# mocking a tenant change:
# 1. first request -> respond with challenge
# 2. second request should be authorized according to the challenge
# 3. third request should match the second (using a cached access token)
# 4. fourth request should also match the second -> respond with a new challenge
# 5. fifth request should be authorized according to the new challenge
# 6. sixth request should match the fifth
transport = async_validating_transport(
requests=(
Request(url),
Request(url,
required_headers={
"Authorization": "Bearer {}".format(first_token)
}),
Request(url,
required_headers={
"Authorization": "Bearer {}".format(first_token)
}),
Request(url,
required_headers={
"Authorization": "Bearer {}".format(first_token)
}),
Request(url,
required_headers={
"Authorization": "Bearer {}".format(second_token)
}),
Request(url,
required_headers={
"Authorization": "Bearer {}".format(second_token)
}),
),
responses=(
mock_response(status_code=401,
headers={
"WWW-Authenticate":
challenge_fmt.format(first_scope)
}),
mock_response(status_code=200),
mock_response(status_code=200),
mock_response(status_code=401,
headers={
"WWW-Authenticate":
challenge_fmt.format(second_scope)
}),
mock_response(status_code=200),
mock_response(status_code=200),
),
)
token = AccessToken(first_token, time.time() + 3600)
async def get_token(*_, **__):
return token
credential = Mock(get_token=Mock(wraps=get_token))
pipeline = AsyncPipeline(
policies=[AsyncChallengeAuthPolicy(credential=credential)],
transport=transport)
# policy should complete and cache the first challenge and access token
for _ in range(2):
await pipeline.run(HttpRequest("GET", url))
assert credential.get_token.call_count == 1
# The next request will receive a new challenge. The policy should handle it and update caches.
token = AccessToken(second_token, time.time() + 3600)
for _ in range(2):
await pipeline.run(HttpRequest("GET", url))
assert credential.get_token.call_count == 2
def get_attestation_token(attestation_uri):
request = HttpRequest("GET",
"{}/generate-test-token".format(attestation_uri))
with Pipeline(transport=RequestsTransport()) as pipeline:
response = pipeline.run(request)
return json.loads(response.http_response.text())["token"]
def test_request_url_with_params_with_none(self):
request = HttpRequest("GET", "/")
request.url = "a/b/c?t=y"
with pytest.raises(ValueError):
request.format_parameters({"g": None})
async def test_connection_error_response():
class MockSession(object):
def __init__(self):
self.auto_decompress = True
@property
def auto_decompress(self):
return self.auto_decompress
class MockTransport(AsyncHttpTransport):
def __init__(self):
self._count = 0
self.session = MockSession
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
async def close(self):
pass
async def open(self):
pass
async def send(self, request, **kwargs):
request = HttpRequest('GET', 'http://127.0.0.1/')
response = AsyncHttpResponse(request, None)
response.status_code = 200
return response
class MockContent():
def __init__(self):
self._first = True
async def read(self, block_size):
if self._first:
self._first = False
raise ConnectionError
return None
class MockInternalResponse():
def __init__(self):
self.headers = {}
self.content = MockContent()
async def close(self):
pass
class AsyncMock(mock.MagicMock):
async def __call__(self, *args, **kwargs):
return super(AsyncMock, self).__call__(*args, **kwargs)
http_request = HttpRequest('GET', 'http://127.0.0.1/')
pipeline = AsyncPipeline(MockTransport())
http_response = AsyncHttpResponse(http_request, None)
http_response.internal_response = MockInternalResponse()
stream = AioHttpStreamDownloadGenerator(pipeline,
http_response,
decompress=False)
with mock.patch('asyncio.sleep', new_callable=AsyncMock):
with pytest.raises(ConnectionError):
await stream.__anext__()
# HttpXTransport does not support stream data:
# File "lib\site-packages\httpx\_content_streams.py", line 404, in encode
# raise TypeError(f"Unexpected type for 'data', {type(data)!r}")
# TypeError: Unexpected type for 'data', <class '__main__.LargeStream'>
#
# start = time.perf_counter()
# req = HttpRequest("PUT", url, data=LargeStream(size), headers=headers)
# resp = pipelinex.run(req)
# stop = time.perf_counter()
# duration = stop - start
# mbps = ((size / duration) * 8) / (1024 * 1024)
# print(f'[PipelineX, stream] Put {size:,} bytes in {duration:.2f} seconds ({mbps:.2f} Mbps), Response={resp.http_response.status_code}')
start = time.perf_counter()
req = HttpRequest("PUT", url, data=array, headers=headers)
resp = pipelinex.run(req)
stop = time.perf_counter()
duration = stop - start
mbps = ((size / duration) * 8) / (1024 * 1024)
print(
f'[PipelineX, array] Put {size:,} bytes in {duration:.2f} seconds ({mbps:.2f} Mbps), Response={resp.http_response.status_code}'
)
start = time.perf_counter()
req = HttpRequest("PUT", url, data=LargeStream(size), headers=headers)
resp = pipeline.run(req)
stop = time.perf_counter()
duration = stop - start
mbps = ((size / duration) * 8) / (1024 * 1024)
print(
async def send(self, request, **kwargs):
request = HttpRequest('GET', 'http://127.0.0.1/')
response = AsyncHttpResponse(request, None)
response.status_code = 200
return response
async def _batch_send(
self,
entities, # type: List[TableEntity]
*reqs: "HttpRequest",
**kwargs
):
"""Given a series of request, do a Storage batch call."""
# Pop it here, so requests doesn't feel bad about additional kwarg
raise_on_any_failure = kwargs.pop("raise_on_any_failure", True)
policies = [StorageHeadersPolicy()]
changeset = HttpRequest("POST", None)
changeset.set_multipart_mixed(
*reqs, policies=policies, boundary="changeset_{}".format(uuid4())
)
request = self._client._client.post( # pylint: disable=protected-access
url="https://{}/$batch".format(self._primary_hostname),
headers={
"x-ms-version": self.api_version,
"DataServiceVersion": "3.0",
"MaxDataServiceVersion": "3.0;NetFx",
},
)
request.set_multipart_mixed(
changeset,
policies=policies,
enforce_https=False,
boundary="batch_{}".format(uuid4()),
)
pipeline_response = await self._pipeline.run(request, **kwargs)
response = pipeline_response.http_response
if response.status_code == 403:
raise ClientAuthenticationError(
message="There was an error authenticating with the service",
response=response,
)
if response.status_code == 404:
raise ResourceNotFoundError(
message="The resource could not be found", response=response
)
if response.status_code != 202:
raise BatchErrorException(
message="There is a failure in the batch operation.",
response=response,
parts=None,
)
parts_iter = response.parts()
parts = []
async for p in parts_iter:
parts.append(p)
transaction_result = BatchTransactionResult(reqs, parts, entities)
if raise_on_any_failure:
if any(p for p in parts if not 200 <= p.status_code < 300):
if any(p for p in parts if p.status_code == 404):
raise ResourceNotFoundError(
message="The resource could not be found", response=response
)
raise BatchErrorException(
message="There is a failure in the batch operation.",
response=response,
parts=parts,
)
return transaction_result
def test_raw_deserializer():
raw_deserializer = ContentDecodePolicy()
context = PipelineContext(None, stream=False)
universal_request = HttpRequest('GET', 'http://127.0.0.1/')
request = PipelineRequest(universal_request, context)
def build_response(body, content_type=None):
class MockResponse(HttpResponse):
def __init__(self, body, content_type):
super(MockResponse, self).__init__(None, None)
self._body = body
self.content_type = content_type
def body(self):
return self._body
return PipelineResponse(request, MockResponse(body, content_type),
context)
response = build_response(b"<groot/>", content_type="application/xml")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result.tag == "groot"
response = build_response(b"\xef\xbb\xbf<utf8groot/>",
content_type="application/xml")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result.tag == "utf8groot"
# The basic deserializer works with unicode XML
response = build_response(u'<groot language="français"/>'.encode('utf-8'),
content_type="application/xml")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result.attrib["language"] == u"français"
# Catch some weird situation where content_type is XML, but content is JSON
response = build_response(b'{"ugly": true}',
content_type="application/xml")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["ugly"] is True
# Be sure I catch the correct exception if it's neither XML nor JSON
response = build_response(b'gibberish', content_type="application/xml")
with pytest.raises(DecodeError) as err:
raw_deserializer.on_response(request, response)
assert err.value.response is response.http_response
response = build_response(b'{{gibberish}}', content_type="application/xml")
with pytest.raises(DecodeError) as err:
raw_deserializer.on_response(request, response)
assert err.value.response is response.http_response
# Simple JSON
response = build_response(b'{"success": true}',
content_type="application/json")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["success"] is True
# Simple JSON with BOM
response = build_response(b'\xef\xbb\xbf{"success": true}',
content_type="application/json")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["success"] is True
# Simple JSON with complex content_type
response = build_response(
b'{"success": true}',
content_type="application/vnd.microsoft.appconfig.kv.v1+json")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["success"] is True
# Simple JSON with complex content_type, v2
response = build_response(
b'{"success": true}',
content_type="text/vnd.microsoft.appconfig.kv.v1+json")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["success"] is True
# For compat, if no content-type, decode JSON
response = build_response(b'"data"')
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == "data"
# Let text/plain let through
response = build_response(b'I am groot', content_type="text/plain")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == "I am groot"
# Let text/plain let through + BOM
response = build_response(b'\xef\xbb\xbfI am groot',
content_type="text/plain")
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == "I am groot"
# Try with a mock of requests
req_response = requests.Response()
req_response.headers["content-type"] = "application/json"
req_response._content = b'{"success": true}'
req_response._content_consumed = True
response = PipelineResponse(None,
RequestsTransportResponse(None, req_response),
PipelineContext(None, stream=False))
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result["success"] is True
# I can enable it per request
request.context.options['response_encoding'] = 'utf-8'
response = build_response(b'\xc3\xa9', content_type="text/plain")
raw_deserializer.on_request(request)
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == u"é"
assert response.context["response_encoding"] == "utf-8"
del request.context['response_encoding']
# I can enable it globally
raw_deserializer = ContentDecodePolicy(response_encoding="utf-8")
response = build_response(b'\xc3\xa9', content_type="text/plain")
raw_deserializer.on_request(request)
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == u"é"
assert response.context["response_encoding"] == "utf-8"
del request.context['response_encoding']
# Per request is more important
request.context.options['response_encoding'] = 'utf-8-sig'
response = build_response(b'\xc3\xa9', content_type="text/plain")
raw_deserializer.on_request(request)
raw_deserializer.on_response(request, response)
result = response.context["deserialized_data"]
assert result == u"é"
assert response.context["response_encoding"] == "utf-8-sig"
del request.context['response_encoding']
async def test_multipart_receive_with_combination_changeset_last():
changeset = HttpRequest(None, None)
changeset.set_multipart_mixed(HttpRequest("DELETE", "/container1/blob1"),
HttpRequest("DELETE", "/container2/blob2"))
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(HttpRequest("DELETE", "/container0/blob0"),
changeset)
body_as_bytes = (
b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 2\r\n'
b'\r\n'
b'HTTP/1.1 200\r\n'
b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n'
b'x-ms-version: 2018-11-09\r\n'
b'\r\n'
b'\r\n'
b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n'
b'Content-Type: multipart/mixed; boundary="changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525"\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 0\r\n'
b'\r\n'
b'HTTP/1.1 202\r\n'
b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n'
b'x-ms-version: 2018-11-09\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 1\r\n'
b'\r\n'
b'HTTP/1.1 404\r\n'
b'x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n'
b'x-ms-version: 2018-11-09\r\n'
b'\r\n'
b'\r\n'
b'--changeset_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n'
b'\r\n'
b'--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--\r\n')
response = MockResponse(
request, body_as_bytes,
"multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed"
)
parts = []
async for part in response.parts():
parts.append(part)
assert len(parts) == 3
assert parts[0].status_code == 200
assert parts[1].status_code == 202
assert parts[2].status_code == 404
def test_http_logger():
class MockHandler(logging.Handler):
def __init__(self):
super(MockHandler, self).__init__()
self.messages = []
def reset(self):
self.messages = []
def emit(self, record):
self.messages.append(record)
mock_handler = MockHandler()
logger = logging.getLogger("testlogger")
logger.addHandler(mock_handler)
logger.setLevel(logging.DEBUG)
policy = HttpLoggingPolicy(logger=logger)
universal_request = HttpRequest('GET', 'http://127.0.0.1/')
http_response = HttpResponse(universal_request, None)
http_response.status_code = 202
request = PipelineRequest(universal_request, PipelineContext(None))
# Basics
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
assert all(m.levelname == 'INFO' for m in mock_handler.messages)
assert len(mock_handler.messages) == 5
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
mock_handler.reset()
# Let's make this request a failure, retried twice
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
assert all(m.levelname == 'INFO' for m in mock_handler.messages)
assert len(mock_handler.messages) == 10
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
mock_handler.reset()
# Headers and query parameters
policy.allowed_query_params = ['country']
universal_request.headers = {
"Accept": "Caramel",
"Hate": "Chocolat",
}
http_response.headers = {
"Content-Type": "Caramel",
"HateToo": "Chocolat",
}
universal_request.url = "http://127.0.0.1/?country=france&city=aix"
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
assert all(m.levelname == 'INFO' for m in mock_handler.messages)
assert len(mock_handler.messages) == 9
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/?country=france&city=REDACTED'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == "Request headers:"
# Dict not ordered in Python, exact logging order doesn't matter
assert set([
mock_handler.messages[3].message, mock_handler.messages[4].message
]) == set([" 'Accept': 'Caramel'", " 'Hate': 'REDACTED'"])
assert mock_handler.messages[5].message == "Response status: 202"
assert mock_handler.messages[6].message == "Response headers:"
# Dict not ordered in Python, exact logging order doesn't matter
assert set([
mock_handler.messages[7].message, mock_handler.messages[8].message
]) == set([" 'Content-Type': 'Caramel'", " 'HateToo': 'REDACTED'"])
mock_handler.reset()
async def test_multipart_send():
# transport = mock.MagicMock(spec=AsyncHttpTransport)
# MagicMock support async cxt manager only after 3.8
# https://github.com/python/cpython/pull/9296
class MockAsyncHttpTransport(AsyncHttpTransport):
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
async def open(self):
pass
async def close(self):
pass
async def send(self, request, **kwargs):
pass
transport = MockAsyncHttpTransport()
class RequestPolicy(object):
async def on_request(self, request):
# type: (PipelineRequest) -> None
request.http_request.headers[
'x-ms-date'] = 'Thu, 14 Jun 2018 16:46:54 GMT'
req0 = HttpRequest("DELETE", "/container0/blob0")
req1 = HttpRequest("DELETE", "/container1/blob1")
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(
req0,
req1,
policies=[RequestPolicy()],
boundary=
"batch_357de4f7-6d0b-4e02-8cd2-6361411a9525" # Fix it so test are deterministic
)
async with AsyncPipeline(transport) as pipeline:
await pipeline.run(request)
assert request.body == (
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 0\r\n'
b'\r\n'
b'DELETE /container0/blob0 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525\r\n'
b'Content-Type: application/http\r\n'
b'Content-Transfer-Encoding: binary\r\n'
b'Content-ID: 1\r\n'
b'\r\n'
b'DELETE /container1/blob1 HTTP/1.1\r\n'
b'x-ms-date: Thu, 14 Jun 2018 16:46:54 GMT\r\n'
b'\r\n'
b'\r\n'
b'--batch_357de4f7-6d0b-4e02-8cd2-6361411a9525--\r\n')
def test_http_logger_operation_level():
class MockHandler(logging.Handler):
def __init__(self):
super(MockHandler, self).__init__()
self.messages = []
def reset(self):
self.messages = []
def emit(self, record):
self.messages.append(record)
mock_handler = MockHandler()
logger = logging.getLogger("testlogger")
logger.addHandler(mock_handler)
logger.setLevel(logging.DEBUG)
policy = HttpLoggingPolicy()
kwargs = {'logger': logger}
universal_request = HttpRequest('GET', 'http://127.0.0.1/')
http_response = HttpResponse(universal_request, None)
http_response.status_code = 202
request = PipelineRequest(universal_request,
PipelineContext(None, **kwargs))
# Basics
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
assert all(m.levelname == 'INFO' for m in mock_handler.messages)
assert len(mock_handler.messages) == 5
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
mock_handler.reset()
# Let's make this request a failure, retried twice
request = PipelineRequest(universal_request,
PipelineContext(None, **kwargs))
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
policy.on_request(request)
response = PipelineResponse(request, http_response, request.context)
policy.on_response(request, response)
assert all(m.levelname == 'INFO' for m in mock_handler.messages)
assert len(mock_handler.messages) == 10
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
assert mock_handler.messages[
0].message == "Request URL: 'http://127.0.0.1/'"
assert mock_handler.messages[1].message == "Request method: 'GET'"
assert mock_handler.messages[2].message == 'Request headers:'
assert mock_handler.messages[3].message == 'Response status: 202'
assert mock_handler.messages[4].message == 'Response headers:'
mock_handler.reset()
async def test_multipart_receive():
class MockResponse(AsyncHttpResponse):
def __init__(self, request, body, content_type):
super(MockResponse, self).__init__(request, None)
self._body = body
self.content_type = content_type
def body(self):
return self._body
class ResponsePolicy(object):
def on_response(self, request, response):
# type: (PipelineRequest, PipelineResponse) -> None
response.http_response.headers['x-ms-fun'] = 'true'
class AsyncResponsePolicy(object):
async def on_response(self, request, response):
# type: (PipelineRequest, PipelineResponse) -> None
response.http_response.headers['x-ms-async-fun'] = 'true'
req0 = HttpRequest("DELETE", "/container0/blob0")
req1 = HttpRequest("DELETE", "/container1/blob1")
request = HttpRequest("POST",
"http://account.blob.core.windows.net/?comp=batch")
request.set_multipart_mixed(
req0, req1, policies=[ResponsePolicy(),
AsyncResponsePolicy()])
body_as_str = (
"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n"
"Content-Type: application/http\r\n"
"Content-ID: 0\r\n"
"\r\n"
"HTTP/1.1 202 Accepted\r\n"
"x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e284f\r\n"
"x-ms-version: 2018-11-09\r\n"
"\r\n"
"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed\r\n"
"Content-Type: application/http\r\n"
"Content-ID: 2\r\n"
"\r\n"
"HTTP/1.1 404 The specified blob does not exist.\r\n"
"x-ms-error-code: BlobNotFound\r\n"
"x-ms-request-id: 778fdc83-801e-0000-62ff-0334671e2852\r\n"
"x-ms-version: 2018-11-09\r\n"
"Content-Length: 216\r\n"
"Content-Type: application/xml\r\n"
"\r\n"
'<?xml version="1.0" encoding="utf-8"?>\r\n'
"<Error><Code>BlobNotFound</Code><Message>The specified blob does not exist.\r\n"
"RequestId:778fdc83-801e-0000-62ff-0334671e2852\r\n"
"Time:2018-06-14T16:46:54.6040685Z</Message></Error>\r\n"
"--batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed--")
response = MockResponse(
request, body_as_str.encode('ascii'),
"multipart/mixed; boundary=batchresponse_66925647-d0cb-4109-b6d3-28efe3e1e5ed"
)
parts = []
async for part in response.parts():
parts.append(part)
assert len(parts) == 2
res0 = parts[0]
assert res0.status_code == 202
assert res0.headers['x-ms-fun'] == 'true'
assert res0.headers['x-ms-async-fun'] == 'true'
res1 = parts[1]
assert res1.status_code == 404
assert res1.headers['x-ms-fun'] == 'true'
assert res1.headers['x-ms-async-fun'] == 'true'
def test_no_log(mock_http_logger):
universal_request = HttpRequest('GET', 'http://127.0.0.1/')
request = PipelineRequest(universal_request, PipelineContext(None))
http_logger = NetworkTraceLoggingPolicy()
response = PipelineResponse(request, HttpResponse(universal_request, None),
request.context)
# By default, no log handler for HTTP
http_logger.on_request(request)
mock_http_logger.debug.assert_not_called()
http_logger.on_response(request, response)
mock_http_logger.debug.assert_not_called()
mock_http_logger.reset_mock()
# I can enable it per request
request.context.options['logging_enable'] = True
http_logger.on_request(request)
assert mock_http_logger.debug.call_count >= 1
mock_http_logger.reset_mock()
request.context.options['logging_enable'] = True
http_logger.on_response(request, response)
assert mock_http_logger.debug.call_count >= 1
mock_http_logger.reset_mock()
# I can enable it per request (bool value should be honored)
request.context.options['logging_enable'] = False
http_logger.on_request(request)
mock_http_logger.debug.assert_not_called()
request.context.options['logging_enable'] = False
http_logger.on_response(request, response)
mock_http_logger.debug.assert_not_called()
mock_http_logger.reset_mock()
# I can enable it globally
request.context.options = {}
http_logger.enable_http_logger = True
http_logger.on_request(request)
assert mock_http_logger.debug.call_count >= 1
http_logger.on_response(request, response)
assert mock_http_logger.debug.call_count >= 1
mock_http_logger.reset_mock()
# I can enable it globally and override it locally
http_logger.enable_http_logger = True
request.context.options['logging_enable'] = False
http_logger.on_request(request)
mock_http_logger.debug.assert_not_called()
response.context['logging_enable'] = False
http_logger.on_response(request, response)
mock_http_logger.debug.assert_not_called()
mock_http_logger.reset_mock()
# Let's make this request a failure, retried twice
request.context.options['logging_enable'] = True
http_logger.on_request(request)
http_logger.on_response(request, response)
first_count = mock_http_logger.debug.call_count
assert first_count >= 1
http_logger.on_request(request)
http_logger.on_response(request, response)
second_count = mock_http_logger.debug.call_count
assert second_count == first_count * 2
