def test_delete_service(
mocked_director_v0_service_api: MockRouter,
mocked_director_v2_scheduler: None,
client: TestClient,
service: Dict[str, Any],
exp_status_code: int,
is_legacy: bool,
can_save: Optional[bool],
exp_save_state: bool,
):
url = URL(f"/v2/dynamic_services/{service['node_uuid']}")
if can_save is not None:
url = url.copy_with(params={"can_save": can_save})
response = client.delete(str(url), allow_redirects=False)
assert (
response.status_code == exp_status_code
), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}"
if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT:
# check redirection header goes to director-v0
assert "location" in response.headers
redirect_url = URL(response.headers["location"])
assert redirect_url.host == "director"
assert (redirect_url.path ==
f"/v0/running_interactive_services/{service['node_uuid']}")
assert redirect_url.params == QueryParams(can_save=exp_save_state)
def update_query(url: httpx.URL, **new_params: Optional[str]) -> httpx.URL:
"""
Adds, replaces, or removes query parameters. For each keyword argument, all
query parameters with that name are removed; then, if the argument is not
None, it's added as the sole new value of the corresponding query
parameter.
>>> str(update_query(httpx.URL("http://example.org/"), yes="1", no=None))
'http://example.org/?yes=1'
>>> str(update_query(httpx.URL("http://example.org/?yes=0&no="), yes="1", no=None))
'http://example.org/?yes=1'
Additionally, this function puts the URL into a canonical form to increase
cache hit rates. A generic client can't make any assumptions about how the
query string is used, but because we only care how WordPress would
interpret it, we can safely de-duplicate and sort the parameters.
>>> str(update_query(httpx.URL("http://example.org/?yes=2&yes=1&no=0")))
'http://example.org/?no=0&yes=1'
"""
params = dict(parse_qsl(url.query))
for k, v in new_params.items():
if v is None:
params.pop(k.encode(), None)
else:
params[k.encode()] = v.encode()
query = urlencode(sorted(params.items())).encode()
return url.copy_with(query=query)
def test_url_copywith_for_userinfo():
copy_with_kwargs = {
"username": "******",
"password": "******",
}
url = URL("https://example.org")
new = url.copy_with(**copy_with_kwargs)
assert str(new) == "https://tom%40example.org:abc123%40%20%[email protected]"
assert new.username == "*****@*****.**"
assert new.password == "abc123@ %"
def test_url_copywith_for_authority():
copy_with_kwargs = {
"username": "******",
"password": "******",
"port": 444,
"host": "example.net",
}
url = URL("https://example.org")
new = url.copy_with(**copy_with_kwargs)
for k, v in copy_with_kwargs.items():
assert getattr(new, k) == v
assert str(new) == "https://*****:*****@example.net:444"
def test_url():
url = URL("https://example.org:123/path/to/somewhere?abc=123#anchor")
assert url.scheme == "https"
assert url.host == "example.org"
assert url.port == 123
assert url.authority == "example.org:123"
assert url.path == "/path/to/somewhere"
assert url.query == "abc=123"
assert url.fragment == "anchor"
assert (repr(url) ==
"URL('https://example.org:123/path/to/somewhere?abc=123#anchor')")
new = url.copy_with(scheme="http", port=None)
assert new == URL("http://example.org/path/to/somewhere?abc=123#anchor")
assert new.scheme == "http"
def _merge_url(self, url):
if isinstance(url, str):
if url.startswith('http://localhost'):
return url
url = URL(url)
new_url = url.copy_with(scheme=self.scheme, host=self.host, port=self.port)
if 's3.' in url.host:
return new_url.copy_with(path='/s3/')
elif 'email.' in url.host:
return new_url.copy_with(path='/ses/')
elif url.host.startswith('sns.'):
if 'bad' in url.path:
return new_url.copy_with(path='/status/400/')
elif url.path.endswith('.pem'):
return new_url.copy_with(path='/sns/certs/')
else:
return new_url.copy_with(path='/status/200/')
else:
# return url
raise ValueError(f'no local endpoint found for "{url}"')
def _ensure_path(self, url: httpx.URL) -> httpx.URL:
if not url._uri_reference.path:
url = url.copy_with(path="/")
return url
class Service:
"""
The service object to facilitate sending requests to other services.
:param service_name: The name of the service to send a request to.
:param partial_path: Base path of the endpoint to send to.
"""
_client = None
def __init__(self, service_name: str, partial_path: str = None):
self.service_name = service_name
self.service_token = jwt_service_encode_handler(
jwt_service_payload_handler(self))
partial_path = partial_path or "/"
self.url = URL(
f"{settings.SERVICE_GLOBAL_SCHEMA}://"
f"{settings.SERVICE_GLOBAL_HOST_TEMPLATE.format(self.service_name)}"
f":{settings.SERVICE_GLOBAL_PORT}{partial_path}")
super().__init__()
@property
def client(self) -> AsyncClient:
"""
The httpx.AsyncClient that will be used to send async requests.
"""
if self._client is None:
limits = Limits(
max_connections=settings.SERVICE_CONNECTOR_MAX,
max_keepalive_connections=settings.
SERVICE_CONNECTOR_MAX_KEEPALIVE,
)
timeout = Timeout(
timeout=settings.SERVICE_TIMEOUT_TOTAL,
connect=getattr(settings, "SERVICE_TIMEOUT_CONNECT", UNSET),
read=getattr(settings, "SERVICE_TIMEOUT_READ", UNSET),
write=getattr(settings, "SERVICE_TIMEOUT_WRITE", UNSET),
pool=getattr(settings, "SERVICE_TIMEOUT_POOL", UNSET),
)
self._client = AsyncClient(
limits=limits,
timeout=timeout,
base_url=self.url,
headers={
"authorization":
f"{settings.JWT_SERVICE_AUTH_AUTH_HEADER_PREFIX} {self.service_token}"
},
)
return self._client
@property
def host(self) -> str:
"""
The host portion of the url.
"""
return self.url.host
@host.setter
def host(self, value: str) -> None:
self.url = self.url.copy_with(host=value)
@property
def port(self) -> int:
"""
The port portion of the url.
"""
return self.url.port
@port.setter
def port(self, value: int) -> None:
self.url = self.url.copy_with(port=value)
async def close_client(self) -> None:
"""
Close the async client on shutdown.
"""
if self._client is not None:
await self._client.aclose()
await asyncio.sleep(0)
def _inject_headers(self, headers: dict):
# need to coerce to str
headers = {k: str(v) for k, v in headers.items()}
headers.update(
{"date": get_utc_datetime().strftime("%a, %d %b %y %T %z")})
# inject user information to request headers
user = context_user()
headers.update({
settings.INTERNAL_REQUEST_USER_HEADER.lower():
to_header_value(user)
})
# inject correlation_id
correlation_id = context_correlation_id()
headers.update(
{settings.REQUEST_ID_HEADER_FIELD.lower(): correlation_id})
return Headers(headers)
def http_dispatch(
self,
method: str,
endpoint: str,
*,
query_params: dict = None,
payload: dict = None,
files: dict = None,
headers: dict = None,
propagate_error: bool = False,
include_status_code: bool = False,
response_timeout: int = UNSET,
retry_count: int = None,
**kwargs,
):
"""
Interface for sending requests to other services.
:param method: method to send request (GET, POST, PATCH, PUT, etc)
:param endpoint: the path to send request to (eg /api/v1/..)
:param query_params: query params to attach to url
:param payload: the data to send on any non GET requests
:param files: if any files to send with request, must be included here
:param headers: headers to send along with request
:param propagate_error: if you want to raise on 400 or greater status codes
:param include_status_code: if you want this method to return the response with the status code
:param response_timeout: if you want to increase the timeout for this requests
:param retry_count: number times you want to retry the request if failed on server errors
"""
files = files or {}
query_params = query_params or {}
payload = payload or {}
headers = self._inject_headers(headers or {})
request = self.client.build_request(
method,
endpoint,
data=payload,
files=files,
params=query_params,
headers=headers,
)
return asyncio.ensure_future(
self._dispatch_future(
request,
propagate_error=propagate_error,
response_timeout=response_timeout,
include_status_code=include_status_code,
retry_count=retry_count,
**kwargs,
))
async def _dispatch_future(
self,
request,
*,
propagate_error: bool = False,
response_timeout: float = None,
include_status_code: bool = False,
retry_count: int = None,
**kwargs,
):
"""
The async method that wraps the actual fetch
:param request:
:param propagate_error:
:param response_timeout:
:param include_status_code:
:param retry_count:
:param kwargs:
:return:
"""
try:
resp = await asyncio.shield(
self._dispatch_send(
request=request,
timeout=response_timeout,
retry_count=retry_count,
))
if propagate_error:
resp.raise_for_status()
response = resp.json()
if include_status_code:
return response, resp.status_code
else:
return response
except HTTPStatusError as e:
try:
response = e.response.json()
except JSONDecodeError:
response = e.response.text
description = response
error_code = GlobalErrorCodes.unknown_error
message = error_code.name
else:
description = response.get("description", response)
error_code = response.get("error_code",
GlobalErrorCodes.unknown_error)
message = response.get("message", error_code.name)
exc = exceptions.APIException(
description=description,
error_code=error_code,
status_code=e.response.status_code,
)
exc.message = message
base_error_message = (
f"ClientResponseError: {e.request.method} {e.request.url} {e.response.status_code} "
f"{e}")
if StatusCode.is_server_error(e.response.status_code):
error_logger.error(base_error_message)
else:
error_logger.info(base_error_message)
raise exc
except httpx.TimeoutException as e:
exc = exceptions.ResponseTimeoutError(
description=str(e),
error_code=GlobalErrorCodes.service_timeout,
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
raise exc
except InvalidURL as e:
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.invalid_url,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
raise exc
except NotRedirectResponse as e:
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.invalid_url,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
raise exc
except CookieConflict as e:
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.client_payload_error,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
raise exc
except StreamError as e:
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.stream_error,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
raise exc
except RequestError as e:
if hasattr(e, "response"):
status_code = getattr(
e.response,
"status_code",
status.HTTP_503_SERVICE_UNAVAILABLE,
)
else:
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.service_unavailable,
status_code=status_code,
)
raise exc
except HTTPError as e:
exc = exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.transport_error,
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
raise exc
except socket.gaierror as e:
raise exceptions.APIException(
description=str(e),
error_code=GlobalErrorCodes.service_unavailable,
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
async def _dispatch_send(
self,
request: Request,
*,
timeout: float = None,
retry_count: int = None,
):
"""
TODO: need better implementation for retry
:param request:
:param timeout:
:param retry_count:
"""
attempts = 1
if request.method == "GET":
attempts += (settings.SERVICE_CONNECTION_DEFAULT_RETRY_COUNT
if retry_count is None else min(
retry_count,
int(settings.SERVICE_CONNECTION_MAX_RETRY_COUNT),
))
for i in range(attempts):
try:
response = await self.client.send(request, timeout=timeout)
if codes.is_server_error(response.status_code):
response.raise_for_status()
except (TransportError, HTTPStatusError,
ConnectionResetError) as e:
error_logger.debug(f"{str(e)} on attempt {i}")
if i + 1 >= attempts:
raise
else:
return response
