def test_valid_shared_client(dispike_object: Dispike):
from httpx import Client, URL
assert isinstance(dispike_object.shared_client,
Client) == True, type(dispike_object.shared_client)
assert dispike_object.shared_client.base_url == URL(
f"https://discord.com/api/v8/applications/{dispike_object._application_id}/"
)
def rebuild_request(request, url, headers, body):
request.url = URL(url)
request.headers.update(headers)
if body:
body = to_bytes(body)
if body != request.content:
request._content = body
request.stream = ByteStream(body)
return request
def test_httpx_response_raw_path_behaviour():
"""Make sure that `Response.raw_path` continues to do what we need.
As the httpx library is evolving rapidly, this is a defensive test to make sure
that `Response.raw_path` continues to behave as we need for StructurizrClient, in
particular not HTTP-escaping parameters, but still ASCII-encoding the URL.
"""
url = URL("http://example.com:8080/api/test?q=motörhead")
assert url.raw_path.decode("ascii") == "/api/test?q=mot%C3%B6rhead"
async def test_client_queryparams_echo():
url = "http://example.org/echo_queryparams"
client_queryparams = "first=str"
request_queryparams = {"second": "dict"}
client = AsyncClient(dispatch=MockDispatch(), params=client_queryparams)
response = await client.get(url, params=request_queryparams)
assert response.status_code == 200
assert response.url == URL(
"http://example.org/echo_queryparams?first=str&second=dict")
async def test_body_redirect():
"""
A 308 redirect should preserve the request body.
"""
client = AsyncClient(transport=AsyncMockTransport())
url = "https://example.org/redirect_body"
data = b"Example request body"
response = await client.post(url, data=data)
assert response.url == URL("https://example.org/redirect_body_target")
assert response.json()["body"] == "Example request body"
assert "content-length" in response.json()["headers"]
async def test_no_body_redirect():
"""
A 303 redirect should remove the request body.
"""
client = AsyncClient(dispatch=MockDispatch())
url = "https://example.org/redirect_no_body"
data = b"Example request body"
response = await client.post(url, data=data)
assert response.url == URL("https://example.org/redirect_body_target")
assert response.json()["body"] == ""
assert "content-length" not in response.json()["headers"]
async def test_head_redirect():
"""
Contrary to Requests, redirects remain enabled by default for HEAD requests.
"""
client = AsyncClient(transport=AsyncMockTransport())
response = await client.head("https://example.org/redirect_302")
assert response.status_code == codes.OK
assert response.url == URL("https://example.org/")
assert response.request.method == "HEAD"
assert len(response.history) == 1
assert response.text == ""
def _merge_url(self, url):
if isinstance(url, str):
if url.startswith('http://localhost'):
return url
url = URL(url)
new_url = url.copy_with(scheme=self.scheme, host=self.host, port=self.port)
if 's3.' in url.host:
return new_url.copy_with(path='/s3/')
elif 'email.' in url.host:
return new_url.copy_with(path='/ses/')
elif url.host.startswith('sns.'):
if 'bad' in url.path:
return new_url.copy_with(path='/status/400/')
elif url.path.endswith('.pem'):
return new_url.copy_with(path='/sns/certs/')
else:
return new_url.copy_with(path='/status/200/')
else:
# return url
raise ValueError(f'no local endpoint found for "{url}"')
def _sign(self, base: str, endpoint: str, params: Dict[str, Any]) -> URL:
params.update(
{
**BilibiliConstants.DEFAULT_PARAMS,
"access_key": BilibiliConstants.ACCESS_KEY,
"appkey": BilibiliConstants.APP_KEY,
"ts": int(time()),
}
)
params = {k: params[k] for k in sorted(params.keys())}
url = self._join(base=base, endpoint=endpoint, params=params)
sign = hashlib.md5(string=(url.query + BilibiliConstants.SECRET)).hexdigest()
return URL(url, params={"sign": sign})
def test_get_service_status(
mocked_director_v0_service_api: MockRouter,
mocked_director_v2_scheduler: None,
client: TestClient,
service: Dict[str, Any],
exp_status_code: int,
is_legacy: bool,
):
url = URL(f"/v2/dynamic_services/{service['node_uuid']}")
response = client.get(str(url), allow_redirects=False)
assert (
response.status_code == exp_status_code
), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}"
if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT:
# check redirection header goes to director-v0
assert "location" in response.headers
redirect_url = URL(response.headers["location"])
assert redirect_url.host == "director"
assert (redirect_url.path ==
f"/v0/running_interactive_services/{service['node_uuid']}")
assert redirect_url.params == QueryParams("") # empty query
def _url_match(self, request: Request) -> bool:
if not self.url:
return True
# re.Pattern was introduced in Python 3.7
if isinstance(
self.url, re._pattern_type if hasattr(re, "_pattern_type") else re.Pattern
):
return self.url.match(str(request.url)) is not None
if isinstance(self.url, str):
return URL(self.url) == request.url
return self.url == request.url
def add_signed_download_params(self,
method: Literal['GET', 'POST'],
url: URL,
expires: int = 86400) -> URL:
assert expires >= 1, f'expires must be greater than or equal to 1, not {expires}'
assert expires <= 604800, f'expires must be less than or equal to 604800, not {expires}'
now = utcnow()
url = url.copy_merge_params({
'X-Amz-Algorithm':
_AUTH_ALGORITHM,
'X-Amz-Credential':
self._aws4_credential(now),
'X-Amz-Date':
_aws4_x_amz_date(now),
'X-Amz-Expires':
str(expires),
'X-Amz-SignedHeaders':
'host',
})
_, signature = self._aws4_signature(now, method, url,
{'host': self.host},
'UNSIGNED-PAYLOAD')
return url.copy_add_param('X-Amz-Signature', signature)
async def _imageDownload(
self,
client: AsyncClient,
data: models.DanbooruImage,
) -> models.ImageDownload:
while self._running >= self._workers:
await asyncio.sleep(1)
self._running += 1
urlParsed = URL(data.imageURL)
logger.trace("Start downloading picture " +
f"{urlParsed.full_path!r} from {urlParsed.host!r}.")
tempfile, hashData, totalWrite = TempFile().create(), HashCreator(), 0
try:
response = await client.get(
urlParsed,
headers={
"User-Agent":
randChoice(ImageSpiderConfig["user-agents"].get(list)
or [f"DanbooruSpider/{VERSION}"]),
},
)
response.raise_for_status()
async with AsyncOpen(str(tempfile), "wb") as f:
async for chunk in response.aiter_bytes():
await hashData.update(chunk)
totalWrite += await f.write(chunk)
logger.trace("Finished downloading picture " +
f"{urlParsed.full_path!r} from {urlParsed.host!r}, " +
f"total write {totalWrite} bytes.")
except HTTPError as e:
raise NetworkException(
"There was an error in the network when processing the picture "
+ f"'{urlParsed}', the reason is: {e}")
except Exception as e:
raise SpiderException(
"There was a unknown error when processing the picture " +
f"'{urlParsed}', the reason is: {e}")
finally:
self._running -= 1
return models.ImageDownload(
**{
"source": str(urlParsed),
"path": tempfile,
"size": totalWrite,
"md5": await hashData.hexdigest(),
"data": data,
})
async def create_user_with_password(endpoint: URL, username: EmailStr,
password: SecretStr) -> int:
try:
async with AsyncClient(base_url=endpoint.join("v0")) as client:
typer.secho(
f"registering user {username} with password {password}",
fg=typer.colors.YELLOW,
)
await register_user(client, username, password)
typer.secho(f"user registered", fg=typer.colors.YELLOW)
await logout_current_user(client)
typer.secho(f"registration done", fg=typer.colors.YELLOW)
except Exception as exc: # pylint: disable=broad-except
typer.secho(f"Unexpected issue: {exc}", fg=typer.colors.RED, err=True)
return 1
return 0
def _join(base: str, endpoint: str, params: Dict[str, Any]) -> URL:
host: ParseResult = urlparse(base)
params = {
k: (v.value if isinstance(v, Enum) else v)
for k, v in params.items() if v is not None
}
return URL(
url=ParseResult(
scheme=host.scheme,
netloc=host.netloc,
path=endpoint.format(**params),
params="",
query="",
fragment="",
).geturl(),
params=params,
)
async def auth_call(self, request, get_response, has_method=True):
content = await request.read()
if has_method:
url, headers, body = self.prepare(
request.method, str(request.url), request.headers, content)
else:
url, headers, body = self.prepare(
str(request.url), request.headers, content)
request.url = URL(url)
request.headers.update(headers)
if body:
body = to_bytes(body)
if body != content:
request.is_streaming = False
request.content = body
return await get_response(request)
def test_retrieve(
mock_retrieve_features: Optional[MockRouter],
mocked_director_v0_service_api: MockRouter,
mocked_director_v2_scheduler: None,
client: TestClient,
service: Dict[str, Any],
exp_status_code: int,
is_legacy: bool,
) -> None:
url = URL(f"/v2/dynamic_services/{service['node_uuid']}:retrieve")
response = client.post(str(url),
json=dict(port_keys=[]),
allow_redirects=False)
assert (
response.status_code == exp_status_code
), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}"
assert (response.json() ==
RetrieveDataOutEnveloped.Config.schema_extra["examples"][0])
async def test_get_default_cluster(
clusters_config: None,
registered_user: Callable[..., Dict],
cluster: Callable[..., Cluster],
async_client: httpx.AsyncClient,
):
user_1 = registered_user()
# NOTE: we should not need the user id for default right?
# NOTE: it should be accessible to everyone to run, and only a handful of elected
# people shall be able to administer it
get_cluster_url = URL("/v2/clusters/default")
response = await async_client.get(get_cluster_url)
assert response.status_code == status.HTTP_200_OK, f"received {response.text}"
returned_cluster = parse_obj_as(ClusterGet, response.json())
assert returned_cluster
assert returned_cluster.name == "Default cluster"
assert 1 in returned_cluster.access_rights # everyone group is always 1
assert returned_cluster.access_rights[1] == CLUSTER_USER_RIGHTS
def test_url_join():
"""
Some basic URL joining tests.
"""
url = URL("https://example.org:123/path/to/somewhere")
assert url.join("/somewhere-else") == "https://example.org:123/somewhere-else"
assert (
url.join("somewhere-else") == "https://example.org:123/path/to/somewhere-else"
)
assert (
url.join("../somewhere-else") == "https://example.org:123/path/somewhere-else"
)
assert url.join("../../somewhere-else") == "https://example.org:123/somewhere-else"
async def clean(
endpoint: URL, username: EmailStr, password: SecretStr, project_id: Optional[str]
) -> int:
try:
async with AsyncClient(
base_url=endpoint.join("v0"), timeout=DEFAULT_TIMEOUT
) as client:
await login_user(client, username, password)
all_projects = []
if project_id:
project = await get_project_for_user(client, project_id)
if not project:
typer.secho(
f"project {project_id} not found!",
fg=typer.colors.RED,
err=True,
)
return 1
all_projects = [project]
if not all_projects:
all_projects = await get_all_projects_for_user(client)
if not all_projects:
typer.secho("no projects found!", fg=typer.colors.RED, err=True)
return 1
total = len(all_projects)
typer.secho(f"{total} projects will be deleted...", fg=typer.colors.YELLOW)
with typer.progressbar(
length=total, label="deleting projects"
) as progressbar:
await asyncio.gather(
*[
delete_project(client, prj["uuid"], progressbar)
for prj in all_projects
]
)
typer.secho(f"completed projects deletion", fg=typer.colors.YELLOW)
except Exception as exc: # pylint: disable=broad-except
typer.secho(
f"Unexpected issue: {exc}, [{type(exc)}]", fg=typer.colors.RED, err=True
)
return 1
return 0
def auth_flow(self,
request: Request) -> Generator[Request, Response, None]:
"""
Update the request.
:param request: A request to be updated with OAuth parameters.
"""
oauth_parameters = self.oauth_base_parameters
hmac_message_query_string = str(
QueryParams(
sorted(
chain(
QueryParams(request.url.query).items(),
(QueryParams(request.content.decode()).items()
if request.method == 'POST'
and request.headers.get('content-type')
== 'application/x-www-form-urlencoded' else []),
oauth_parameters.items(),
))))
oauth_parameters['oauth_signature'] = b64encode(s=hmac_new(
key='&'.join([
quote(string=self.consumer_secret, safe=b"~"),
quote(string=self.oauth_access_token_secret or '', safe=b"~")
]).encode(),
msg='&'.join(
quote(string=element, safe=b'~') for element in
(request.method.upper(),
f'{request.url.scheme}://{request.url.host}{request.url.path}',
hmac_message_query_string)).encode(),
digestmod=sha1).digest()).decode()
# The OAuth parameters are added to the query.
request.url = URL(url=request.url, params=oauth_parameters)
# request.headers['Authorization'] = 'OAuth ' + ', '.join([
# f'{key}="{value}"' for key, value in oauth_parameters.items()]
# )
yield request
async def test_create_dynamic_services(
minimal_config: None,
mocked_director_v0_service_api,
docker_swarm: None,
mocked_director_v2_scheduler: None,
client: TestClient,
dynamic_sidecar_headers: Dict[str, str],
service: Dict[str, Any],
exp_status_code: int,
):
# dynamic-sidecar components
await setup_scheduler(client.app)
await setup_api_client(client.app)
post_data = DynamicServiceCreate(**service)
response = client.post(
"/v2/dynamic_services",
headers=dynamic_sidecar_headers,
json=json.loads(post_data.json()),
)
assert (
response.status_code == exp_status_code
), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}"
if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT:
# check redirection header goes to director-v0
assert "location" in response.headers
redirect_url = URL(response.headers["location"])
assert redirect_url.host == "director"
assert redirect_url.path == "/v0/running_interactive_services"
assert redirect_url.params["user_id"] == str(service["user_id"])
assert redirect_url.params["project_id"] == service["project_id"]
assert redirect_url.params["service_uuid"] == service["node_uuid"]
assert redirect_url.params["service_key"] == service["key"]
assert redirect_url.params["service_version"] == service["version"]
assert redirect_url.params["service_basepath"] == service["basepath"]
if exp_status_code == status.HTTP_201_CREATED:
# check the returned data
pass
def _sign(self, endpoint: str, params: Dict[str, Any]) -> URL:
def random_digit(length: int) -> str:
return "".join(map(str, [randint(0, 9) for _ in range(length)]))
params.update({
"_client_id":
"wappc_" + random_digit(13) + "_" + random_digit(3),
"_client_type":
2,
"_client_version":
"9.9.8.32",
**{
k.upper(): v
for k, v in Config["net"]["params"].as_dict().items() if v
},
})
params = {k: params[k] for k in sorted(params.keys())}
url = self._join(self.base, endpoint, params)
sign = (hashlib.md5(url.query.replace(b"&", b"") +
b"tiebaclient!!!").hexdigest().upper())
return URL(url, params={"sign": sign})
def __init__(self, username: str, password: str, **kwargs) -> None:
"""
Initialize VictorAsyncClient.
:param username: User name used to access Victor API.
:param username: Password used to access Victor API.
:param kwargs: Arguments to pass to the httpx.AsyncClient constructor.
"""
super().__init__(**kwargs)
if not username:
raise ValueError("User name is required.")
if not password:
raise ValueError("Password is required.")
if self.base_url == URL():
self.base_url = DEFAULT_BASE_URL
self._credentials = {"password": password, "username": username}
self._token = None
async def login(
cls,
*,
account: Optional[Tuple[str, str]] = None,
refresh_token: Optional[str] = None,
):
assert (account and refresh_token) is None
url = URL(PixivConstants.AUTH_HOST + "/auth/token")
time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00")
headers = {
**PixivConstants.DEFAULT_HEADERS,
"X-Client-Time":
time,
"X-Client-Hash":
hashlib.md5(time.encode() +
PixivConstants.HASH_SECRET.encode()).hexdigest(),
}
data = {
"get_secure_url": 1,
"client_id": PixivConstants.CLIENT_ID,
"client_secret": PixivConstants.CLIENT_SECRET,
}
if refresh_token:
data.update({
"grant_type": "refresh_token",
"refresh_token": refresh_token,
})
elif account:
username, password = account
data.update({
"grant_type": "password",
"username": username,
"password": password,
})
async with AsyncHTTPClient(proxies=PixivConstants.CONFIG["proxy"].get(
Dict[str, str]) # type:ignore
) as client:
response = await client.post(url, data=data, headers=headers)
response.raise_for_status()
return cls.parse_obj(response.json())
def __init__(self, **kwargs):
data = {
'body': {},
'headers': {},
'path': {},
'query': {},
}
data['header'] = data['headers']
skip_validation = kwargs.pop('esi_skip_validation', False)
errors = {}
for param in self.params:
value = kwargs.pop(param.safe_name)
if value == param.default:
continue
try:
param.validate(value)
except ValidationError as e:
e.update_error_dict(errors, param.safe_name)
data[param.part][param.name] = value
if errors and not skip_validation:
raise ValidationError(errors)
cls = self.__class__
body = None
if data['body']:
body = list(data['body'].values())[0]
self.method = cls.method.upper()
self.url = URL(self.path.format(**data['path']),
allow_relative=True,
params=data['query'])
self.headers = Headers(data['header'])
self.stream = encode(body, None, None)
self.timer = ElapsedTimer()
self.prepare()
# Clear out headers we don't need (These will be set by the session)
for key in ["User-Agent"]:
if key in self.headers:
del self.headers[key]
async def test_create_cluster(
clusters_config: None,
registered_user: Callable[..., Dict],
cluster_simple_authentication: Callable,
async_client: httpx.AsyncClient,
faker: Faker,
postgres_db: sa.engine.Engine,
clusters_cleaner,
):
user_1 = registered_user()
create_cluster_url = URL(f"/v2/clusters?user_id={user_1['id']}")
cluster_data = ClusterCreate(
endpoint=faker.uri(),
authentication=cluster_simple_authentication(),
name=faker.name(),
type=random.choice(list(ClusterType)),
)
response = await async_client.post(create_cluster_url,
json=cluster_data.dict(
by_alias=True, exclude_unset=True))
assert response.status_code == status.HTTP_201_CREATED, f"received: {response.text}"
created_cluster = parse_obj_as(ClusterGet, response.json())
assert created_cluster
for k in created_cluster.dict(
exclude={"id", "owner", "access_rights"}).keys():
assert getattr(created_cluster, k) == getattr(cluster_data, k)
assert created_cluster.id is not None
assert created_cluster.owner == user_1["primary_gid"]
assert created_cluster.access_rights == {
user_1["primary_gid"]: CLUSTER_ADMIN_RIGHTS
}
# let's check that DB is correctly setup, there is one entry
with postgres_db.connect() as conn:
cluster_entry = conn.execute(
sa.select([clusters
]).where(clusters.c.name == cluster_data.name)).one()
async def create_payment(self,
amount_unit: int,
currency: str,
body_params: Optional[dict] = None,
headers: Optional[Headers] = None) -> Response:
target = URL('/g_business/v1/payments')
try:
headers.update({'Content-Type': 'application/json'})
except AttributeError:
headers = Headers({'Content-Type': 'application/json'})
try:
body_params.update({
'flow': 'MATCH_CODE',
'amount_unit': amount_unit,
'currency': currency
})
except AttributeError:
body_params = {
'flow': 'MATCH_CODE',
'amount_unit': amount_unit,
'currency': currency
}
return await self.post(target, json=body_params, headers=headers)
async def test_follow_next_link_until_exausted(self):
async with TestClient(app) as client:
response = await client.get("/test/pagination/links/?limit=10")
assert response.status_code == 200
responses = []
while response.links.get("next", {}).get("url"):
assert int(response.headers["x-total-count"]) == 30
assert len(response.json()) == 10
url = URL(response.links["next"]["url"])
next = f"{url.path}?{url.query}"
logger.warning(next)
response = await client.get(next)
responses.append(response)
assert response.status_code == 200
# get last one
assert int(response.headers["x-total-count"]) == 30
assert len(response.json()) == 10
responses.append(response)
assert len(responses) == 3
async def test_cross_subdomain_redirect():
client = AsyncClient(dispatch=MockDispatch())
url = "https://example.com/cross_subdomain"
response = await client.get(url)
assert response.url == URL("https://www.example.org/cross_subdomain")
