def test_valid_shared_client(dispike_object: Dispike): from httpx import Client, URL assert isinstance(dispike_object.shared_client, Client) == True, type(dispike_object.shared_client) assert dispike_object.shared_client.base_url == URL( f"https://discord.com/api/v8/applications/{dispike_object._application_id}/" )
def rebuild_request(request, url, headers, body): request.url = URL(url) request.headers.update(headers) if body: body = to_bytes(body) if body != request.content: request._content = body request.stream = ByteStream(body) return request
def test_httpx_response_raw_path_behaviour(): """Make sure that `Response.raw_path` continues to do what we need. As the httpx library is evolving rapidly, this is a defensive test to make sure that `Response.raw_path` continues to behave as we need for StructurizrClient, in particular not HTTP-escaping parameters, but still ASCII-encoding the URL. """ url = URL("http://example.com:8080/api/test?q=motörhead") assert url.raw_path.decode("ascii") == "/api/test?q=mot%C3%B6rhead"
async def test_client_queryparams_echo(): url = "http://example.org/echo_queryparams" client_queryparams = "first=str" request_queryparams = {"second": "dict"} client = AsyncClient(dispatch=MockDispatch(), params=client_queryparams) response = await client.get(url, params=request_queryparams) assert response.status_code == 200 assert response.url == URL( "http://example.org/echo_queryparams?first=str&second=dict")
async def test_body_redirect(): """ A 308 redirect should preserve the request body. """ client = AsyncClient(transport=AsyncMockTransport()) url = "https://example.org/redirect_body" data = b"Example request body" response = await client.post(url, data=data) assert response.url == URL("https://example.org/redirect_body_target") assert response.json()["body"] == "Example request body" assert "content-length" in response.json()["headers"]
async def test_no_body_redirect(): """ A 303 redirect should remove the request body. """ client = AsyncClient(dispatch=MockDispatch()) url = "https://example.org/redirect_no_body" data = b"Example request body" response = await client.post(url, data=data) assert response.url == URL("https://example.org/redirect_body_target") assert response.json()["body"] == "" assert "content-length" not in response.json()["headers"]
async def test_head_redirect(): """ Contrary to Requests, redirects remain enabled by default for HEAD requests. """ client = AsyncClient(transport=AsyncMockTransport()) response = await client.head("https://example.org/redirect_302") assert response.status_code == codes.OK assert response.url == URL("https://example.org/") assert response.request.method == "HEAD" assert len(response.history) == 1 assert response.text == ""
def _merge_url(self, url): if isinstance(url, str): if url.startswith('http://localhost'): return url url = URL(url) new_url = url.copy_with(scheme=self.scheme, host=self.host, port=self.port) if 's3.' in url.host: return new_url.copy_with(path='/s3/') elif 'email.' in url.host: return new_url.copy_with(path='/ses/') elif url.host.startswith('sns.'): if 'bad' in url.path: return new_url.copy_with(path='/status/400/') elif url.path.endswith('.pem'): return new_url.copy_with(path='/sns/certs/') else: return new_url.copy_with(path='/status/200/') else: # return url raise ValueError(f'no local endpoint found for "{url}"')
def _sign(self, base: str, endpoint: str, params: Dict[str, Any]) -> URL: params.update( { **BilibiliConstants.DEFAULT_PARAMS, "access_key": BilibiliConstants.ACCESS_KEY, "appkey": BilibiliConstants.APP_KEY, "ts": int(time()), } ) params = {k: params[k] for k in sorted(params.keys())} url = self._join(base=base, endpoint=endpoint, params=params) sign = hashlib.md5(string=(url.query + BilibiliConstants.SECRET)).hexdigest() return URL(url, params={"sign": sign})
def test_get_service_status( mocked_director_v0_service_api: MockRouter, mocked_director_v2_scheduler: None, client: TestClient, service: Dict[str, Any], exp_status_code: int, is_legacy: bool, ): url = URL(f"/v2/dynamic_services/{service['node_uuid']}") response = client.get(str(url), allow_redirects=False) assert ( response.status_code == exp_status_code ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: # check redirection header goes to director-v0 assert "location" in response.headers redirect_url = URL(response.headers["location"]) assert redirect_url.host == "director" assert (redirect_url.path == f"/v0/running_interactive_services/{service['node_uuid']}") assert redirect_url.params == QueryParams("") # empty query
def _url_match(self, request: Request) -> bool: if not self.url: return True # re.Pattern was introduced in Python 3.7 if isinstance( self.url, re._pattern_type if hasattr(re, "_pattern_type") else re.Pattern ): return self.url.match(str(request.url)) is not None if isinstance(self.url, str): return URL(self.url) == request.url return self.url == request.url
def add_signed_download_params(self, method: Literal['GET', 'POST'], url: URL, expires: int = 86400) -> URL: assert expires >= 1, f'expires must be greater than or equal to 1, not {expires}' assert expires <= 604800, f'expires must be less than or equal to 604800, not {expires}' now = utcnow() url = url.copy_merge_params({ 'X-Amz-Algorithm': _AUTH_ALGORITHM, 'X-Amz-Credential': self._aws4_credential(now), 'X-Amz-Date': _aws4_x_amz_date(now), 'X-Amz-Expires': str(expires), 'X-Amz-SignedHeaders': 'host', }) _, signature = self._aws4_signature(now, method, url, {'host': self.host}, 'UNSIGNED-PAYLOAD') return url.copy_add_param('X-Amz-Signature', signature)
async def _imageDownload( self, client: AsyncClient, data: models.DanbooruImage, ) -> models.ImageDownload: while self._running >= self._workers: await asyncio.sleep(1) self._running += 1 urlParsed = URL(data.imageURL) logger.trace("Start downloading picture " + f"{urlParsed.full_path!r} from {urlParsed.host!r}.") tempfile, hashData, totalWrite = TempFile().create(), HashCreator(), 0 try: response = await client.get( urlParsed, headers={ "User-Agent": randChoice(ImageSpiderConfig["user-agents"].get(list) or [f"DanbooruSpider/{VERSION}"]), }, ) response.raise_for_status() async with AsyncOpen(str(tempfile), "wb") as f: async for chunk in response.aiter_bytes(): await hashData.update(chunk) totalWrite += await f.write(chunk) logger.trace("Finished downloading picture " + f"{urlParsed.full_path!r} from {urlParsed.host!r}, " + f"total write {totalWrite} bytes.") except HTTPError as e: raise NetworkException( "There was an error in the network when processing the picture " + f"'{urlParsed}', the reason is: {e}") except Exception as e: raise SpiderException( "There was a unknown error when processing the picture " + f"'{urlParsed}', the reason is: {e}") finally: self._running -= 1 return models.ImageDownload( **{ "source": str(urlParsed), "path": tempfile, "size": totalWrite, "md5": await hashData.hexdigest(), "data": data, })
async def create_user_with_password(endpoint: URL, username: EmailStr, password: SecretStr) -> int: try: async with AsyncClient(base_url=endpoint.join("v0")) as client: typer.secho( f"registering user {username} with password {password}", fg=typer.colors.YELLOW, ) await register_user(client, username, password) typer.secho(f"user registered", fg=typer.colors.YELLOW) await logout_current_user(client) typer.secho(f"registration done", fg=typer.colors.YELLOW) except Exception as exc: # pylint: disable=broad-except typer.secho(f"Unexpected issue: {exc}", fg=typer.colors.RED, err=True) return 1 return 0
def _join(base: str, endpoint: str, params: Dict[str, Any]) -> URL: host: ParseResult = urlparse(base) params = { k: (v.value if isinstance(v, Enum) else v) for k, v in params.items() if v is not None } return URL( url=ParseResult( scheme=host.scheme, netloc=host.netloc, path=endpoint.format(**params), params="", query="", fragment="", ).geturl(), params=params, )
async def auth_call(self, request, get_response, has_method=True): content = await request.read() if has_method: url, headers, body = self.prepare( request.method, str(request.url), request.headers, content) else: url, headers, body = self.prepare( str(request.url), request.headers, content) request.url = URL(url) request.headers.update(headers) if body: body = to_bytes(body) if body != content: request.is_streaming = False request.content = body return await get_response(request)
def test_retrieve( mock_retrieve_features: Optional[MockRouter], mocked_director_v0_service_api: MockRouter, mocked_director_v2_scheduler: None, client: TestClient, service: Dict[str, Any], exp_status_code: int, is_legacy: bool, ) -> None: url = URL(f"/v2/dynamic_services/{service['node_uuid']}:retrieve") response = client.post(str(url), json=dict(port_keys=[]), allow_redirects=False) assert ( response.status_code == exp_status_code ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" assert (response.json() == RetrieveDataOutEnveloped.Config.schema_extra["examples"][0])
async def test_get_default_cluster( clusters_config: None, registered_user: Callable[..., Dict], cluster: Callable[..., Cluster], async_client: httpx.AsyncClient, ): user_1 = registered_user() # NOTE: we should not need the user id for default right? # NOTE: it should be accessible to everyone to run, and only a handful of elected # people shall be able to administer it get_cluster_url = URL("/v2/clusters/default") response = await async_client.get(get_cluster_url) assert response.status_code == status.HTTP_200_OK, f"received {response.text}" returned_cluster = parse_obj_as(ClusterGet, response.json()) assert returned_cluster assert returned_cluster.name == "Default cluster" assert 1 in returned_cluster.access_rights # everyone group is always 1 assert returned_cluster.access_rights[1] == CLUSTER_USER_RIGHTS
def test_url_join(): """ Some basic URL joining tests. """ url = URL("https://example.org:123/path/to/somewhere") assert url.join("/somewhere-else") == "https://example.org:123/somewhere-else" assert ( url.join("somewhere-else") == "https://example.org:123/path/to/somewhere-else" ) assert ( url.join("../somewhere-else") == "https://example.org:123/path/somewhere-else" ) assert url.join("../../somewhere-else") == "https://example.org:123/somewhere-else"
async def clean( endpoint: URL, username: EmailStr, password: SecretStr, project_id: Optional[str] ) -> int: try: async with AsyncClient( base_url=endpoint.join("v0"), timeout=DEFAULT_TIMEOUT ) as client: await login_user(client, username, password) all_projects = [] if project_id: project = await get_project_for_user(client, project_id) if not project: typer.secho( f"project {project_id} not found!", fg=typer.colors.RED, err=True, ) return 1 all_projects = [project] if not all_projects: all_projects = await get_all_projects_for_user(client) if not all_projects: typer.secho("no projects found!", fg=typer.colors.RED, err=True) return 1 total = len(all_projects) typer.secho(f"{total} projects will be deleted...", fg=typer.colors.YELLOW) with typer.progressbar( length=total, label="deleting projects" ) as progressbar: await asyncio.gather( *[ delete_project(client, prj["uuid"], progressbar) for prj in all_projects ] ) typer.secho(f"completed projects deletion", fg=typer.colors.YELLOW) except Exception as exc: # pylint: disable=broad-except typer.secho( f"Unexpected issue: {exc}, [{type(exc)}]", fg=typer.colors.RED, err=True ) return 1 return 0
def auth_flow(self, request: Request) -> Generator[Request, Response, None]: """ Update the request. :param request: A request to be updated with OAuth parameters. """ oauth_parameters = self.oauth_base_parameters hmac_message_query_string = str( QueryParams( sorted( chain( QueryParams(request.url.query).items(), (QueryParams(request.content.decode()).items() if request.method == 'POST' and request.headers.get('content-type') == 'application/x-www-form-urlencoded' else []), oauth_parameters.items(), )))) oauth_parameters['oauth_signature'] = b64encode(s=hmac_new( key='&'.join([ quote(string=self.consumer_secret, safe=b"~"), quote(string=self.oauth_access_token_secret or '', safe=b"~") ]).encode(), msg='&'.join( quote(string=element, safe=b'~') for element in (request.method.upper(), f'{request.url.scheme}://{request.url.host}{request.url.path}', hmac_message_query_string)).encode(), digestmod=sha1).digest()).decode() # The OAuth parameters are added to the query. request.url = URL(url=request.url, params=oauth_parameters) # request.headers['Authorization'] = 'OAuth ' + ', '.join([ # f'{key}="{value}"' for key, value in oauth_parameters.items()] # ) yield request
async def test_create_dynamic_services( minimal_config: None, mocked_director_v0_service_api, docker_swarm: None, mocked_director_v2_scheduler: None, client: TestClient, dynamic_sidecar_headers: Dict[str, str], service: Dict[str, Any], exp_status_code: int, ): # dynamic-sidecar components await setup_scheduler(client.app) await setup_api_client(client.app) post_data = DynamicServiceCreate(**service) response = client.post( "/v2/dynamic_services", headers=dynamic_sidecar_headers, json=json.loads(post_data.json()), ) assert ( response.status_code == exp_status_code ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: # check redirection header goes to director-v0 assert "location" in response.headers redirect_url = URL(response.headers["location"]) assert redirect_url.host == "director" assert redirect_url.path == "/v0/running_interactive_services" assert redirect_url.params["user_id"] == str(service["user_id"]) assert redirect_url.params["project_id"] == service["project_id"] assert redirect_url.params["service_uuid"] == service["node_uuid"] assert redirect_url.params["service_key"] == service["key"] assert redirect_url.params["service_version"] == service["version"] assert redirect_url.params["service_basepath"] == service["basepath"] if exp_status_code == status.HTTP_201_CREATED: # check the returned data pass
def _sign(self, endpoint: str, params: Dict[str, Any]) -> URL: def random_digit(length: int) -> str: return "".join(map(str, [randint(0, 9) for _ in range(length)])) params.update({ "_client_id": "wappc_" + random_digit(13) + "_" + random_digit(3), "_client_type": 2, "_client_version": "9.9.8.32", **{ k.upper(): v for k, v in Config["net"]["params"].as_dict().items() if v }, }) params = {k: params[k] for k in sorted(params.keys())} url = self._join(self.base, endpoint, params) sign = (hashlib.md5(url.query.replace(b"&", b"") + b"tiebaclient!!!").hexdigest().upper()) return URL(url, params={"sign": sign})
def __init__(self, username: str, password: str, **kwargs) -> None: """ Initialize VictorAsyncClient. :param username: User name used to access Victor API. :param username: Password used to access Victor API. :param kwargs: Arguments to pass to the httpx.AsyncClient constructor. """ super().__init__(**kwargs) if not username: raise ValueError("User name is required.") if not password: raise ValueError("Password is required.") if self.base_url == URL(): self.base_url = DEFAULT_BASE_URL self._credentials = {"password": password, "username": username} self._token = None
async def login( cls, *, account: Optional[Tuple[str, str]] = None, refresh_token: Optional[str] = None, ): assert (account and refresh_token) is None url = URL(PixivConstants.AUTH_HOST + "/auth/token") time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00") headers = { **PixivConstants.DEFAULT_HEADERS, "X-Client-Time": time, "X-Client-Hash": hashlib.md5(time.encode() + PixivConstants.HASH_SECRET.encode()).hexdigest(), } data = { "get_secure_url": 1, "client_id": PixivConstants.CLIENT_ID, "client_secret": PixivConstants.CLIENT_SECRET, } if refresh_token: data.update({ "grant_type": "refresh_token", "refresh_token": refresh_token, }) elif account: username, password = account data.update({ "grant_type": "password", "username": username, "password": password, }) async with AsyncHTTPClient(proxies=PixivConstants.CONFIG["proxy"].get( Dict[str, str]) # type:ignore ) as client: response = await client.post(url, data=data, headers=headers) response.raise_for_status() return cls.parse_obj(response.json())
def __init__(self, **kwargs): data = { 'body': {}, 'headers': {}, 'path': {}, 'query': {}, } data['header'] = data['headers'] skip_validation = kwargs.pop('esi_skip_validation', False) errors = {} for param in self.params: value = kwargs.pop(param.safe_name) if value == param.default: continue try: param.validate(value) except ValidationError as e: e.update_error_dict(errors, param.safe_name) data[param.part][param.name] = value if errors and not skip_validation: raise ValidationError(errors) cls = self.__class__ body = None if data['body']: body = list(data['body'].values())[0] self.method = cls.method.upper() self.url = URL(self.path.format(**data['path']), allow_relative=True, params=data['query']) self.headers = Headers(data['header']) self.stream = encode(body, None, None) self.timer = ElapsedTimer() self.prepare() # Clear out headers we don't need (These will be set by the session) for key in ["User-Agent"]: if key in self.headers: del self.headers[key]
async def test_create_cluster( clusters_config: None, registered_user: Callable[..., Dict], cluster_simple_authentication: Callable, async_client: httpx.AsyncClient, faker: Faker, postgres_db: sa.engine.Engine, clusters_cleaner, ): user_1 = registered_user() create_cluster_url = URL(f"/v2/clusters?user_id={user_1['id']}") cluster_data = ClusterCreate( endpoint=faker.uri(), authentication=cluster_simple_authentication(), name=faker.name(), type=random.choice(list(ClusterType)), ) response = await async_client.post(create_cluster_url, json=cluster_data.dict( by_alias=True, exclude_unset=True)) assert response.status_code == status.HTTP_201_CREATED, f"received: {response.text}" created_cluster = parse_obj_as(ClusterGet, response.json()) assert created_cluster for k in created_cluster.dict( exclude={"id", "owner", "access_rights"}).keys(): assert getattr(created_cluster, k) == getattr(cluster_data, k) assert created_cluster.id is not None assert created_cluster.owner == user_1["primary_gid"] assert created_cluster.access_rights == { user_1["primary_gid"]: CLUSTER_ADMIN_RIGHTS } # let's check that DB is correctly setup, there is one entry with postgres_db.connect() as conn: cluster_entry = conn.execute( sa.select([clusters ]).where(clusters.c.name == cluster_data.name)).one()
async def create_payment(self, amount_unit: int, currency: str, body_params: Optional[dict] = None, headers: Optional[Headers] = None) -> Response: target = URL('/g_business/v1/payments') try: headers.update({'Content-Type': 'application/json'}) except AttributeError: headers = Headers({'Content-Type': 'application/json'}) try: body_params.update({ 'flow': 'MATCH_CODE', 'amount_unit': amount_unit, 'currency': currency }) except AttributeError: body_params = { 'flow': 'MATCH_CODE', 'amount_unit': amount_unit, 'currency': currency } return await self.post(target, json=body_params, headers=headers)
async def test_follow_next_link_until_exausted(self): async with TestClient(app) as client: response = await client.get("/test/pagination/links/?limit=10") assert response.status_code == 200 responses = [] while response.links.get("next", {}).get("url"): assert int(response.headers["x-total-count"]) == 30 assert len(response.json()) == 10 url = URL(response.links["next"]["url"]) next = f"{url.path}?{url.query}" logger.warning(next) response = await client.get(next) responses.append(response) assert response.status_code == 200 # get last one assert int(response.headers["x-total-count"]) == 30 assert len(response.json()) == 10 responses.append(response) assert len(responses) == 3
async def test_cross_subdomain_redirect(): client = AsyncClient(dispatch=MockDispatch()) url = "https://example.com/cross_subdomain" response = await client.get(url) assert response.url == URL("https://www.example.org/cross_subdomain")