def __init__(self, app: FastAPI):
dynamic_sidecar_settings: DynamicSidecarSettings = (
app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR)
self._app: FastAPI = app
self._client: httpx.AsyncClient = httpx.AsyncClient(
timeout=httpx.Timeout(
dynamic_sidecar_settings.DYNAMIC_SIDECAR_API_REQUEST_TIMEOUT,
connect=dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT,
))
# timeouts
self._health_request_timeout: httpx.Timeout = httpx.Timeout(
1.0, connect=1.0)
self._save_restore_timeout: httpx.Timeout = httpx.Timeout(
dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT,
connect=dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT,
)
self._restart_containers_timeout: httpx.Timeout = httpx.Timeout(
dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_RESTART_CONTAINERS_TIMEOUT,
connect=dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT,
)
def test_all_bot_args_custom(self, builder, bot, monkeypatch):
defaults = Defaults()
request = HTTPXRequest()
get_updates_request = HTTPXRequest()
builder.token(bot.token).base_url("base_url").base_file_url(
"base_file_url").private_key(PRIVATE_KEY).defaults(
defaults).arbitrary_callback_data(42).request(
request).get_updates_request(get_updates_request)
built_bot = builder.build().bot
# In the following we access some private attributes of bot and request. this is not
# really nice as we want to test the public interface, but here it's hard to ensure by
# other means that the parameters are passed correctly
assert built_bot.token == bot.token
assert built_bot.base_url == "base_url" + bot.token
assert built_bot.base_file_url == "base_file_url" + bot.token
assert built_bot.defaults is defaults
assert built_bot.request is request
assert built_bot._request[0] is get_updates_request
assert built_bot.callback_data_cache.maxsize == 42
assert built_bot.private_key
@dataclass
class Client:
timeout: object
proxies: object
limits: object
monkeypatch.setattr(httpx, "AsyncClient", Client)
builder = ApplicationBuilder().token(bot.token)
builder.connection_pool_size(1).connect_timeout(2).pool_timeout(
3).read_timeout(4).write_timeout(5).proxy_url("proxy_url")
app = builder.build()
client = app.bot.request._client
assert client.timeout == httpx.Timeout(pool=3,
connect=2,
read=4,
write=5)
assert client.limits == httpx.Limits(max_connections=1,
max_keepalive_connections=1)
assert client.proxies == "proxy_url"
builder = ApplicationBuilder().token(bot.token)
builder.get_updates_connection_pool_size(
1).get_updates_connect_timeout(2).get_updates_pool_timeout(
3).get_updates_read_timeout(4).get_updates_write_timeout(
5).get_updates_proxy_url("proxy_url")
app = builder.build()
client = app.bot._request[0]._client
assert client.timeout == httpx.Timeout(pool=3,
connect=2,
read=4,
write=5)
assert client.limits == httpx.Limits(max_connections=1,
max_keepalive_connections=1)
assert client.proxies == "proxy_url"
def test_init(self, monkeypatch):
@dataclass
class Client:
timeout: object
proxies: object
limits: object
monkeypatch.setattr(httpx, "AsyncClient", Client)
request = HTTPXRequest()
assert request._client.timeout == httpx.Timeout(connect=5.0, read=5.0, write=5.0, pool=1.0)
assert request._client.proxies is None
assert request._client.limits == httpx.Limits(
max_connections=1, max_keepalive_connections=1
)
request = HTTPXRequest(
connection_pool_size=42,
proxy_url="proxy_url",
connect_timeout=43,
read_timeout=44,
write_timeout=45,
pool_timeout=46,
)
assert request._client.proxies == "proxy_url"
assert request._client.limits == httpx.Limits(
max_connections=42, max_keepalive_connections=42
)
assert request._client.timeout == httpx.Timeout(connect=43, read=44, write=45, pool=46)
async def test_do_request_manual_timeouts(self, monkeypatch, httpx_request):
default_timeouts = httpx.Timeout(connect=42, read=43, write=44, pool=45)
manual_timeouts = httpx.Timeout(connect=52, read=53, write=54, pool=55)
async def make_assertion(_, **kwargs):
self.test_flag = kwargs.get("timeout") == manual_timeouts
return httpx.Response(HTTPStatus.OK)
async with HTTPXRequest(
connect_timeout=default_timeouts.connect,
read_timeout=default_timeouts.read,
write_timeout=default_timeouts.write,
pool_timeout=default_timeouts.pool,
) as httpx_request:
monkeypatch.setattr(httpx.AsyncClient, "request", make_assertion)
await httpx_request.do_request(
method="GET",
url="URL",
connect_timeout=manual_timeouts.connect,
read_timeout=manual_timeouts.read,
write_timeout=manual_timeouts.write,
pool_timeout=manual_timeouts.pool,
)
assert self.test_flag
def test_timeout_repr():
timeout = httpx.Timeout(timeout=5.0)
assert repr(timeout) == "Timeout(timeout=5.0)"
timeout = httpx.Timeout(None, read=5.0)
assert repr(
timeout) == "Timeout(connect=None, read=5.0, write=None, pool=None)"
def test_timeout_repr():
timeout = httpx.Timeout(timeout=5.0)
assert repr(timeout) == "Timeout(timeout=5.0)"
timeout = httpx.Timeout(read_timeout=5.0)
assert repr(timeout) == ("Timeout(connect_timeout=None, read_timeout=5.0, "
"write_timeout=None, pool_timeout=None)")
def test_default_values(self, bot, monkeypatch, builder):
@dataclass
class Client:
timeout: object
proxies: object
limits: object
monkeypatch.setattr(httpx, "AsyncClient", Client)
app = builder.token(bot.token).build()
assert isinstance(app, Application)
assert app.concurrent_updates == 0
assert isinstance(app.bot, ExtBot)
assert isinstance(app.bot.request, HTTPXRequest)
assert "api.telegram.org" in app.bot.base_url
assert bot.token in app.bot.base_url
assert "api.telegram.org" in app.bot.base_file_url
assert bot.token in app.bot.base_file_url
assert app.bot.private_key is None
assert app.bot.arbitrary_callback_data is False
assert app.bot.defaults is None
get_updates_client = app.bot._request[0]._client
assert get_updates_client.limits == httpx.Limits(
max_connections=1, max_keepalive_connections=1)
assert get_updates_client.proxies is None
assert get_updates_client.timeout == httpx.Timeout(connect=5.0,
read=5.0,
write=5.0,
pool=1.0)
client = app.bot.request._client
assert client.limits == httpx.Limits(max_connections=256,
max_keepalive_connections=256)
assert client.proxies is None
assert client.timeout == httpx.Timeout(connect=5.0,
read=5.0,
write=5.0,
pool=1.0)
assert isinstance(app.update_queue, asyncio.Queue)
assert isinstance(app.updater, Updater)
assert app.updater.bot is app.bot
assert app.updater.update_queue is app.update_queue
assert isinstance(app.job_queue, JobQueue)
assert app.job_queue.application is app
assert app.persistence is None
assert app.post_init is None
assert app.post_shutdown is None
def test_deprecated_verbose_timeout_params():
with pytest.warns(DeprecationWarning):
httpx.Timeout(None, read_timeout=1.0)
with pytest.warns(DeprecationWarning):
httpx.Timeout(None, write_timeout=1.0)
with pytest.warns(DeprecationWarning):
httpx.Timeout(None, connect_timeout=1.0)
with pytest.warns(DeprecationWarning):
httpx.Timeout(None, pool_timeout=1.0)
def __init__(self, app: FastAPI):
dynamic_sidecar_settings: DynamicSidecarSettings = (
app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR)
self._app: FastAPI = app
self._heatlth_request_timeout: httpx.Timeout = httpx.Timeout(
1.0, connect=1.0)
self._base_timeout = httpx.Timeout(
dynamic_sidecar_settings.DYNAMIC_SIDECAR_API_REQUEST_TIMEOUT,
connect=dynamic_sidecar_settings.
DYNAMIC_SIDECAR_API_CONNECT_TIMEOUT,
)
async def backup(data: str, timestamp: int) -> None:
async with httpx.AsyncClient(
headers=Y_DISK_AUTH_HEADERS,
timeout=httpx.Timeout(read_timeout=45.0)) as client:
logger.info('Записываем в Я.Диск')
upload_info_response = await client.get(
Y_DISK_API_URL,
params={'path': f'/{Y_DISK_FOLDER}/gps.txt{timestamp}'})
upload_info_response.raise_for_status()
upload_info = UploadInfo(**upload_info_response.json())
upload_methods = {'PUT': client.put, 'POST': client.post}
try:
backup_response = await upload_methods[upload_info.method
](upload_info.href,
data=data)
backup_response.raise_for_status()
except KeyError:
logger.error('Метод загрузки не реализован {}', upload_info.method)
return
except (httpx.HTTPStatusError, httpx.RequestError) as exc:
if exc.response.status_code == httpx.codes.CONFLICT:
""" Скорее всего файл был записан, можно пропустить"""
else:
logger.exception(exc)
return
logger.success('Координаты записанны')
def kelvin_client_session(school_authority: SchoolAuthorityConfiguration,
plugin_name: str) -> Session:
m: Match = kelvin_url_regex().match(school_authority.url)
if not m:
raise ValueError(
f"Bad Kelvin URL in school authority {school_authority!r}: {school_authority.url!r}."
)
host = m.groupdict()["host"]
try:
username = school_authority.plugin_configs[plugin_name]["username"]
password = school_authority.plugin_configs[plugin_name][
"password"].get_secret_value()
except KeyError as exc:
raise ValueError(
f"Missing {exc!s} in Kelvin plugin configuration of school authority "
f"{school_authority.dict()!r}.")
timeout = httpx.Timeout(timeout=HTTP_REQUEST_TIMEOUT)
certificate_path = fetch_ucs_certificate(host)
ssl_context: ssl.SSLContext = httpx.create_ssl_context(
verify=str(certificate_path))
for k, v in school_authority.plugin_configs[plugin_name].get(
"ssl_context", {}).items():
logger.info("Applying to SSL context: %r=%r", k, v)
setattr(ssl_context, k, v)
return Session(
username=username,
password=password,
host=host,
verify=ssl_context,
timeout=timeout,
)
async def process(self, loop, url):
print(f"processing: {url}")
self.todo.remove(url)
self.busy.add(url)
transport = httpx.AsyncHTTPTransport(retries=3)
client = httpx.AsyncClient(verify=False,
transport=transport,
timeout=httpx.Timeout(10.0, connect=20.0))
try:
resp = await client.get(url)
if resp.status_code != 200:
self.done[url] = False
self.failed[url] = resp.status_code
else:
parsed_feed = await loop.run_in_executor(
None, self._parse_feed, resp.text)
self.done[url] = True
except Exception as exc:
self.done[url] = False
self.failed[url] = {
'type': 'exception',
'message': str(exc),
'other': type(exc)
}
finally:
await client.aclose()
self.busy.remove(url)
async def get_buildkite_pr_buildtime() -> List[BuildkitePRBuildTime]:
http_client = httpx.AsyncClient(timeout=httpx.Timeout(60))
async with http_client:
resp = await http_client.post(
"https://graphql.buildkite.com/v1",
headers={
"Authorization": f"Bearer {os.environ['BUILDKITE_TOKEN']}"
},
json={"query": PR_TIME_QUERY},
)
resp.raise_for_status()
builds = resp.json()["data"]["pipeline"]["builds"]["edges"]
return [
BuildkitePRBuildTime(
commit=build["node"]["commit"],
created_by=list(build["node"].get("createdBy", {
"_": "unknown"
}).values())[0],
state=build["node"]["state"],
url=build["node"]["url"],
created_at=build["node"].get("createdAt"),
started_at=build["node"].get("startedAt"),
finished_at=build["node"].get("finishedAt"),
pull_id=build["node"].get("pullRequest", {
"id": None
}).get("id"),
) for build in builds if build["node"] is not None
and build["node"].get("pullRequest") is not None
]
async def test_connect_timeout(server, backend):
timeout = httpx.Timeout(connect_timeout=1e-6)
async with httpx.Client(timeout=timeout) as client:
with pytest.raises(httpx.ConnectTimeout):
# See https://stackoverflow.com/questions/100841/
await client.get("http://10.255.255.1/")
async def async_request(
method: Text,
url: URLTypes,
params: QueryParamTypes = None,
data: RequestData = None,
files: RequestFiles = None,
json: Any = None,
headers: HeaderTypes = None,
cookies: CookieTypes = None,
timeout: Optional[int] = DEFAULT_TIMEOUT
):
"""
Makes an asynchronous request.
DEFAULT_TIMEOUT is 10 secongs.
"""
if isinstance(timeout, int):
timeout = httpx.Timeout(timeout=timeout)
async with httpx.AsyncClient(
cookies=cookies,
timeout=timeout
) as client:
return await client.request(
method=method,
url=url,
params=params,
data=data,
files=files,
json=json,
headers=headers
)
async def execute_with_callback(request, answer_coalesce_type, callback_url, guid):
"""
Executes an asynchronous ARAGORN request
:param request:
:param answer_coalesce_type:
:param callback_url:
:param guid:
:return:
"""
# capture if this is a test request
if 'test' in request:
test_mode = True
else:
test_mode = False
logger.info(f'{guid}: Awaiting async execute with callback URL: {callback_url}')
# make the asynchronous request
final_msg, status_code = await asyncexecute(request, answer_coalesce_type, guid)
# for some reason the "mock" test endpoint doesnt like the async client post
if test_mode:
callback(callback_url, final_msg, guid)
else:
try:
# send back the result to the specified aragorn callback end point
async with httpx.AsyncClient(timeout=httpx.Timeout(timeout=600.0)) as client:
response = await client.post(callback_url, json=final_msg)
logger.info(f'{guid}: Executed POST to callback URL {callback_url}, response: {response.status_code}')
except Exception as e:
logger.exception(f'{guid}: Exception detected: POSTing to callback {callback_url}', e)
def __init__(
self,
connection_pool_size: int = 1,
proxy_url: str = None,
read_timeout: Optional[float] = 5.0,
write_timeout: Optional[float] = 5.0,
connect_timeout: Optional[float] = 5.0,
pool_timeout: Optional[float] = 1.0,
):
timeout = httpx.Timeout(
connect=connect_timeout,
read=read_timeout,
write=write_timeout,
pool=pool_timeout,
)
limits = httpx.Limits(
max_connections=connection_pool_size,
max_keepalive_connections=connection_pool_size,
)
self._client_kwargs = dict(
timeout=timeout,
proxies=proxy_url,
limits=limits,
)
self._client = self._build_client()
def create_http_client() -> httpx.AsyncClient:
timeout = httpx.Timeout(read=300, pool=200)
pool_limits = httpx.Limits(
max_connections=10,
max_keepalive_connections=GNS3_CONTROLLER_NUM_MAX_CONN)
client = httpx.AsyncClient(pool_limits=pool_limits, timeout=timeout)
return client
async def fetch(params, *, secrets):
access_token = (secrets.get("access_token") or {}).get("secret")
if not access_token:
return i18n.trans("badParam.access_token.empty",
"Please sign in to Intercom")
bearer_token = access_token["access_token"]
try:
# 5min timeouts ... and we'll assume Intercom is quick enough
async with httpx.AsyncClient(timeout=httpx.Timeout(300)) as client:
users = await fetch_users(client, bearer_token)
companies = await fetch_companies(client, bearer_token)
segments = await fetch_segments(client, bearer_token)
tags = await fetch_tags(client, bearer_token)
except httpx.RequestError as err:
return i18n.trans(
"error.httpError.general",
"Error querying Intercom: {error}",
{"error": str(err)},
)
except RuntimeError as err:
return i18n.trans(
"error.unexpectedIntercomJson.general",
"Error handling Intercom response: {error}",
{"error": str(err)},
)
return build_dataframe(users, companies, segments, tags)
def timeout(self, new_timeout: TimeoutTypes):
if isinstance(
new_timeout,
(tuple, float, int, httpx.Timeout)) or new_timeout is None:
self._client.timeout = httpx.Timeout(new_timeout)
else:
raise TypeError
def __init__(
self,
limit: int = 30,
max_connections: int = 100,
timeout: Optional[float] = 20,
env: bool = False,
internal: Optional[bool] = False,
proxies: Optional[str] = None,
):
"""
:param limit:
:param max_connections:
:param timeout:
:param env: debug输出:HTTPX_LOG_LEVEL=debug
:param internal:
:param proxies:
"""
self.internal: Optional[int] = internal
self.client: httpx.AsyncClient = httpx.AsyncClient(
verify=False,
timeout=httpx.Timeout(timeout, connect=60),
proxies=proxies, # type: ignore
limits=httpx.Limits(
max_keepalive_connections=limit, max_connections=max_connections
),
trust_env=env,
follow_redirects=True,
event_hooks={"response": [raise_on_4xx_5xx]},
)
def set_api_timeout(instance: DenonAVRFoundation, attribute: attr.Attribute,
value: float) -> float:
"""Change API timeout on timeout changes too."""
# First change _device.api.host then return value
timeout = httpx.Timeout(value, read=max(value, 15.0))
instance._device.api.timeout = timeout # pylint: disable=protected-access
return value
async def _run_query(self, params: dict) -> Any:
method = params.pop("method_").lower()
timeout = httpx.Timeout(self.TIMEOUT)
if method == "selenium":
try:
browser = webdriver.Remote(
command_executor=self.selenium_url,
desired_capabilities=DesiredCapabilities.CHROME,
)
browser.get(self.url)
source = browser.page_source
finally:
browser.quit()
return source
async with httpx.AsyncClient(headers=self.headers,
timeout=timeout) as client:
handler = getattr(client, method)
if method == "get":
req = handler(self.url, params=params)
else:
req = handler(self.url, data=params)
try:
resp = await req
assert resp.status_code == 200
return resp.content
except AssertionError:
pass
raise NoResponseException
def __init__(self, session=None):
# Base API URL
self.base_url = 'https://mingle.kijiji.ca/api'
# Common HTTP header fields
self.headers = {
'Accept': 'application/xml',
'Accept-Language': 'en-CA',
'User-Agent':
'com.ebay.kijiji.ca 14.0.2 (LGE Nexus 5; Android 6.0.1; en_CA)',
'X-ECG-VER': '1.84',
}
if session:
if not isinstance(session, httpx.Client):
raise KijijiApiException(
"'session' kwarg must be an httpx.Client object")
self.session = session
# Append common headers
self.session.headers = self.headers
else:
# Kijiji sometimes takes a bit longer to respond to API requests
# e.g. for loading conversations
timeout = httpx.Timeout(30.0, connect=30.0)
self.session = httpx.Client(timeout=timeout, headers=self.headers)
def all_records(base_id,
table,
api_key,
http_read_timeout,
sleep=0.2,
user_agent=None):
headers = {"Authorization": "Bearer {}".format(api_key)}
if user_agent is not None:
headers["user-agent"] = user_agent
if http_read_timeout:
timeout = httpx.Timeout(5, read=http_read_timeout)
client = httpx.Client(timeout=timeout)
else:
client = httpx
first = True
offset = None
while first or offset:
first = False
url = "https://api.airtable.com/v0/{}/{}".format(base_id, quote(table))
if offset:
url += "?" + urlencode({"offset": offset})
response = client.get(url, headers=headers)
response.raise_for_status()
data = response.json()
offset = data.get("offset")
yield from data["records"]
if offset and sleep:
time.sleep(sleep)
def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[HostPort]]:
try:
context = httpx.create_ssl_context(http2=True)
# TODO: Support resolution via Authority (SOA) to add support for
# AAAA (IPv6) query
r = httpx.get(
'https://{0}.cloudflare-dns.com/dns-query?name={1}&type=A'.format(
self.flags.cloudflare_dns_mode, host,
),
headers={'accept': 'application/dns-json'},
verify=context,
timeout=httpx.Timeout(timeout=5.0),
proxies={
'all://': None,
},
)
if r.status_code != 200:
return None, None
response = r.json()
answers = response.get('Answer', [])
if len(answers) == 0:
return None, None
# TODO: Utilize TTL to cache response locally
# instead of making a DNS query repeatedly for the same host.
return answers[0]['data'], None
except Exception as e:
logger.info(
'Unable to resolve DNS-over-HTTPS for host {0} : {1}'.format(
host, str(e),
),
)
return None, None
def __init__(self, email: str, password: str):
self.email: str = email
self.password: str = password
# Certain endpoints, such as the one used by GetPlayerLoadouts,
# take a bit longer to recieve data; Hence the increased read_timeout.
self.session: httpx.AsyncClient = httpx.AsyncClient(
timeout=httpx.Timeout(read_timeout=10))
async def test_write_timeout(server):
timeout = httpx.Timeout(None, write=1e-6)
async with httpx.AsyncClient(timeout=timeout) as client:
with pytest.raises(httpx.WriteTimeout):
data = b"*" * 1024 * 1024 * 100
await client.put(server.url.copy_with(path="/slow_response"),
content=data)
async def test_pool_timeout(server):
limits = httpx.Limits(max_connections=1)
timeout = httpx.Timeout(None, pool=1e-4)
async with httpx.AsyncClient(limits=limits, timeout=timeout) as client:
async with client.stream("GET", server.url):
with pytest.raises(httpx.PoolTimeout):
await client.get("http://localhost:8000/")
async def test_write_timeout(server, backend):
timeout = httpx.Timeout(write_timeout=1e-6)
async with httpx.Client(timeout=timeout) as client:
with pytest.raises(httpx.WriteTimeout):
data = b"*" * 1024 * 1024 * 100
await client.put(server.url.copy_with(path="/slow_response"),
data=data)
