async def main(): parser = argparse.ArgumentParser() parser.add_argument('--mount', nargs=2, action="append", default=[]) parser.add_argument('--recurse-in-domain', action="append", default=[]) parser.add_argument('--user-agent', default=USER_AGENT) parser.add_argument('--ignore', action="append", default=[]) parser.add_argument('--ignore-glob', action="append", default=[]) parser.add_argument('--ignore-http-code', action="append", default=[]) parser.add_argument('--disable-certificate-verification', action="append", default=[]) parser.add_argument('urls', nargs='*', default=[]) args = parser.parse_args() mounts = {"all://": httpx.AsyncHTTPTransport(http2=True)} for hostname in args.disable_certificate_verification: mounts[f"https://{hostname}"] = httpx.AsyncHTTPTransport(verify=False) for directory, url in args.mount: mounts[url] = AsyncStaticFileTransport(directory=directory) async with httpx.AsyncClient(mounts=mounts) as client: return await check_urls( client=client, urls=args.urls, recurse_in_domains=set(args.recurse_in_domain), user_agent=args.user_agent, ignore=set(args.ignore), ignore_glob=set(args.ignore_glob), ignore_http_codes=set(int(c) for c in args.ignore_http_code))
async def process(self, loop, url): print(f"processing: {url}") self.todo.remove(url) self.busy.add(url) transport = httpx.AsyncHTTPTransport(retries=3) client = httpx.AsyncClient(verify=False, transport=transport, timeout=httpx.Timeout(10.0, connect=20.0)) try: resp = await client.get(url) if resp.status_code != 200: self.done[url] = False self.failed[url] = resp.status_code else: parsed_feed = await loop.run_in_executor( None, self._parse_feed, resp.text) self.done[url] = True except Exception as exc: self.done[url] = False self.failed[url] = { 'type': 'exception', 'message': str(exc), 'other': type(exc) } finally: await client.aclose() self.busy.remove(url)
async def query_items(collection, url, params, max_sync_queries=10, item_template="${year}/${month}/${day}"): found = hits(url, params) limit = params['limit'] semaphore = asyncio.Semaphore(max_sync_queries) total_pages = math.ceil(found / limit) logger.info(f"Found {found} items ({total_pages} pages)") queries = [] transport = httpx.AsyncHTTPTransport(retries=3) limits = httpx.Limits(max_keepalive_connections=None, max_connections=5000) async with httpx.AsyncClient(timeout=None, pool_limits=limits, transport=transport) as client: for p in range(1, total_pages + 1): _params = {'page': p} _params.update(params) queries.append( query_items_page(collection, url, client, semaphore, params=_params, item_template=item_template)) return await asyncio.gather(*queries)
def _instrument_client( client: typing.Union[httpx.Client, httpx.AsyncClient], tracer_provider: TracerProvider = None, request_hook: typing.Optional[RequestHook] = None, response_hook: typing.Optional[ResponseHook] = None, ) -> None: """Enables instrumentation for the given Client or AsyncClient""" # pylint: disable=protected-access if isinstance(client, httpx.Client): transport = client._transport or httpx.HTTPTransport() telemetry_transport = SyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) elif isinstance(client, httpx.AsyncClient): transport = client._transport or httpx.AsyncHTTPTransport() telemetry_transport = AsyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) else: raise TypeError("Invalid client provided") client._transport = telemetry_transport
async def test_uds(): async with respx.mock: uds = httpx.AsyncHTTPTransport(uds="/tmp/foobar.sock") async with httpx.AsyncClient(transport=uds) as client: request = respx.get("https://foo.bar/") % 202 response = await client.get("https://foo.bar/") assert request.called is True assert response.status_code == 202
def __init__(self, conf: Dict) -> None: super().__init__(conf) self.session = httpx.AsyncClient( auth=DigestAuthCached( username=self._conf.auth.user, password=self._conf.auth.password, ), transport=httpx.AsyncHTTPTransport(verify=False, retries=3), )
async def client(): if httpx_version >= (0, 20): transport = httpx.AsyncHTTPTransport(uds=SOCKPATH) else: transport = httpcore.AsyncConnectionPool(uds=SOCKPATH) for _ in range(40): async with httpx.AsyncClient(transport=transport) as client: r = await client.get("http://localhost/sleep/0.1") assert r.status_code == 200 assert r.text == "Slept 0.1 seconds.\n"
def get_client() -> httpx.AsyncClient: """Returns a httpx client that can be used with get_updates().""" transport = httpx.AsyncHTTPTransport( limits=httpx.Limits(max_keepalive_connections=5), retries=5, ) headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'packet-windoze', } return httpx.AsyncClient(headers=headers, timeout=30, transport=transport)
async def client(app, loop): if httpx_version >= (0, 20): transport = httpx.AsyncHTTPTransport(uds=SOCKPATH) else: transport = httpcore.AsyncConnectionPool(uds=SOCKPATH) try: async with httpx.AsyncClient(transport=transport) as client: r = await client.get("http://myhost.invalid/") assert r.status_code == 200 assert r.text == os.path.abspath(SOCKPATH) finally: app.stop()
def get_transport(verify, http2, local_address, proxy_url, limit, retries): verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify return httpx.AsyncHTTPTransport( # pylint: disable=protected-access verify=verify, http2=http2, limits=limit, proxy=httpx._config.Proxy(proxy_url) if proxy_url else None, local_address=local_address, retries=retries, **TRANSPORT_KWARGS, )
def create_transport( self, tracer_provider: typing.Optional["TracerProvider"] = None, request_hook: typing.Optional["AsyncRequestHook"] = None, response_hook: typing.Optional["AsyncResponseHook"] = None, ): transport = httpx.AsyncHTTPTransport() telemetry_transport = AsyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) return telemetry_transport
async def instrumented_async_send(wrapped, instance, args, kwargs): if context.get_value("suppress_instrumentation"): return await wrapped(*args, **kwargs) transport = instance._transport or httpx.AsyncHTTPTransport() telemetry_transport = AsyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) instance._transport = telemetry_transport return await wrapped(*args, **kwargs)
def instrument_client( client: typing.Union[httpx.Client, httpx.AsyncClient], tracer_provider: TracerProvider = None, request_hook: typing.Optional[RequestHook] = None, response_hook: typing.Optional[ResponseHook] = None, ) -> None: """Instrument httpx Client or AsyncClient Args: client: The httpx Client or AsyncClient instance tracer_provider: A TracerProvider, defaults to global request_hook: A hook that receives the span and request that is called right after the span is created response_hook: A hook that receives the span, request, and response that is called right before the span ends """ # pylint: disable=protected-access if not hasattr(client, "_is_instrumented_by_opentelemetry"): client._is_instrumented_by_opentelemetry = False if not client._is_instrumented_by_opentelemetry: if isinstance(client, httpx.Client): client._original_transport = client._transport transport = client._transport or httpx.HTTPTransport() client._transport = SyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) client._is_instrumented_by_opentelemetry = True if isinstance(client, httpx.AsyncClient): transport = client._transport or httpx.AsyncHTTPTransport() client._transport = AsyncOpenTelemetryTransport( transport, tracer_provider=tracer_provider, request_hook=request_hook, response_hook=response_hook, ) client._is_instrumented_by_opentelemetry = True else: _logger.warning( "Attempting to instrument Httpx client while already instrumented" )
def __init__( self, base_url, portal_name, save_metadata, download_only_new_data, base_path=Path(), ): # TODO: try/validate the base url self.base_url = base_url # TODO: check valid portal_name (solo letras/numeros . - _) # Esto va a ser un nombre del dir tambien, que sea lindo sin espacios # y que arranque con una letra por las dudas self.portal_name = portal_name self.save_metadata = save_metadata self.download_only_new_data = download_only_new_data self.client = httpx.AsyncClient(transport=httpx.AsyncHTTPTransport( retries=10)) self.p_base = base_path / self.portal_name self.p_items_md = self.p_base / "items_metadata.json" self.p_files = self.p_base / "files" self.p_metadata = self.p_base / "metadata" self.p_internal_md = self.p_base / "internal_metadata.json" self.p_base.mkdir(exist_ok=True) self.p_files.mkdir(exist_ok=True) self.p_metadata.mkdir(exist_ok=True) self.url_package_list = urljoin(self.base_url, "/api/3/action/package_list") self.url_package_show = urljoin(self.base_url, "/api/3/action/package_show") if not self.p_internal_md.exists(): self.know_resoureces = set() else: self.know_resoureces = set( json.loads(line).get("resource_id") for line in self.p_internal_md.open()) # TODO: add valid formats from config file self.valid_formats = ["csv"]
async def _query(self, query, variables=None, **kwargs) -> Dict: # for graphql queries kwargs["headers"] = { "Content-Type": "application/json", "Accept": "application/json", "Accept-Encoding": "gzip, deflate, br", } payload = {"query": query} if variables: payload["variables"] = variables async with httpx.AsyncClient( proxies=self._proxy, transport=httpx.AsyncHTTPTransport(retries=3)) as client: response = await client.post(f"{self.API_BASE}/graphql", json=payload, **kwargs) return response.json()
async def test_gateway_endpoint( endpoint: AnyUrl, authentication: ClusterAuthentication ) -> None: """This method will try to connect to a gateway endpoint and raise a ConfigurationError in case of problem :raises ConfigurationError: contians some information as to why the connection failed """ try: gateway_auth = await get_gateway_auth_from_params(authentication) async with dask_gateway.Gateway( address=f"{endpoint}", auth=gateway_auth, asynchronous=True ) as gateway: # this does not yet create any connection to the underlying gateway. # since using a fct from dask gateway is going to timeout after a long time # we bypass the pinging by calling in ourselves with a short timeout async with httpx.AsyncClient( transport=httpx.AsyncHTTPTransport(retries=2) ) as client: # try to get something the api shall return fast response = await client.get( f"{endpoint}/api/version", timeout=_PING_TIMEOUT_S ) response.raise_for_status() # now we try to list the clusters to check the gateway responds in a sensible way await gateway.list_clusters() logger.debug("Pinging %s, succeeded", f"{endpoint=}") except ( dask_gateway.GatewayServerError, ClientConnectionError, ClientResponseError, httpx.HTTPError, ) as exc: logger.debug("Pinging %s, failed: %s", f"{endpoint=}", f"{exc=!r}") raise ConfigurationError( f"Could not connect to cluster in {endpoint}: error: {exc}" ) from exc
async def https( q: dns.message.Message, where: str, timeout: Optional[float] = None, port: int = 443, source: Optional[str] = None, source_port: int = 0, one_rr_per_rrset: bool = False, ignore_trailing: bool = False, client: Optional["httpx.AsyncClient"] = None, path: str = "/dns-query", post: bool = True, verify: Union[bool, str] = True, ) -> dns.message.Message: """Return the response obtained after sending a query via DNS-over-HTTPS. *client*, a ``httpx.AsyncClient``. If provided, the client to use for the query. Unlike the other dnspython async functions, a backend cannot be provided in this function because httpx always auto-detects the async backend. See :py:func:`dns.query.https()` for the documentation of the other parameters, exceptions, and return type of this method. """ if not _have_httpx: raise NoDOH("httpx is not available.") # pragma: no cover wire = q.to_wire() try: af = dns.inet.af_for_address(where) except ValueError: af = None transport = None headers = {"accept": "application/dns-message"} if af is not None: if af == socket.AF_INET: url = "https://{}:{}{}".format(where, port, path) elif af == socket.AF_INET6: url = "https://[{}]:{}{}".format(where, port, path) else: url = where if source is not None: transport = httpx.AsyncHTTPTransport(local_address=source[0]) if client: cm: contextlib.AbstractAsyncContextManager = NullContext(client) else: cm = httpx.AsyncClient(http1=True, http2=_have_http2, verify=verify, transport=transport) async with cm as the_client: # see https://tools.ietf.org/html/rfc8484#section-4.1.1 for DoH # GET and POST examples if post: headers.update({ "content-type": "application/dns-message", "content-length": str(len(wire)), }) response = await the_client.post(url, headers=headers, content=wire, timeout=timeout) else: wire = base64.urlsafe_b64encode(wire).rstrip(b"=") twire = wire.decode() # httpx does a repr() if we give it bytes response = await the_client.get(url, headers=headers, timeout=timeout, params={"dns": twire}) # see https://tools.ietf.org/html/rfc8484#section-4.2.1 for info about DoH # status codes if response.status_code < 200 or response.status_code > 299: raise ValueError("{} responded with status code {}" "\nResponse body: {!r}".format( where, response.status_code, response.content)) r = dns.message.from_wire( response.content, keyring=q.keyring, request_mac=q.request_mac, one_rr_per_rrset=one_rr_per_rrset, ignore_trailing=ignore_trailing, ) r.time = response.elapsed.total_seconds() if not q.is_response(r): raise BadResponse return r
def __init__(self): transport = httpx.AsyncHTTPTransport(retries=3) self.client = httpx.AsyncClient(transport=transport)
async def https(q, where, timeout=None, port=443, source=None, source_port=0, one_rr_per_rrset=False, ignore_trailing=False, client=None, path='/dns-query', post=True, verify=True): """Return the response obtained after sending a query via DNS-over-HTTPS. *client*, a ``httpx.AsyncClient``. If provided, the client to use for the query. Unlike the other dnspython async functions, a backend cannot be provided in this function because httpx always auto-detects the async backend. See :py:func:`dns.query.https()` for the documentation of the other parameters, exceptions, and return type of this method. """ if not _have_httpx: raise NoDOH('httpx is not available.') # pragma: no cover wire = q.to_wire() try: af = dns.inet.af_for_address(where) except ValueError: af = None transport = None headers = {"accept": "application/dns-message"} if af is not None: if af == socket.AF_INET: url = 'https://{}:{}{}'.format(where, port, path) elif af == socket.AF_INET6: url = 'https://[{}]:{}{}'.format(where, port, path) else: url = where if source is not None: transport = httpx.AsyncHTTPTransport(local_address=source[0]) # After 3.6 is no longer supported, this can use an AsyncExitStack client_to_close = None try: if not client: client = httpx.AsyncClient(http1=True, http2=_have_http2, verify=verify, transport=transport) client_to_close = client # see https://tools.ietf.org/html/rfc8484#section-4.1.1 for DoH # GET and POST examples if post: headers.update({ "content-type": "application/dns-message", "content-length": str(len(wire)) }) response = await client.post(url, headers=headers, content=wire, timeout=timeout) else: wire = base64.urlsafe_b64encode(wire).rstrip(b"=") wire = wire.decode() # httpx does a repr() if we give it bytes response = await client.get(url, headers=headers, timeout=timeout, params={"dns": wire}) finally: if client_to_close: await client.aclose() # see https://tools.ietf.org/html/rfc8484#section-4.2.1 for info about DoH # status codes if response.status_code < 200 or response.status_code > 299: raise ValueError('{} responded with status code {}' '\nResponse body: {}'.format(where, response.status_code, response.content)) r = dns.message.from_wire(response.content, keyring=q.keyring, request_mac=q.request_mac, one_rr_per_rrset=one_rr_per_rrset, ignore_trailing=ignore_trailing) r.time = response.elapsed if not q.is_response(r): raise BadResponse return r
async def worker(args): global num_users, spinner rel = args[0] rows = args[1] global start_time async with rows: async for row in rows: user_profile_complete = build_profile(row) async with httpx.AsyncClient( timeout=120, transport=httpx.AsyncHTTPTransport(retries=5)) as client: try: r = await client.post( org + rel, headers=headers, data=json.dumps(user_profile_complete)) except httpx.ConnectError as exc: spinner.info( f"yamit encountered a name resolution error and is logging your progress. Process failed on {row[attributes.index('login')]}" ) with open('log.csv', 'a', newline='') as logger: w = csv.writer(logger) w.writerow([ 'Failure', row[attributes.index('login')], "Connect error, unable to resolve name.", exc.request.url ]) logger.close() sys.exit(1) while r.status_code == 429: if reset_time_in_seconds != 0: await trio.sleep(reset_time_in_seconds) else: await trio.sleep( int(r.headers['x-rate-limit-reset']) - int(time()) + 10) try: r = await client.post( org + rel, headers=headers, data=json.dumps(user_profile_complete)) except httpx.ConnectError as exc: with open('log.csv', 'a', newline='') as logger: w = csv.writer(logger) w.writerow([ 'Failure', row[attributes.index('login')], "Connect error, unable to resolve name.", exc.request.url ]) logger.close() sys.exit(1) num_users += 1 if num_users % notify == 0: spinner.text = f"Last imported {row[attributes.index('login')]} \t total {num_users} \t runtime {int(int(time() - start_time)/60)} minutes \t status {r.status_code}" if speed != 100: limit = int(r.headers['x-rate-limit-limit']) remaining = int(r.headers['x-rate-limit-remaining']) if (remaining <= (limit + N - (limit * speed / 100)) ) and (int(r.headers['x-rate-limit-reset']) - int(time())) > 0: await trio.sleep( int(r.headers['x-rate-limit-reset']) - int(time())) if r.status_code != 200 and r.status_code != 429: with open('log.csv', 'a', newline='') as logger: w = csv.writer(logger) try: w.writerow([ 'Failure', row[attributes.index('login')], r.json()['errorSummary'], r.status_code ]) except json.decoder.JSONDecodeError: w.writerow([ 'Failure', "", "Error decoding JSON. More information on next line.", r.status_code ]) w.writerow([ 'Failure', row[attributes.index('login')], r.text, r.status_code ]) logger.close()
async def _fetch_binary_data(self, url: str) -> bytes: async with httpx.AsyncClient( proxies=self._proxy, transport=httpx.AsyncHTTPTransport(retries=3)) as client: response = await client.get(url) return response.read()
app.state.config = app_config timeout = httpx.Timeout( app_config.origin_request_timeout_seconds(), connect=app_config.origin_request_connect_timeout_seconds(), ) limits = httpx.Limits( max_keepalive_connections=app_config. origin_request_max_keepalive_connections(), max_connections=app_config.origin_request_max_connections(), ) transport = httpx.AsyncHTTPTransport( http2=app_config.origin_request_http2(), limits=limits, local_address=app_config.origin_request_local_address(), ) app.state.httpx_client = httpx.AsyncClient(timeout=timeout, transport=transport) if app_config.sentry_dsn(): if app_config.sentry_traces_sample_rate(): sentry_sdk.init( # pylint: disable=abstract-class-instantiated; see https://github.com/getsentry/sentry-python/issues/1081 dsn=app_config.sentry_dsn(), traces_sample_rate=app_config.sentry_traces_sample_rate(), ) else: sentry_sdk.init( # pylint: disable=abstract-class-instantiated dsn=app_config.sentry_dsn())