def test_load_ssl_config_cert_and_key_invalid_password(
cert_pem_file, cert_encrypted_private_key_file
):
with pytest.raises(ssl.SSLError):
httpx.create_ssl_context(
cert=(cert_pem_file, cert_encrypted_private_key_file, "password1")
)
def resolve_dns(self, host: str, port: int) -> Tuple[Optional[str], Optional[HostPort]]:
try:
context = httpx.create_ssl_context(http2=True)
# TODO: Support resolution via Authority (SOA) to add support for
# AAAA (IPv6) query
r = httpx.get(
'https://{0}.cloudflare-dns.com/dns-query?name={1}&type=A'.format(
self.flags.cloudflare_dns_mode, host,
),
headers={'accept': 'application/dns-json'},
verify=context,
timeout=httpx.Timeout(timeout=5.0),
proxies={
'all://': None,
},
)
if r.status_code != 200:
return None, None
response = r.json()
answers = response.get('Answer', [])
if len(answers) == 0:
return None, None
# TODO: Utilize TTL to cache response locally
# instead of making a DNS query repeatedly for the same host.
return answers[0]['data'], None
except Exception as e:
logger.info(
'Unable to resolve DNS-over-HTTPS for host {0} : {1}'.format(
host, str(e),
),
)
return None, None
def kelvin_client_session(school_authority: SchoolAuthorityConfiguration,
plugin_name: str) -> Session:
m: Match = kelvin_url_regex().match(school_authority.url)
if not m:
raise ValueError(
f"Bad Kelvin URL in school authority {school_authority!r}: {school_authority.url!r}."
)
host = m.groupdict()["host"]
try:
username = school_authority.plugin_configs[plugin_name]["username"]
password = school_authority.plugin_configs[plugin_name][
"password"].get_secret_value()
except KeyError as exc:
raise ValueError(
f"Missing {exc!s} in Kelvin plugin configuration of school authority "
f"{school_authority.dict()!r}.")
timeout = httpx.Timeout(timeout=HTTP_REQUEST_TIMEOUT)
certificate_path = fetch_ucs_certificate(host)
ssl_context: ssl.SSLContext = httpx.create_ssl_context(
verify=str(certificate_path))
for k, v in school_authority.plugin_configs[plugin_name].get(
"ssl_context", {}).items():
logger.info("Applying to SSL context: %r=%r", k, v)
setattr(ssl_context, k, v)
return Session(
username=username,
password=password,
host=host,
verify=ssl_context,
timeout=timeout,
)
def test_load_ssl_config_cert_and_encrypted_key(
cert_pem_file, cert_encrypted_private_key_file, password):
context = httpx.create_ssl_context(cert=(cert_pem_file,
cert_encrypted_private_key_file,
password))
assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED
assert context.check_hostname is True
def test_load_ssl_config_verify_env_file(https_server, ca_cert_pem_file,
config, cert_authority):
os.environ[config] = (ca_cert_pem_file if config.endswith("_FILE") else
str(Path(ca_cert_pem_file).parent))
context = httpx.create_ssl_context(trust_env=True)
cert_authority.configure_trust(context)
assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED
assert context.check_hostname is True
assert len(context.get_ca_certs()) == 1
def test_ssl_config_support_for_keylog_file(tmpdir, monkeypatch): # pragma: nocover
with monkeypatch.context() as m:
m.delenv("SSLKEYLOGFILE", raising=False)
context = httpx.create_ssl_context(trust_env=True)
assert context.keylog_filename is None # type: ignore
filename = str(tmpdir.join("test.log"))
with monkeypatch.context() as m:
m.setenv("SSLKEYLOGFILE", filename)
context = httpx.create_ssl_context(trust_env=True)
assert context.keylog_filename == filename # type: ignore
context = httpx.create_ssl_context(trust_env=False)
assert context.keylog_filename is None # type: ignore
def get_sslcontexts(proxy_url=None,
cert=None,
verify=True,
trust_env=True,
http2=False):
global SSLCONTEXTS
key = (proxy_url, cert, verify, trust_env, http2)
if key not in SSLCONTEXTS:
SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env,
http2)
return SSLCONTEXTS[key]
def test_http2_via_proxy(self) -> None:
assert self.PROXY
response = httpx.get(
'https://httpbin.org/get',
headers={'accept': 'application/json'},
verify=httpx.create_ssl_context(http2=True),
timeout=httpx.Timeout(timeout=5.0),
proxies={
'all://': 'http://localhost:%d' % self.PROXY.flags.port,
},
)
self.assertEqual(response.status_code, 200)
def _make_session(self) -> httpx.Client: connection_pool = httpcore.SyncConnectionPool( local_address = self._session_laddr, uds = self._session_uds_path, #XXX: Argument values duplicated from httpx._client.Client._init_transport: keepalive_expiry = 5.0, #XXX: Value duplicated from httpx._client.KEEPALIVE_EXPIRY max_connections = 100, #XXX: Value duplicated from httpx._config.DEFAULT_LIMITS max_keepalive_connections = 20, #XXX: Value duplicated from httpx._config.DEFAULT_LIMITS ssl_context = httpx.create_ssl_context(trust_env=True), ) return httpx.Client(**self._session_kwargs, base_url = self._session_base, transport = connection_pool)
def download_datasets():
data_folder = Path('data')
if data_folder.exists():
shutil.rmtree(data_folder)
data_folder.mkdir()
data_sets = {
'CovidFaelle_Timeline.csv':
'https://covid19-dashboard.ages.at/data/CovidFaelle_Timeline.csv',
'CovidFallzahlen.csv':
'https://covid19-dashboard.ages.at/data/CovidFallzahlen.csv',
}
# Something about the DH key changed on 2021-03-16 15:00
# UTC+1. Disabling that part of the check for now:
context = httpx.create_ssl_context()
ciphers = DEFAULT_CIPHERS + ':HIGH:!DH:!aNULL'
context.set_ciphers(ciphers)
for key, url in data_sets.items():
(data_folder / key).write_text(
httpx.get(url, verify=context).content.decode('utf-8'))
def __init__(self, hostname, username, password):
self.hostname = hostname
self.username = username
self.password = password
self.cjar = None
self.api_key_enc = None
self.api_iv_enc = None
self.api_key_bytes = None
self.api_iv_bytes = None
# Debugging
logging.basicConfig(level=logging.DEBUG)
# The device uses an ancient DH key that is too small, and self signed
ssl_context = httpx.create_ssl_context(verify=False)
ssl_context.options ^= ssl.OP_NO_TLSv1_1
ssl_context.set_ciphers(
'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:TLS13-AES-128-GCM-SHA256:ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!eNULL:!MD5HIGH:!DH:!aNULL'
)
self.client = httpx.AsyncClient(base_url=f'https://{self.hostname}',
verify=ssl_context)
async def load(self) -> None:
self.loader = await config.load_kube_config(
context=self.context, client_configuration=self.config, persist_config=False
)
self.reload_task = asyncio.create_task(
config.refresh_token(self.loader, self.config)
)
self.api = client.ApiClient(configuration=self.config)
self.core_v1 = client.CoreV1Api(self.api)
# Build an HTTPX client that can talk to kube-apiserver.
client_cert = None
if self.config.cert_file:
client_cert = (self.config.cert_file, self.config.key_file)
ssl_context = httpx.create_ssl_context(
verify=self.config.verify_ssl,
cert=client_cert,
)
if self.config.ssl_ca_cert:
ssl_context.load_verify_locations(cafile=self.config.ssl_ca_cert)
self.client = httpx.AsyncClient(
auth=KubernetesAuth(self.config),
base_url=urljoin(self.config.host, "api/"),
verify=ssl_context,
)
def client(self):
# Construct or reconstruct an AsyncClient instance using parameters
if self._client is None:
ssl_context = httpx.create_ssl_context()
ssl_context.check_hostname = self._secure
ssl_context.verify_mode = ssl.CERT_REQUIRED if self._secure else ssl.CERT_NONE
# Allows dead protocols like SSL and TLS1
ssl_context.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED
self._client = httpx.AsyncClient(
auth=self._auth,
headers=self._headers,
cookies=self._cookies,
verify=ssl_context,
proxies=self._proxies,
timeout=self._timeout,
event_hooks={"request": [drop_cookies_from_request]}
if self._drop_cookies else None,
transport=self._transport)
self._client.max_redirects = 5
return self._client
def test_load_ssl_config():
context = httpx.create_ssl_context()
assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED
assert context.check_hostname is True
def test_create_ssl_context_with_get_request(server, cert_pem_file):
context = httpx.create_ssl_context(verify=cert_pem_file)
response = httpx.get(server.url, verify=context)
assert response.status_code == 200
def test_load_ssl_context():
ssl_context = ssl.create_default_context()
context = httpx.create_ssl_context(verify=ssl_context)
assert context is ssl_context
def test_load_ssl_config_no_verify():
context = httpx.create_ssl_context(verify=False)
assert context.verify_mode == ssl.VerifyMode.CERT_NONE
assert context.check_hostname is False
def test_load_ssl_config_cert_without_key_raises(cert_pem_file):
with pytest.raises(ssl.SSLError):
httpx.create_ssl_context(cert=cert_pem_file)
def test_load_ssl_config_verify_directory():
path = Path(certifi.where()).parent
context = httpx.create_ssl_context(verify=str(path))
assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED
assert context.check_hostname is True
def test_load_ssl_config_verify_existing_file():
context = httpx.create_ssl_context(verify=certifi.where())
assert context.verify_mode == ssl.VerifyMode.CERT_REQUIRED
assert context.check_hostname is True
async def single_download(self, task: Task):
headers = {'User-Agent': ua.random}
http_request = task.request
if http_request['referer']:
headers.setdefault('Referer', http_request['referer'])
elif 'referer' in task.parameters:
headers.setdefault('Referer', task.parameters["referer"])
if http_request['extra_headers']:
headers.update(http_request['extra_headers'])
method = http_request['method']
posts_data = None
if http_request['post_data']:
posts_data = http_request['post_data']
if http_request['use_proxy']:
use_proxy = http_request['use_proxy']
else:
use_proxy = task.parameters['use_proxy']
if http_request['timeout']:
timeout = int(http_request['timeout'])
else:
timeout = int(task.parameters['timeout'])
url = http_request['start_url']
if 'cookies' in task.parameters and task.parameters['cookies']:
cookies = task.parameters['cookies']
else:
cookies = None
self.logger.info(
Colored.green("[Downloader()]: 目前下载URL =>{0}".format(url)))
retry_times = 0
max_time = int(http_request['sleep_time'])
sleep_time = random.uniform(1, max_time)
max_retry = random.randint(4, 15)
while retry_times < max_retry:
proxy = get_random_proxy() if use_proxy else None
if proxy is not None:
self.logger.info(
Colored.green(
"[Downloader()]: 目前使用代理 =>{0}".format(proxy)))
try:
request = Any
ssl_context = httpx.create_ssl_context()
session = httpx.AsyncClient(proxies=proxy, verify=False)
if http_request.get('use_session',
None) or task.parameters.get(
'use_session', None):
self.session_container.setdefault(
task.parameters['spider_name'], session)
session = self.session_container.get(
task.parameters['spider_name'])
async with session:
if method == GET:
request: Coroutine = session.get(url,
headers=headers,
timeout=timeout,
cookies=cookies,
follow_redirects=True)
elif method == POST:
request: Coroutine = session.post(
url,
data=posts_data,
headers=headers,
timeout=timeout,
cookies=cookies,
follow_redirects=True)
response: Response = await request
if response.status_code != httpx.codes.OK:
response.raise_for_status()
if response is None:
return None
byte_content: bytes = await response.aread()
# 返回请求内容
process_type = http_request['return_type']
self.logger.info(
Colored.green(
"[Downloader()]: 请求返回类型>>{0}".format(process_type)))
encoding = chardet.detect(byte_content)['encoding']
time.sleep(sleep_time)
if process_type:
if process_type == TEXT:
return decode_content(byte_content, encoding)
elif process_type == JSON:
text = decode_content(byte_content, encoding)
return json.loads(text)
elif process_type == CSS:
return bs(byte_content, 'html5lib')
text = decode_content(byte_content, encoding).replace(
'\x00', '').strip().encode('utf-8')
parser = HTMLParser(encoding=encoding, remove_comments=True)
return etree.fromstring(text, parser)
except Exception as e:
retry_times += 1
self.logger.error(Colored.red(
"[Downloader()]: 目前下载URL =>{0} 重试次数[{1}/{2}] ERROR: {3}".
format(url, retry_times, max_retry, e)),
exc_info=True)
sleep_time *= random.random() * self.sleep_factor
time.sleep(sleep_time)
def test_load_ssl_config_verify_non_existing_path():
with pytest.raises(IOError):
httpx.create_ssl_context(verify="/path/to/nowhere")
def _send_http2_request_to_server(self, request_headers, req_body,
client_stream_id):
if not self.is_h2_to_server:
raise RuntimeError(
"Unexpected received non is_h2_to_server in _send_http2_request_to_server"
)
request_headers_message = HttpHeaders()
for name, value in request_headers:
request_headers_message.add_header(name.decode("utf-8"),
value.decode("utf-8"))
request_headers = request_headers_message
request_headers = ProxyRequestHandler.filter_headers(request_headers)
scheme = request_headers[':scheme']
replay_server = f"127.0.0.1:{self.server_port}"
path = request_headers[':path']
url = f'{scheme}://{replay_server}{path}'
method = request_headers[':method']
original_request_headers = request_headers
request_headers = self._remove_pseudo_headers(request_headers)
try:
origin = (scheme, replay_server, self.client_sni)
if origin not in self.tls.http_conns:
ssl_context = httpx.create_ssl_context(cert=self.cert_file,
verify=False)
if self.client_sni:
setattr(ssl_context, "old_wrap_socket",
ssl_context.wrap_socket)
def new_wrap_socket(sock, *args, **kwargs):
kwargs['server_hostname'] = self.client_sni
return ssl_context.old_wrap_socket(
sock, *args, **kwargs)
setattr(ssl_context, "wrap_socket", new_wrap_socket)
http2_connection = httpx.Client(verify=ssl_context, http2=True)
self.tls.http_conns[origin] = http2_connection
client = self.tls.http_conns[origin]
response_from_server = client.request(
method=method,
url=url,
headers=request_headers.items(),
content=req_body)
response_body = response_from_server.content
except Exception as e:
if origin in self.tls.http_conns:
del self.tls.http_conns[origin]
self.listening_conn.send_headers(client_stream_id,
((':status', '502')),
end_stream=True)
authority = request_headers.get(':authority', '')
print(f"Connection to '{replay_server}' initiated with request to "
f"'{scheme}://{authority}{path}' failed: {e}")
traceback.print_exc(file=sys.stdout)
return
setattr(
response_from_server, 'headers',
ProxyRequestHandler.filter_headers(response_from_server.headers))
response_headers = ((':status',
str(response_from_server.status_code)), )
previous_k = b''
previous_v = b''
for k, v in response_from_server.headers.raw:
if k == b'date' and k == previous_k:
# This happens with date, which HTTPHeaderMap annoyingly splits
# on the comma:
# "Sat, 16 Mar 2019 01:13:21 GMT"
#
# This yields the following two tuples:
# (b'date', b'Sat')
# (b'date', b'16 Mar 2019 01:13:21 GMT')
v = previous_v + b', ' + v
response_headers = response_headers[0:-1]
response_headers += ((k, v), )
previous_k, previous_v = k, v
# httpx will insert a reason phrase for HTTP/2, but there technically
# isn't one, so don't confuse the output with it.
empty_reason_phrase = ''
self.print_info(original_request_headers, req_body, response_headers,
response_body, response_from_server.status_code,
empty_reason_phrase)
return response_headers, response_body
