def check_updated_certs(_address, _port, certhashlist, newhash=None, timeout=config.default_timeout, connect_timeout=config.connect_timeout, traversefunc=None):
update_list = []
if None in [_address, _port]:
logging.error("address or port empty")
return None
addr, _port = url_to_ipv6(_address, _port)
cont = default_sslcont()
con = HTTPSConnection(addr, _port, context=cont, timeout=connect_timeout)
try:
con.connect()
except (ConnectionRefusedError, socket.timeout):
if not traversefunc:
logging.warning("Connection failed")
return None
# try_traverse does not work here, scnreqest creates loop
con.sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
con.sock.bind(('', 0))
traversefunc(("", con.sock.getsockname()[1]))
con.sock.settimeout(connect_timeout)
for count in range(0, config.traverse_retries):
try:
con.sock.connect((addr, _port))
break
except Exception:
pass
else:
logging.warning("traversal failed")
return None
con.sock.settimeout(timeout)
oldhash = dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip())
if newhash and newhash != oldhash:
return None
oldsslcont = con.sock.context
for _hash, _security in certhashlist:
con.request("POST", "/usebroken/{hash}".format(hash=_hash), headers=cert_update_header)
con.sock = con.sock.unwrap()
con.sock = cont.wrap_socket(con.sock, server_side=False)
con.sock.do_handshake()
brokensslcert = ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()
con.sock = con.sock.unwrap()
# without next line the connection would be unencrypted now
con.sock = oldsslcont.wrap_socket(con.sock, server_side=False)
# con.sock.do_handshake()
ret = con.getresponse()
if ret.status != 200:
logging.info("checking cert failed, code: %s, reason: %s", ret.status, ret.reason)
continue
if con.sock and oldhash != dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()):
logging.error("certificate switch detected, stop checking")
break
if dhash(brokensslcert) == _hash:
update_list.append((_hash, _security))
con.close()
return update_list
def check_one_site(site):
domain = site["domain"]
log.info("Checking {}".format(domain))
site["status"] = "bad"
site["checks"] = checks = []
good_connection = Check()
checks.append(good_connection)
try:
addrs = socket.getaddrinfo(domain,
443,
socket.AF_INET,
proto=socket.IPPROTO_TCP)
except socket.gaierror:
log.warning("DNS lookup for {} failed!".format(domain))
return
info = random.choice(addrs)
sock = socket.socket(info[0], info[1], info[2])
sock.settimeout(10)
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3
# Some platforms (OS X) do not have OP_NO_COMPRESSION
if hasattr(ssl, "OP_NO_COMPRESSION"):
context.options |= ssl.OP_NO_COMPRESSION
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = True
context.load_verify_locations("moz-certs.pem")
secure_sock = context.wrap_socket(sock, server_hostname=domain)
try:
secure_sock.connect(info[4])
except ConnectionRefusedError:
good_connection.fail("Nothing is listening on port 443.")
return
except socket.timeout:
good_connection.fail("<code>connect()</code> to port 443 times out.")
return
except ssl.SSLError as e:
if e.reason == "CERTIFICATE_VERIFY_FAILED":
desc = "Certificate not trusted by Mozilla cert store."
else:
desc = "A TLS-related error ({}) occurs when trying to connect.".format(
e.reason)
good_connection.fail(desc)
return
except ssl.CertificateError:
good_connection.fail("Certificate hostname verification fails.")
return
except OSError as e:
error_name = errno.errorcode[e.errno]
good_connection.fail(
"<code>connect()</code> returns with error {}.".format(error_name))
return
msg = "A verified TLS connection can be established. "
grade = site.get("ssllabs_grade")
if grade is not None:
msg += "<a href=\"https://www.ssllabs.com/ssltest/analyze.html?d={}\">SSL Labs grade</a> is " + grade + "."
else:
msg += "(<a href=\"https://www.ssllabs.com/ssltest/analyze.html?d={}\">SSL Labs report</a>)"
good_connection.succeed(msg.format(domain))
mediocre = False
https_load = Check()
checks.append(https_load)
http = HTTPSConnection(domain, context=context)
http.sock = secure_sock
try:
def stop_on_http_or_domain_change(url):
return url.scheme == "http" or (url.netloc
and url.netloc != domain)
final, resp, tree = fetch_through_redirects(
http, stop_on_http_or_domain_change)
if final is not None and final.scheme == "http":
https_load.fail("The HTTPS site redirects to HTTP.")
return
if tree is not None and has_mixed_content(tree):
https_load.fail("HTML page loaded over HTTPS has mixed content.")
return
good_sts = Check()
checks.append(good_sts)
sts = resp.getheader("Strict-Transport-Security")
if sts is not None:
m = re.search("max-age=(\d+)", sts)
if m is not None:
age = int(m.group(1))
if age >= 2592000:
good_sts.succeed(
"<code>Strict-Transport-Security</code> header is set with a long <code>max-age</code> directive."
)
else:
good_sts.fail(
"<code>Strict-Transport-Security</code> header is set but the <code>max-age</code> is less than 30 days."
)
else:
good_sts.fail(
"<code>Strict-Transport-Security</code> header doesn't contain a <code>max-age</code> directive."
)
else:
good_sts.fail(
"<code>Strict-Transport-Security</code> header is not set.")
if good_sts.failed:
mediocre = True
except socket.timeout:
https_load.fail("Requesting HTTPS page times out.")
return
except Not200 as e:
https_load.fail(
"The HTTPS site returns an error status ({}) on request.".format(
e.status))
return
except OSError as e:
err_msg = errno.errorcode[e.errno]
https_load.fail(
"Encountered error ({}) while loading HTTPS site.".format(err_msg))
return
finally:
http.close()
https_load.succeed("A page can be successfully fetched over HTTPS.")
http_redirect = Check()
checks.append(http_redirect)
http = HTTPConnection(domain)
try:
def stop_on_https(url):
return url.scheme == "https"
final, resp, tree = fetch_through_redirects(http, stop_on_https)
if final is not None and final.scheme == "https":
http_redirect.succeed("HTTP site redirects to HTTPS.")
else:
http_redirect.fail("HTTP site doesn't redirect to HTTPS.")
mediocre = True
except HTTPException:
http_redirect.fail("Encountered HTTP error while loading HTTP site.")
return
except Not200 as e:
http_redirect.fail(
"The HTTP site returns an error status ({}) on request.".format(
e.status))
return
except OSError as e:
err_msg = errno.errorcode[e.errno]
http_redirect.fail(
"Encountered error ({}) while loading HTTP site.".format(err_msg))
return
finally:
http.close()
site["status"] = "mediocre" if mediocre else "good"
def retry_using_http_auth(self, target, req, infourl, auth_methods,
headers):
connection = headers.get('connection')
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
url = req.get_full_url()
host = req.host
if not host:
raise URLError('no host given')
user, pw = self.passwd.find_user_password(None, target)
domain = None
if user is not None and '\\' in user:
domain, user = user.split('\\', 1)
certificate = infourl_to_ssl_certificate(infourl)
try:
more, method, payload = self.create_auth1_message(
domain, user, pw, url, auth_methods, certificate)
except AuthenticationError:
self.logger.warning('No way to perform authentication: URL=%s',
url)
return None
self.logger.debug('Selected auth method=%s payload=%s more=%s', method,
payload, more)
headers.update({
self.auth_header_request:
' '.join([method, payload]),
'Connection':
'keep-alive' if (more or self.auth_is_proxy) else 'close'
})
h = None
if url.startswith('https://'):
try:
old_ssl_context = infourl_to_ssl_context(infourl)
self.logger.debug('Reuse SSL Context: %s', old_ssl_context)
except Exception as e:
self.logger.exception('SSL Context not found')
old_ssl_context = None
h = HTTPSConnection(host, context=old_ssl_context)
else:
h = HTTPConnection(host)
if self._debuglevel or self.logger.getEffectiveLevel(
) == logging.DEBUG:
h.set_debuglevel(1)
if connection and connection.lower() == 'close':
self.logger.debug('New connection required: host=%s', host)
infourl.close()
infourl = None
else:
h.sock = infourl_to_sock(infourl)
self.logger.debug('Reuse connection socket %s', h.sock)
# We must keep the connection because NTLM authenticates the
# connection, not single requests
headers = dict((name.title(), val) for name, val in headers.items())
self.logger.debug('Send request, headers=%s', headers)
payload = None
if sys.version_info.major > 2:
selector = req.selector
else:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
response = h.getresponse()
if response.getheader('set-cookie'):
# this is important for some web applications that store authentication-related
# info in cookies (it took a long time to figure out)
headers['Cookie'] = response.getheader('set-cookie')
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
expected_header = self.auth_header_response.lower()
for header, value in response.getheaders():
if header.lower() != expected_header:
self.logger.debug('Not matched header: %s = %s (!= %s)',
header.lower(), value, expected_header)
continue
match = re.match(r'^(?:{}\s+)?([a-zA-Z].*)$'.format(method), value,
re.IGNORECASE)
if not match:
self.logger.debug('Not matched value: %s = %s (method=%s)',
header, value, method)
continue
payload, = match.groups()
self.logger.debug('Found auth header: %s = %s', header, payload)
break
if more:
if not payload:
self.logger.error(
'Auth header response not found, Status=%s URL=%s',
response.status, url)
return None
self.logger.debug('Step2: Method: %s, Payload: %s', method,
payload)
try:
more, method, payload = self.create_auth2_message(payload)
except AuthenticationError as e:
self.logger.error('Step2: Authentication failed (%s)', e)
return None
if more:
self.logger.debug('Step2: Method: %s, Response Payload: %s',
method, payload)
headers[self.auth_header_request] = ' '.join([method, payload])
try:
consume_response_body(response)
if sys.version_info.major > 2:
selector = req.selector
else:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example
# redirect-responses are not handled!
response = h.getresponse()
except socket.error as err:
self.logger.exception('')
raise URLError(err)
else:
self.logger.debug('Step2: Method: %s, Continuation not required',
method)
infourl = make_infourl(response, req)
if infourl.code == self.auth_code:
self.logger.warning('Authentication failed: URL=%s, CODE=%s',
req.get_full_url(), infourl.code)
else:
self.logger.info('Authentication OK: URL=%s', req.get_full_url())
return infourl
