def check_updated_certs(_address, _port, certhashlist, newhash=None, timeout=config.default_timeout, connect_timeout=config.connect_timeout, traversefunc=None): update_list = [] if None in [_address, _port]: logging.error("address or port empty") return None addr, _port = url_to_ipv6(_address, _port) cont = default_sslcont() con = HTTPSConnection(addr, _port, context=cont, timeout=connect_timeout) try: con.connect() except (ConnectionRefusedError, socket.timeout): if not traversefunc: logging.warning("Connection failed") return None # try_traverse does not work here, scnreqest creates loop con.sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) con.sock.bind(('', 0)) traversefunc(("", con.sock.getsockname()[1])) con.sock.settimeout(connect_timeout) for count in range(0, config.traverse_retries): try: con.sock.connect((addr, _port)) break except Exception: pass else: logging.warning("traversal failed") return None con.sock.settimeout(timeout) oldhash = dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()) if newhash and newhash != oldhash: return None oldsslcont = con.sock.context for _hash, _security in certhashlist: con.request("POST", "/usebroken/{hash}".format(hash=_hash), headers=cert_update_header) con.sock = con.sock.unwrap() con.sock = cont.wrap_socket(con.sock, server_side=False) con.sock.do_handshake() brokensslcert = ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip() con.sock = con.sock.unwrap() # without next line the connection would be unencrypted now con.sock = oldsslcont.wrap_socket(con.sock, server_side=False) # con.sock.do_handshake() ret = con.getresponse() if ret.status != 200: logging.info("checking cert failed, code: %s, reason: %s", ret.status, ret.reason) continue if con.sock and oldhash != dhash(ssl.DER_cert_to_PEM_cert(con.sock.getpeercert(True)).strip().rstrip()): logging.error("certificate switch detected, stop checking") break if dhash(brokensslcert) == _hash: update_list.append((_hash, _security)) con.close() return update_list
def check_one_site(site): domain = site["domain"] log.info("Checking {}".format(domain)) site["status"] = "bad" site["checks"] = checks = [] good_connection = Check() checks.append(good_connection) try: addrs = socket.getaddrinfo(domain, 443, socket.AF_INET, proto=socket.IPPROTO_TCP) except socket.gaierror: log.warning("DNS lookup for {} failed!".format(domain)) return info = random.choice(addrs) sock = socket.socket(info[0], info[1], info[2]) sock.settimeout(10) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 # Some platforms (OS X) do not have OP_NO_COMPRESSION if hasattr(ssl, "OP_NO_COMPRESSION"): context.options |= ssl.OP_NO_COMPRESSION context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_verify_locations("moz-certs.pem") secure_sock = context.wrap_socket(sock, server_hostname=domain) try: secure_sock.connect(info[4]) except ConnectionRefusedError: good_connection.fail("Nothing is listening on port 443.") return except socket.timeout: good_connection.fail("<code>connect()</code> to port 443 times out.") return except ssl.SSLError as e: if e.reason == "CERTIFICATE_VERIFY_FAILED": desc = "Certificate not trusted by Mozilla cert store." else: desc = "A TLS-related error ({}) occurs when trying to connect.".format( e.reason) good_connection.fail(desc) return except ssl.CertificateError: good_connection.fail("Certificate hostname verification fails.") return except OSError as e: error_name = errno.errorcode[e.errno] good_connection.fail( "<code>connect()</code> returns with error {}.".format(error_name)) return msg = "A verified TLS connection can be established. " grade = site.get("ssllabs_grade") if grade is not None: msg += "<a href=\"https://www.ssllabs.com/ssltest/analyze.html?d={}\">SSL Labs grade</a> is " + grade + "." else: msg += "(<a href=\"https://www.ssllabs.com/ssltest/analyze.html?d={}\">SSL Labs report</a>)" good_connection.succeed(msg.format(domain)) mediocre = False https_load = Check() checks.append(https_load) http = HTTPSConnection(domain, context=context) http.sock = secure_sock try: def stop_on_http_or_domain_change(url): return url.scheme == "http" or (url.netloc and url.netloc != domain) final, resp, tree = fetch_through_redirects( http, stop_on_http_or_domain_change) if final is not None and final.scheme == "http": https_load.fail("The HTTPS site redirects to HTTP.") return if tree is not None and has_mixed_content(tree): https_load.fail("HTML page loaded over HTTPS has mixed content.") return good_sts = Check() checks.append(good_sts) sts = resp.getheader("Strict-Transport-Security") if sts is not None: m = re.search("max-age=(\d+)", sts) if m is not None: age = int(m.group(1)) if age >= 2592000: good_sts.succeed( "<code>Strict-Transport-Security</code> header is set with a long <code>max-age</code> directive." ) else: good_sts.fail( "<code>Strict-Transport-Security</code> header is set but the <code>max-age</code> is less than 30 days." ) else: good_sts.fail( "<code>Strict-Transport-Security</code> header doesn't contain a <code>max-age</code> directive." ) else: good_sts.fail( "<code>Strict-Transport-Security</code> header is not set.") if good_sts.failed: mediocre = True except socket.timeout: https_load.fail("Requesting HTTPS page times out.") return except Not200 as e: https_load.fail( "The HTTPS site returns an error status ({}) on request.".format( e.status)) return except OSError as e: err_msg = errno.errorcode[e.errno] https_load.fail( "Encountered error ({}) while loading HTTPS site.".format(err_msg)) return finally: http.close() https_load.succeed("A page can be successfully fetched over HTTPS.") http_redirect = Check() checks.append(http_redirect) http = HTTPConnection(domain) try: def stop_on_https(url): return url.scheme == "https" final, resp, tree = fetch_through_redirects(http, stop_on_https) if final is not None and final.scheme == "https": http_redirect.succeed("HTTP site redirects to HTTPS.") else: http_redirect.fail("HTTP site doesn't redirect to HTTPS.") mediocre = True except HTTPException: http_redirect.fail("Encountered HTTP error while loading HTTP site.") return except Not200 as e: http_redirect.fail( "The HTTP site returns an error status ({}) on request.".format( e.status)) return except OSError as e: err_msg = errno.errorcode[e.errno] http_redirect.fail( "Encountered error ({}) while loading HTTP site.".format(err_msg)) return finally: http.close() site["status"] = "mediocre" if mediocre else "good"
def retry_using_http_auth(self, target, req, infourl, auth_methods, headers): connection = headers.get('connection') # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) url = req.get_full_url() host = req.host if not host: raise URLError('no host given') user, pw = self.passwd.find_user_password(None, target) domain = None if user is not None and '\\' in user: domain, user = user.split('\\', 1) certificate = infourl_to_ssl_certificate(infourl) try: more, method, payload = self.create_auth1_message( domain, user, pw, url, auth_methods, certificate) except AuthenticationError: self.logger.warning('No way to perform authentication: URL=%s', url) return None self.logger.debug('Selected auth method=%s payload=%s more=%s', method, payload, more) headers.update({ self.auth_header_request: ' '.join([method, payload]), 'Connection': 'keep-alive' if (more or self.auth_is_proxy) else 'close' }) h = None if url.startswith('https://'): try: old_ssl_context = infourl_to_ssl_context(infourl) self.logger.debug('Reuse SSL Context: %s', old_ssl_context) except Exception as e: self.logger.exception('SSL Context not found') old_ssl_context = None h = HTTPSConnection(host, context=old_ssl_context) else: h = HTTPConnection(host) if self._debuglevel or self.logger.getEffectiveLevel( ) == logging.DEBUG: h.set_debuglevel(1) if connection and connection.lower() == 'close': self.logger.debug('New connection required: host=%s', host) infourl.close() infourl = None else: h.sock = infourl_to_sock(infourl) self.logger.debug('Reuse connection socket %s', h.sock) # We must keep the connection because NTLM authenticates the # connection, not single requests headers = dict((name.title(), val) for name, val in headers.items()) self.logger.debug('Send request, headers=%s', headers) payload = None if sys.version_info.major > 2: selector = req.selector else: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) response = h.getresponse() if response.getheader('set-cookie'): # this is important for some web applications that store authentication-related # info in cookies (it took a long time to figure out) headers['Cookie'] = response.getheader('set-cookie') # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one expected_header = self.auth_header_response.lower() for header, value in response.getheaders(): if header.lower() != expected_header: self.logger.debug('Not matched header: %s = %s (!= %s)', header.lower(), value, expected_header) continue match = re.match(r'^(?:{}\s+)?([a-zA-Z].*)$'.format(method), value, re.IGNORECASE) if not match: self.logger.debug('Not matched value: %s = %s (method=%s)', header, value, method) continue payload, = match.groups() self.logger.debug('Found auth header: %s = %s', header, payload) break if more: if not payload: self.logger.error( 'Auth header response not found, Status=%s URL=%s', response.status, url) return None self.logger.debug('Step2: Method: %s, Payload: %s', method, payload) try: more, method, payload = self.create_auth2_message(payload) except AuthenticationError as e: self.logger.error('Step2: Authentication failed (%s)', e) return None if more: self.logger.debug('Step2: Method: %s, Response Payload: %s', method, payload) headers[self.auth_header_request] = ' '.join([method, payload]) try: consume_response_body(response) if sys.version_info.major > 2: selector = req.selector else: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example # redirect-responses are not handled! response = h.getresponse() except socket.error as err: self.logger.exception('') raise URLError(err) else: self.logger.debug('Step2: Method: %s, Continuation not required', method) infourl = make_infourl(response, req) if infourl.code == self.auth_code: self.logger.warning('Authentication failed: URL=%s, CODE=%s', req.get_full_url(), infourl.code) else: self.logger.info('Authentication OK: URL=%s', req.get_full_url()) return infourl