def __init__(self, ip, check_cert=True): self.result = Check_result() self.ip = ip self.timeout = 5 self.check_cert = check_cert if check_cert: self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%. else: self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt self.max_retry = 3 self.timeout = 3 self.max_timeout = 5 self.max_thread_num = 40 self.connection_pool_num = 30 self.conn_pool = Connect_pool() #Queue.PriorityQueue() # set_ciphers as Modern Browsers # http://www.openssl.org/docs/apps/ciphers.html #ssl_ciphers = [x for x in self.ssl_ciphers if random.random() > 0.5] self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) #, cipher_suites=ssl_ciphers) # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt self.max_retry = 3 self.timeout = 1.5 self.max_timeout = 15 self.thread_num = 0 self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10 self.connection_pool_max_num = config.CONFIG.getint("connect_manager", "https_connection_pool_max") #20/30 self.connection_pool_min_num = config.CONFIG.getint("connect_manager", "https_connection_pool_min") #20/30 self.conn_pool = Connect_pool() #Queue.PriorityQueue() self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) if self.keep_alive: p = threading.Thread(target = self.keep_alive_thread) p.daemon = True p.start() self.keep_alive = True
def test2(self): work_ciphers = ["AES128-SHA"] for cipher in self.cipher_list: if cipher in work_ciphers: continue else: work_ciphers.append(cipher) xlog.debug("%s", cipher) cipher_suites = (work_ciphers) openssl_context = SSLConnection.context_builder( ca_certs=g_cacertfile, cipher_suites=cipher_suites) try: ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context) server_type = test_server_type(ssl, self.ip) xlog.debug("%s", server_type) if "gws" not in server_type: work_ciphers.remove(cipher) except Exception as e: xlog.warn("err:%s", e) try: work_ciphers.remove(cipher) except: pass work_str = "" for cipher in work_ciphers: work_str += cipher + ":" xlog.info("work ciphers:%s", work_str)
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) self.timeout = 4 self.max_timeout = 60 self.thread_num = 0 # after new created ssl_sock timeout(50 seconds) # call the callback. # This callback will put ssl to worker self.ssl_timeout_cb = None self.connecting_more_thread = None self.load_config() p = threading.Thread(target=self.keep_alive_thread) p.daemon = True p.start() if self.connection_pool_min_num: p = threading.Thread(target=self.create_connection_daemon) p.daemon = True p.start() self.create_more_connection()
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) self.timeout = 4 self.max_timeout = 15 self.thread_num = 0 self.load_config() if self.keep_alive: p = threading.Thread(target = self.keep_alive_thread) p.daemon = True p.start() p = threading.Thread(target = self.create_connection_daemon) p.daemon = True p.start()
def test2(self): work_ciphers = ["AES128-SHA"] for cipher in self.cipher_list: if cipher in work_ciphers: continue else: work_ciphers.append(cipher) xlog.debug("%s", cipher) cipher_suites = (work_ciphers) openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites) try: ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context) server_type = test_server_type(ssl, self.ip) xlog.debug("%s", server_type) if "gws" not in server_type: work_ciphers.remove(cipher) except Exception as e: xlog.warn("err:%s", e) try: work_ciphers.remove(cipher) except: pass work_str = "" for cipher in work_ciphers: work_str += cipher + ":" xlog.info("work ciphers:%s", work_str)
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt self.timeout = 2 self.max_timeout = 15 self.thread_num = 0 self.max_thread_num = config.CONFIG.getint( "connect_manager", "https_max_connect_thread") #10 self.connection_pool_max_num = config.CONFIG.getint( "connect_manager", "https_connection_pool_max") #20/30 self.connection_pool_min_num = config.CONFIG.getint( "connect_manager", "https_connection_pool_min") #20/30 self.new_conn_pool = Connect_pool() self.gae_conn_pool = Connect_pool() self.host_conn_pool = {} self.openssl_context = SSLConnection.context_builder( ssl_version="TLSv1", ca_certs=g_cacertfile) # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode( OpenSSL.SSL.SESS_CACHE_BOTH) if self.keep_alive: p = threading.Thread(target=self.keep_alive_thread) p.daemon = True p.start()
def connect_ssl(ip, port=443, timeout=5, openssl_context=None): import struct ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate #server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None #if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock return ssl_sock, connct_time, handshake_time
def test(self): for cipher in self.cipher_list: xlog.debug("%s", cipher) openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,)) try: ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context) server_type = test_server_type(ssl, self.ip) xlog.debug("%s", server_type) except Exception as e: xlog.warn("err:%s", e)
def connect_ssl(ip, port=443, timeout=5, openssl_context=None): import struct ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate # server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None # if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) logging.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock return ssl_sock, connct_time, handshake_time
def load_config(self): self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10 self.connection_pool_max_num = config.CONFIG.getint("connect_manager", "https_connection_pool_max") #20/30 self.connection_pool_min_num = config.CONFIG.getint("connect_manager", "https_connection_pool_min") #20/30 self.keep_alive = config.CONFIG.getint("connect_manager", "https_keep_alive") #1 self.new_conn_pool = Connect_pool() self.gae_conn_pool = Connect_pool() self.host_conn_pool = {} self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt self.max_retry = 3 self.timeout = 3 self.max_timeout = 5 self.thread_num = 0 self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10 self.connection_pool_num = config.CONFIG.getint("connect_manager", "https_connection_pool") #20/30 self.conn_pool = Connect_pool() #Queue.PriorityQueue() self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self): # http://docs.python.org/dev/library/ssl.html # http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html # http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c # openssl s_server -accept 443 -key CA.crt -cert CA.crt # ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) try: self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) except: pass if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) self.class_name = "Https_connection_manager" self.timeout = 4 self.max_timeout = 60 self.thread_num = 0 # after new created ssl_sock timeout(50 seconds) # call the callback. # This callback will put ssl to worker self.ssl_timeout_cb = None self.connecting_more_thread = None self.load_config() p = threading.Thread(target=self.keep_alive_thread) p.daemon = True p.start() if self.connection_pool_min_num: p = threading.Thread(target=self.keep_connection_daemon) p.daemon = True p.start() self.create_more_connection()
def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True): ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock, ip) ssl_sock.set_connect_state() time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() #report_network_ok global network_stat, last_check_time, continue_fail_count network_stat = "OK" last_check_time = time_handshaked continue_fail_count = 0 cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % (issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time return ssl_sock
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.connect_timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, ip_manager.ssl_closed) ssl_sock.set_connect_state() host = random.choice(ns) ssl_sock.set_tlsext_host_name(host) ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #ip_manager.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('COMODO'): ip_manager.report_connect_fail(ip, force_remove=True) raise socket.error( ' certficate is issued by %r, not COMODO' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) handshake_time = int((time_handshaked - time_connected) * 1000) try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False #xlog.deubg("alpn h2:%s", h2) except: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) # ip_manager.update_ip(ip, handshake_time) # handshake time is not the response time, # cloudflare don't have global back-bond network like google. # the reasonable response RTT time should be the HTTP test RTT. xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, handshake_time, ssl_sock.h2) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time.time() ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = self.host connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.connect_timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) ip_manager.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)
current_path = os.path.dirname(os.path.abspath(__file__)) import OpenSSL SSLError = OpenSSL.SSL.WantReadError from config import config import cert_util from openssl_wrap import SSLConnection from appids_manager import appid_manager from proxy import xlog g_cacertfile = os.path.join(current_path, "cacert.pem") openssl_context = SSLConnection.context_builder( ca_certs=g_cacertfile ) # check cacert cost too many cpu, 100 check thread cost 60%. max_timeout = 5 default_socket = socket.socket def load_proxy_config(): global default_socket if config.PROXY_ENABLE: if config.PROXY_TYPE == "HTTP": proxy_type = socks.HTTP elif config.PROXY_TYPE == "SOCKS4": proxy_type = socks.SOCKS4
def check(self, callback=None, check_ca=False): timeout = 5 openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%. ssl_sock = None try: def connect_ssl(ip): import struct ip_port = (ip, 443) if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate #server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None #if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() self.result.connct_time = int((time_connected - time_begin) * 1000) self.result.handshake_time = int((time_handshaked - time_connected) * 1000) logging.debug("conn: %d handshake:%d", self.result.connct_time, self.result.handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock return ssl_sock ssl_sock = connect_ssl(self.ip) # verify SSL certificate issuer. def check_ssl_cert(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) ssl_cert = cert_util.SSLCert(cert) logging.info("CN:%s", ssl_cert.cn) self.result.domain = ssl_cert.cn if check_ca: check_ssl_cert(ssl_sock) if callback: return callback(ssl_sock, self.ip) return True except SSLError as e: logging.debug("Check_appengine %s SSLError:%s", self.ip, e) except IOError as e: logging.debug("Check %s IOError:%s", self.ip, e) pass except httplib.BadStatusLine: #logging.debug('Check_appengine http.bad status line ip:%s', ip) #import traceback #traceback.print_exc() pass except Exception as e: if len(e.args)>0: errno_str = e.args[0] else: errno_str = e.message logging.debug('check_appengine %s %s err:%s', self.ip, errno_str, e) finally: if ssl_sock: ssl_sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def check(self, callback=None, check_ca=False): timeout = 5 openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%. ssl_sock = None try: def connect_ssl(ip): import struct ip_port = (ip, 443) if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate #server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None #if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() self.result.connct_time = int((time_connected - time_begin) * 1000) self.result.handshake_time = int((time_handshaked - time_connected) * 1000) logging.debug("conn: %d handshake:%d", self.result.connct_time, self.result.handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock return ssl_sock ssl_sock = connect_ssl(self.ip) # verify SSL certificate issuer. def check_ssl_cert(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) ssl_cert = cert_util.SSLCert(cert) logging.info("CN:%s", ssl_cert.cn) self.result.domain = ssl_cert.cn if check_ca: check_ssl_cert(ssl_sock) if callback: return callback(ssl_sock, self.ip) return True except SSLError as e: logging.debug("Check_appengine %s SSLError:%s", self.ip, e) except IOError as e: logging.debug("Check %s IOError:%s", self.ip, e) pass except httplib.BadStatusLine: #logging.debug('Check_appengine http.bad status line ip:%s', ip) #import traceback #traceback.print_exc() pass except Exception as e: if len(e.args)>0: errno_str = e.args[0] else: errno_str = e.message logging.debug('check_appengine %s %s err:%s', self.ip, errno_str, e) finally: if ssl_sock: ssl_sock.close() return False
def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True): ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock, ip) ssl_sock.set_connect_state() time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() # report network ok check_local_network.network_stat = "OK" check_local_network.last_check_time = time_handshaked check_local_network.continue_fail_count = 0 cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(" certficate is none") if check_cert: issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "") if __name__ == "__main__": xlog.debug("issued by:%s", issuer_commonname) if not issuer_commonname.startswith("Google"): raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) # xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time return ssl_sock
def get_ssl_socket(sock, server_hostname=None, context=g_context): ssl_sock = SSLConnection(context, sock) if server_hostname: ssl_sock.set_tlsext_host_name(server_hostname) return ssl_sock
from proxy_dir import current_path import OpenSSL SSLError = OpenSSL.SSL.WantReadError import socks import check_local_network from config import config import cert_util from openssl_wrap import SSLConnection from xlog import getLogger xlog = getLogger("gae_proxy") g_cacertfile = os.path.join(current_path, "cacert.pem") openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) max_timeout = 5 default_socket = socket.socket def load_proxy_config(): global default_socket if config.PROXY_ENABLE: if config.PROXY_TYPE == "HTTP": proxy_type = socks.HTTP
def _create_ssl_connection(self, ip_port): sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate #server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None #if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.ip = ip ssl_sock.sock = sock ssl_sock.create_time = time_begin ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) return ssl_sock except Exception as e: logging.debug("create_ssl %s fail:%s c:%d h:%d", ip, e, connect_time, handshake_time) google_ip.report_connect_fail(ip) if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate server_hostname = random_hostname() if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"): ssl_sock.set_tlsext_host_name(server_hostname) pass ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.ip = ip ssl_sock.sock = sock ssl_sock.create_time = time_begin ssl_sock.handshake_time = handshake_time ssl_sock.host = "" def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(" certficate is none") issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "") if not issuer_commonname.startswith("Google"): google_ip.report_bad_ip(ssl_sock.ip) connect_control.fall_into_honeypot() raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def connect_ssl(ip, port=443, timeout=5, openssl_context=None): ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time return ssl_sock
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)
def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True): ip_port = (ip, port) if not openssl_context: openssl_context = SSLConnection.context_builder() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET) else: sock = socket.socket(socket.AF_INET) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = SSLConnection(openssl_context, sock) ssl_sock.set_connect_state() time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time #report_network_ok() global network_ok, last_check_time network_ok = True last_check_time = time_handshaked return ssl_sock
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() def verify_SSL_certificate_issuer(ssl_sock): #cert = ssl_sock.get_peer_certificate() #if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') #issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') #if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: google_ip.report_connect_fail(ip, force_remove=True) raise socket.error('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey( OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: google_ip.report_connect_fail(ip, force_remove=True) raise socket.error( 'The intermediate CA is mismatching.') issuer_commonname = next( (v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error( ' certficate is issued by %r, not Google' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) handshake_time = int((time_handshaked - time_connected) * 1000) try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False #xlog.deubg("alpn h2:%s", h2) except: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, handshake_time, ssl_sock.h2) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)
current_path = os.path.dirname(os.path.abspath(__file__)) import OpenSSL SSLError = OpenSSL.SSL.WantReadError from config import config import cert_util from openssl_wrap import SSLConnection from appids_manager import appid_manager from proxy import xlog g_cacertfile = os.path.join(current_path, "cacert.pem") openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%. max_timeout = 5 default_socket = socket.socket def load_proxy_config(): global default_socket if config.PROXY_ENABLE: if config.PROXY_TYPE == "HTTP": proxy_type = socks.HTTP elif config.PROXY_TYPE == "SOCKS4": proxy_type = socks.SOCKS4 elif config.PROXY_TYPE == "SOCKS5":
def _create_ssl_connection(ip_port): sock = None ssl_sock = None ip = ip_port[0] try: sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate #server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None #if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): # ssl_sock.set_tlsext_host_name(server_hostname) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.sock = sock # verify SSL certificate issuer. def check_ssl_cert(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error( ' certficate is issued by %r, not Google' % (issuer_commonname)) check_ssl_cert(ssl_sock) return ssl_sock except Exception as e: logging.debug("create_ssl %s fail:%s", ip, e) google_ip.report_connect_fail(ip) if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate server_hostname = random_hostname() if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): ssl_sock.set_tlsext_host_name(server_hostname) pass ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.ip = ip ssl_sock.sock = sock ssl_sock.create_time = time_begin ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_bad_ip(ssl_sock.ip) connect_control.fall_into_honeypot() raise socket.error( ' certficate is issued by %r, not Google' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
SSLError = OpenSSL.SSL.WantReadError import socks import check_local_network from config import config import cert_util from openssl_wrap import SSLConnection from xlog import getLogger xlog = getLogger("gae_proxy") import hyper g_cacertfile = os.path.join(current_path, "cacert.pem") openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10))) if hasattr(OpenSSL.SSL, "SESS_CACHE_BOTH"): openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) max_timeout = 5 default_socket = socket.socket def load_proxy_config(): global default_socket if config.PROXY_ENABLE: if config.PROXY_TYPE == "HTTP": proxy_type = socks.HTTP