def __init__(self, ip, check_cert=True):
self.result = Check_result()
self.ip = ip
self.timeout = 5
self.check_cert = check_cert
if check_cert:
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
else:
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
def __init__(self, ip, check_cert=True):
self.result = Check_result()
self.ip = ip
self.timeout = 5
self.check_cert = check_cert
if check_cert:
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
else:
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
self.max_retry = 3
self.timeout = 3
self.max_timeout = 5
self.max_thread_num = 40
self.connection_pool_num = 30
self.conn_pool = Connect_pool() #Queue.PriorityQueue()
# set_ciphers as Modern Browsers
# http://www.openssl.org/docs/apps/ciphers.html
#ssl_ciphers = [x for x in self.ssl_ciphers if random.random() > 0.5]
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) #, cipher_suites=ssl_ciphers)
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
self.max_retry = 3
self.timeout = 1.5
self.max_timeout = 15
self.thread_num = 0
self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10
self.connection_pool_max_num = config.CONFIG.getint("connect_manager", "https_connection_pool_max") #20/30
self.connection_pool_min_num = config.CONFIG.getint("connect_manager", "https_connection_pool_min") #20/30
self.conn_pool = Connect_pool() #Queue.PriorityQueue()
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile)
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
if self.keep_alive:
p = threading.Thread(target = self.keep_alive_thread)
p.daemon = True
p.start()
self.keep_alive = True
def test2(self):
work_ciphers = ["AES128-SHA"]
for cipher in self.cipher_list:
if cipher in work_ciphers:
continue
else:
work_ciphers.append(cipher)
xlog.debug("%s", cipher)
cipher_suites = (work_ciphers)
openssl_context = SSLConnection.context_builder(
ca_certs=g_cacertfile, cipher_suites=cipher_suites)
try:
ssl, _, _ = connect_ssl(self.ip,
openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
if "gws" not in server_type:
work_ciphers.remove(cipher)
except Exception as e:
xlog.warn("err:%s", e)
try:
work_ciphers.remove(cipher)
except:
pass
work_str = ""
for cipher in work_ciphers:
work_str += cipher + ":"
xlog.info("work ciphers:%s", work_str)
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
self.timeout = 4
self.max_timeout = 60
self.thread_num = 0
# after new created ssl_sock timeout(50 seconds)
# call the callback.
# This callback will put ssl to worker
self.ssl_timeout_cb = None
self.connecting_more_thread = None
self.load_config()
p = threading.Thread(target=self.keep_alive_thread)
p.daemon = True
p.start()
if self.connection_pool_min_num:
p = threading.Thread(target=self.create_connection_daemon)
p.daemon = True
p.start()
self.create_more_connection()
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
self.timeout = 4
self.max_timeout = 15
self.thread_num = 0
self.load_config()
if self.keep_alive:
p = threading.Thread(target = self.keep_alive_thread)
p.daemon = True
p.start()
p = threading.Thread(target = self.create_connection_daemon)
p.daemon = True
p.start()
def test2(self):
work_ciphers = ["AES128-SHA"]
for cipher in self.cipher_list:
if cipher in work_ciphers:
continue
else:
work_ciphers.append(cipher)
xlog.debug("%s", cipher)
cipher_suites = (work_ciphers)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
if "gws" not in server_type:
work_ciphers.remove(cipher)
except Exception as e:
xlog.warn("err:%s", e)
try:
work_ciphers.remove(cipher)
except:
pass
work_str = ""
for cipher in work_ciphers:
work_str += cipher + ":"
xlog.info("work ciphers:%s", work_str)
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
self.timeout = 2
self.max_timeout = 15
self.thread_num = 0
self.max_thread_num = config.CONFIG.getint(
"connect_manager", "https_max_connect_thread") #10
self.connection_pool_max_num = config.CONFIG.getint(
"connect_manager", "https_connection_pool_max") #20/30
self.connection_pool_min_num = config.CONFIG.getint(
"connect_manager", "https_connection_pool_min") #20/30
self.new_conn_pool = Connect_pool()
self.gae_conn_pool = Connect_pool()
self.host_conn_pool = {}
self.openssl_context = SSLConnection.context_builder(
ssl_version="TLSv1", ca_certs=g_cacertfile)
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(
OpenSSL.SSL.SESS_CACHE_BOTH)
if self.keep_alive:
p = threading.Thread(target=self.keep_alive_thread)
p.daemon = True
p.start()
def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
import struct
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
return ssl_sock, connct_time, handshake_time
def test(self):
for cipher in self.cipher_list:
xlog.debug("%s", cipher)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,))
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
except Exception as e:
xlog.warn("err:%s", e)
def test(self):
for cipher in self.cipher_list:
xlog.debug("%s", cipher)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,))
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
except Exception as e:
xlog.warn("err:%s", e)
def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
import struct
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
# server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
# if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
logging.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
return ssl_sock, connct_time, handshake_time
def load_config(self):
self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10
self.connection_pool_max_num = config.CONFIG.getint("connect_manager", "https_connection_pool_max") #20/30
self.connection_pool_min_num = config.CONFIG.getint("connect_manager", "https_connection_pool_min") #20/30
self.keep_alive = config.CONFIG.getint("connect_manager", "https_keep_alive") #1
self.new_conn_pool = Connect_pool()
self.gae_conn_pool = Connect_pool()
self.host_conn_pool = {}
self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
self.max_retry = 3
self.timeout = 3
self.max_timeout = 5
self.thread_num = 0
self.max_thread_num = config.CONFIG.getint("connect_manager", "https_max_connect_thread") #10
self.connection_pool_num = config.CONFIG.getint("connect_manager", "https_connection_pool") #20/30
self.conn_pool = Connect_pool() #Queue.PriorityQueue()
self.openssl_context = SSLConnection.context_builder(ssl_version="TLSv1")
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
def __init__(self):
# http://docs.python.org/dev/library/ssl.html
# http://blog.ivanristic.com/2009/07/examples-of-the-information-collected-from-ssl-handshakes.html
# http://src.chromium.org/svn/trunk/src/net/third_party/nss/ssl/sslenum.c
# openssl s_server -accept 443 -key CA.crt -cert CA.crt
# ref: http://vincent.bernat.im/en/blog/2011-ssl-session-reuse-rfc5077.html
self.openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
try:
self.openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
except:
pass
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
self.class_name = "Https_connection_manager"
self.timeout = 4
self.max_timeout = 60
self.thread_num = 0
# after new created ssl_sock timeout(50 seconds)
# call the callback.
# This callback will put ssl to worker
self.ssl_timeout_cb = None
self.connecting_more_thread = None
self.load_config()
p = threading.Thread(target=self.keep_alive_thread)
p.daemon = True
p.start()
if self.connection_pool_min_num:
p = threading.Thread(target=self.keep_connection_daemon)
p.daemon = True
p.start()
self.create_more_connection()
def connect_ssl(ip,
port=443,
timeout=5,
openssl_context=None,
check_cert=True):
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock, ip)
ssl_sock.set_connect_state()
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
#report_network_ok
global network_stat, last_check_time, continue_fail_count
network_stat = "OK"
last_check_time = time_handshaked
continue_fail_count = 0
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components() if k == 'CN'),
'')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' %
(issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
return ssl_sock
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.connect_timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
ip_manager.ssl_closed)
ssl_sock.set_connect_state()
host = random.choice(ns)
ssl_sock.set_tlsext_host_name(host)
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#ip_manager.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('COMODO'):
ip_manager.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not COMODO' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
# ip_manager.update_ip(ip, handshake_time)
# handshake time is not the response time,
# cloudflare don't have global back-bond network like google.
# the reasonable response RTT time should be the HTTP test RTT.
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time.time()
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = self.host
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.connect_timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
ip_manager.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
current_path = os.path.dirname(os.path.abspath(__file__))
import OpenSSL
SSLError = OpenSSL.SSL.WantReadError
from config import config
import cert_util
from openssl_wrap import SSLConnection
from appids_manager import appid_manager
from proxy import xlog
g_cacertfile = os.path.join(current_path, "cacert.pem")
openssl_context = SSLConnection.context_builder(
ca_certs=g_cacertfile
) # check cacert cost too many cpu, 100 check thread cost 60%.
max_timeout = 5
default_socket = socket.socket
def load_proxy_config():
global default_socket
if config.PROXY_ENABLE:
if config.PROXY_TYPE == "HTTP":
proxy_type = socks.HTTP
elif config.PROXY_TYPE == "SOCKS4":
proxy_type = socks.SOCKS4
def check(self, callback=None, check_ca=False):
timeout = 5
openssl_context = SSLConnection.context_builder(ssl_version="TLSv1") #, ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
ssl_sock = None
try:
def connect_ssl(ip):
import struct
ip_port = (ip, 443)
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
self.result.connct_time = int((time_connected - time_begin) * 1000)
self.result.handshake_time = int((time_handshaked - time_connected) * 1000)
logging.debug("conn: %d handshake:%d", self.result.connct_time, self.result.handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
return ssl_sock
ssl_sock = connect_ssl(self.ip)
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
ssl_cert = cert_util.SSLCert(cert)
logging.info("CN:%s", ssl_cert.cn)
self.result.domain = ssl_cert.cn
if check_ca:
check_ssl_cert(ssl_sock)
if callback:
return callback(ssl_sock, self.ip)
return True
except SSLError as e:
logging.debug("Check_appengine %s SSLError:%s", self.ip, e)
except IOError as e:
logging.debug("Check %s IOError:%s", self.ip, e)
pass
except httplib.BadStatusLine:
#logging.debug('Check_appengine http.bad status line ip:%s', ip)
#import traceback
#traceback.print_exc()
pass
except Exception as e:
if len(e.args)>0:
errno_str = e.args[0]
else:
errno_str = e.message
logging.debug('check_appengine %s %s err:%s', self.ip, errno_str, e)
finally:
if ssl_sock:
ssl_sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def check(self, callback=None, check_ca=False):
timeout = 5
openssl_context = SSLConnection.context_builder(ssl_version="TLSv1", ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
ssl_sock = None
try:
def connect_ssl(ip):
import struct
ip_port = (ip, 443)
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
self.result.connct_time = int((time_connected - time_begin) * 1000)
self.result.handshake_time = int((time_handshaked - time_connected) * 1000)
logging.debug("conn: %d handshake:%d", self.result.connct_time, self.result.handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
return ssl_sock
ssl_sock = connect_ssl(self.ip)
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
ssl_cert = cert_util.SSLCert(cert)
logging.info("CN:%s", ssl_cert.cn)
self.result.domain = ssl_cert.cn
if check_ca:
check_ssl_cert(ssl_sock)
if callback:
return callback(ssl_sock, self.ip)
return True
except SSLError as e:
logging.debug("Check_appengine %s SSLError:%s", self.ip, e)
except IOError as e:
logging.debug("Check %s IOError:%s", self.ip, e)
pass
except httplib.BadStatusLine:
#logging.debug('Check_appengine http.bad status line ip:%s', ip)
#import traceback
#traceback.print_exc()
pass
except Exception as e:
if len(e.args)>0:
errno_str = e.args[0]
else:
errno_str = e.message
logging.debug('check_appengine %s %s err:%s', self.ip, errno_str, e)
finally:
if ssl_sock:
ssl_sock.close()
return False
def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock, ip)
ssl_sock.set_connect_state()
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
# report network ok
check_local_network.network_stat = "OK"
check_local_network.last_check_time = time_handshaked
check_local_network.continue_fail_count = 0
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(" certficate is none")
if check_cert:
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "")
if __name__ == "__main__":
xlog.debug("issued by:%s", issuer_commonname)
if not issuer_commonname.startswith("Google"):
raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
# xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
return ssl_sock
def get_ssl_socket(sock, server_hostname=None, context=g_context):
ssl_sock = SSLConnection(context, sock)
if server_hostname:
ssl_sock.set_tlsext_host_name(server_hostname)
return ssl_sock
from proxy_dir import current_path
import OpenSSL
SSLError = OpenSSL.SSL.WantReadError
import socks
import check_local_network
from config import config
import cert_util
from openssl_wrap import SSLConnection
from xlog import getLogger
xlog = getLogger("gae_proxy")
g_cacertfile = os.path.join(current_path, "cacert.pem")
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'):
openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
max_timeout = 5
default_socket = socket.socket
def load_proxy_config():
global default_socket
if config.PROXY_ENABLE:
if config.PROXY_TYPE == "HTTP":
proxy_type = socks.HTTP
def _create_ssl_connection(self, ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s c:%d h:%d", ip, e, connect_time, handshake_time)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ""
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(" certficate is none")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "")
if not issuer_commonname.startswith("Google"):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
return ssl_sock
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def connect_ssl(ip, port=443, timeout=5, openssl_context=None, check_cert=True):
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
#report_network_ok()
global network_ok, last_check_time
network_ok = True
last_check_time = time_handshaked
return ssl_sock
def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
return ssl_sock
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
#cert = ssl_sock.get_peer_certificate()
#if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
#issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
#if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
'The intermediate CA is mismatching.')
issuer_commonname = next(
(v for k, v in certs[0].get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
current_path = os.path.dirname(os.path.abspath(__file__))
import OpenSSL
SSLError = OpenSSL.SSL.WantReadError
from config import config
import cert_util
from openssl_wrap import SSLConnection
from appids_manager import appid_manager
from proxy import xlog
g_cacertfile = os.path.join(current_path, "cacert.pem")
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile) # check cacert cost too many cpu, 100 check thread cost 60%.
max_timeout = 5
default_socket = socket.socket
def load_proxy_config():
global default_socket
if config.PROXY_ENABLE:
if config.PROXY_TYPE == "HTTP":
proxy_type = socks.HTTP
elif config.PROXY_TYPE == "SOCKS4":
proxy_type = socks.SOCKS4
elif config.PROXY_TYPE == "SOCKS5":
def _create_ssl_connection(ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
try:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
check_ssl_cert(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s", ip, e)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
SSLError = OpenSSL.SSL.WantReadError
import socks
import check_local_network
from config import config
import cert_util
from openssl_wrap import SSLConnection
from xlog import getLogger
xlog = getLogger("gae_proxy")
import hyper
g_cacertfile = os.path.join(current_path, "cacert.pem")
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile)
openssl_context.set_session_id(binascii.b2a_hex(os.urandom(10)))
if hasattr(OpenSSL.SSL, "SESS_CACHE_BOTH"):
openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH)
max_timeout = 5
default_socket = socket.socket
def load_proxy_config():
global default_socket
if config.PROXY_ENABLE:
if config.PROXY_TYPE == "HTTP":
proxy_type = socks.HTTP