def _create_connection(ip_port, timeout, queobj):
ip = ip_port[0]
sock = None
try:
# create a ipv4/ipv6 socket object
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout or self.max_timeout)
# start connection time record
start_time = time.time()
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
google_ip.update_ip(ip, conn_time * 2000)
# put ssl socket object to output queobj
queobj.put(sock)
except (socket.error, OSError) as e:
# any socket.error, put Excpetions to output queobj.
queobj.put(e)
google_ip.report_connect_fail(ip)
if sock:
sock.close()
def _create_connection(ip_port, delay=0):
time.sleep(delay)
ip = ip_port[0]
sock = None
# start connection time record
start_time = time.time()
conn_time = 0
try:
# create a ipv4/ipv6 socket object
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
logging.debug("tcp conn %s time:%d", ip, conn_time * 1000)
if conn_time * 1000 < 400:
google_ip.report_bad_ip(ip)
logging.warn("ip:%s conn_time:%d", ip, conn_time * 1000)
sock.close()
return
google_ip.update_ip(ip, conn_time * 2000)
#logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000)
# put ssl socket object to output queobj
#sock.ip = ip
self.tcp_connection_cache.put((time.time(), sock))
except Exception as e:
conn_time = int((time.time() - start_time) * 1000)
logging.debug("tcp conn %s fail t:%d", ip, conn_time)
google_ip.report_connect_fail(ip)
#logging.info("create_tcp report fail ip:%s", ip)
if sock:
sock.close()
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def _create_connection(ip_port, delay=0):
time.sleep(delay)
ip = ip_port[0]
sock = None
# start connection time record
start_time = time.time()
conn_time = 0
try:
# create a ipv4/ipv6 socket object
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
logging.debug("tcp conn %s time:%d", ip, conn_time * 1000)
if conn_time * 1000 < 400:
google_ip.report_bad_ip(ip)
logging.warn("ip:%s conn_time:%d", ip, conn_time * 1000)
sock.close()
return
google_ip.update_ip(ip, conn_time * 2000)
#logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000)
# put ssl socket object to output queobj
#sock.ip = ip
self.tcp_connection_cache.put((time.time(), sock))
except Exception as e:
conn_time = int((time.time() - start_time) * 1000)
logging.debug("tcp conn %s fail t:%d", ip, conn_time)
google_ip.report_connect_fail(ip)
#logging.info("create_tcp report fail ip:%s", ip)
if sock:
sock.close()
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip,
port=443,
timeout=self.timeout,
check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip, port=443, timeout=self.timeout, check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def head_request(self):
# for keep alive
start_time = time.time()
# xlog.debug("head request %s", host)
request_data = 'GET /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host
try:
data = request_data.encode()
ret = self.ssl_sock.send(data)
if ret != len(data):
xlog.warn("h1 head send len:%r %d %s", ret, len(data), self.ip)
return False
response = simple_http_client.Response(self.ssl_sock)
response.begin(timeout=15)
status = response.status
if status != 200:
xlog.debug("%s appid:%s head fail status:%d", self.ip,
self.ssl_sock.appid, status)
return False
content = response.readall(timeout=15)
self.rtt = (time.time() - start_time) * 1000
self.ssl_sock.last_use_time = start_time
google_ip.update_ip(self.ip, self.rtt)
return True
except httplib.BadStatusLine as e:
time_now = time.time()
inactive_time = time_now - self.last_active_time
head_timeout = time_now - start_time
xlog.debug("%s keep alive fail, inactive_time:%d head_timeout:%d",
self.ssl_sock.ip, inactive_time, head_timeout)
except Exception as e:
inactive = time.time() - self.ssl_sock.last_use_time
xlog.debug(
"h1 %s appid:%s HEAD keep alive request, inactive time:%d fail:%r",
self.ssl_sock.ip, self.ssl_sock.appid, inactive, e)
def head_request(self):
# for keep alive
start_time = time.time()
# xlog.debug("head request %s", host)
request_data = 'GET /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host
try:
data = request_data.encode()
ret = self.ssl_sock.send(data)
if ret != len(data):
xlog.warn("h1 head send len:%r %d %s", ret, len(data), self.ip)
return False
response = simple_http_client.Response(self.ssl_sock)
response.begin(timeout=15)
status = response.status
if status != 200:
xlog.debug("%s appid:%s head fail status:%d", self.ip, self.ssl_sock.appid, status)
return False
content = response.readall(timeout=15)
self.rtt = (time.time() - start_time) * 1000
self.ssl_sock.last_use_time = start_time
google_ip.update_ip(self.ip, self.rtt)
return True
except httplib.BadStatusLine as e:
time_now = time.time()
inactive_time = time_now - self.last_active_time
head_timeout = time_now - start_time
xlog.debug("%s keep alive fail, inactive_time:%d head_timeout:%d",
self.ssl_sock.ip, inactive_time, head_timeout)
except Exception as e:
inactive = time.time() - self.ssl_sock.last_use_time
xlog.debug("h1 %s appid:%s HEAD keep alive request, inactive time:%d fail:%r",
self.ssl_sock.ip, self.ssl_sock.appid, inactive, e)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
#cert = ssl_sock.get_peer_certificate()
#if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
#issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
#if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
'The intermediate CA is mismatching.')
issuer_commonname = next(
(v for k, v in certs[0].get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s c:%d h:%d", ip, e, connect_time, handshake_time)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ""
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(" certficate is none")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "")
if not issuer_commonname.startswith("Google"):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
try:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
check_ssl_cert(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s", ip, e)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False