def _create_connection(ip_port, timeout, queobj):
ip = ip_port[0]
sock = None
try:
# create a ipv4/ipv6 socket object
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout or self.max_timeout)
# start connection time record
start_time = time.time()
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
google_ip.update_ip(ip, conn_time * 2000)
# put ssl socket object to output queobj
queobj.put(sock)
except (socket.error, OSError) as e:
# any socket.error, put Excpetions to output queobj.
queobj.put(e)
google_ip.report_connect_fail(ip)
if sock:
sock.close()
def process_appid_not_exist(self, appid, ip):
if check_ip.test_gae_ip(ip, "xxnet-1"):
self.set_appid_not_exist(appid)
else:
xlog.warn("process_appid_not_exist, remove ip:%s", ip)
from google_ip import google_ip
google_ip.report_connect_fail(ip, force_remove=True)
def work_loop(self):
last_ssl_active_time = self.ssl_sock.create_time
last_request_time = time.time()
while connect_control.keep_running and self.keep_running:
time_to_ping = min(0, 55 - (time.time() - last_ssl_active_time))
try:
task = self.task_queue.get(True, timeout=time_to_ping)
if not task:
# None task to exit
return
except:
if time.time() - last_request_time > self.idle_time:
self.close("idle 2 mins")
return
last_ssl_active_time = time.time()
if not self.head_request():
google_ip.report_connect_fail(self.ssl_sock.ip,
force_remove=True)
# now many gvs don't support gae
self.close("keep alive, maybe not support")
return
else:
continue
last_request_time = time.time()
self.request_task(task)
def process_appid_not_exist(self, appid, ip):
if check_ip.test_gae_ip(ip, "xxnet-1"):
self.set_appid_not_exist(appid)
else:
xlog.warn("process_appid_not_exist, remove ip:%s", ip)
from google_ip import google_ip
google_ip.report_connect_fail(ip, force_remove=True)
def work_loop(self):
last_ssl_active_time = self.ssl_sock.create_time
last_request_time = time.time()
while connect_control.keep_running and self.keep_running:
time_to_ping = min(0, 55 - (time.time() - last_ssl_active_time))
try:
task = self.task_queue.get(True, timeout=time_to_ping)
if not task:
# None task to exit
return
except:
if time.time() - last_request_time > self.idle_time:
self.close("idle 2 mins")
return
last_ssl_active_time = time.time()
if not self.head_request():
google_ip.report_connect_fail(self.ssl_sock.ip, force_remove=True)
# now many gvs don't support gae
self.close("keep alive, maybe not support")
return
else:
continue
last_request_time = time.time()
self.request_task(task)
def request_gae_proxy(method, url, headers, body):
time_request = time.time()
while True:
if time.time() - time_request > 60: #time out
return False
try:
response = fetch_by_gae(method, url, headers, body)
if response.app_status < 300:
return response
xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
if response.app_status == 506:
# fetch fail at http request
continue
server_type = response.app_headers.get('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s",
response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.worker.close("ip not support GAE")
continue
if response.app_status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid,
response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" %
response.ssl_sock.appid)
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.worker.close("ip not support GAE")
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it. %s',
response.ssl_sock.appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota")
continue
except GAE_Exception as e:
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
def _create_connection(ip_port, delay=0):
time.sleep(delay)
ip = ip_port[0]
sock = None
# start connection time record
start_time = time.time()
conn_time = 0
try:
# create a ipv4/ipv6 socket object
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
logging.debug("tcp conn %s time:%d", ip, conn_time * 1000)
if conn_time * 1000 < 400:
google_ip.report_bad_ip(ip)
logging.warn("ip:%s conn_time:%d", ip, conn_time * 1000)
sock.close()
return
google_ip.update_ip(ip, conn_time * 2000)
#logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000)
# put ssl socket object to output queobj
#sock.ip = ip
self.tcp_connection_cache.put((time.time(), sock))
except Exception as e:
conn_time = int((time.time() - start_time) * 1000)
logging.debug("tcp conn %s fail t:%d", ip, conn_time)
google_ip.report_connect_fail(ip)
#logging.info("create_tcp report fail ip:%s", ip)
if sock:
sock.close()
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def request_gae_proxy(method, url, headers, body):
time_request = time.time()
while True:
if time.time() - time_request > 60: #time out
return False
try:
response = fetch_by_gae(method, url, headers, body)
if response.app_status < 300:
return response
xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
if response.app_status == 506:
# fetch fail at http request
continue
server_type = response.app_headers.get('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.worker.close("ip not support GAE")
continue
if response.app_status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.worker.close("ip not support GAE")
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it. %s', response.ssl_sock.appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota")
continue
except GAE_Exception as e:
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def _create_connection(ip_port, delay=0):
time.sleep(delay)
ip = ip_port[0]
sock = None
# start connection time record
start_time = time.time()
conn_time = 0
try:
# create a ipv4/ipv6 socket object
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
logging.debug("tcp conn %s time:%d", ip, conn_time * 1000)
if conn_time * 1000 < 400:
google_ip.report_bad_ip(ip)
logging.warn("ip:%s conn_time:%d", ip, conn_time * 1000)
sock.close()
return
google_ip.update_ip(ip, conn_time * 2000)
#logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000)
# put ssl socket object to output queobj
#sock.ip = ip
self.tcp_connection_cache.put((time.time(), sock))
except Exception as e:
conn_time = int((time.time() - start_time) * 1000)
logging.debug("tcp conn %s fail t:%d", ip, conn_time)
google_ip.report_connect_fail(ip)
#logging.info("create_tcp report fail ip:%s", ip)
if sock:
sock.close()
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip,
port=443,
timeout=self.timeout,
check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip, port=443, timeout=self.timeout, check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def verify_SSL_certificate_issuer(ssl_sock):
#cert = ssl_sock.get_peer_certificate()
#if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
#issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
#if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('The intermediate CA is mismatching.')
issuer_commonname = next((v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
try:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
except Exception as e:
errors.append(e)
xlog.warn('gae_handler.handler %r %s , retry...', e, url)
continue
if response.app_status == 404:
#xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it. %s', response.ssl_sock.appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s", time.time()-time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
body_length = end - start + 1
last_read_time = time.time()
time_response = time.time()
while True:
if start > end:
time_finished = time.time()
if body_length > 1024 and time_finished - time_response > 0:
speed = body_length / (time_finished - time_response)
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, int(speed), response.status, url)
else:
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, response.status, url)
response.ssl_sock.received_size += body_length
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, call_time=time_request)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
google_ip.report_connect_closed(response.ssl_sock.ip, "down fail")
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
google_ip.report_connect_closed(response.ssl_sock.ip, "Net")
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
#cert = ssl_sock.get_peer_certificate()
#if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
#issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
#if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
'The intermediate CA is mismatching.')
issuer_commonname = next(
(v for k, v in certs[0].get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status,
url)
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s",
time.time() - time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
last_read_time = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
xlog.info("GAE t:%d s:%d %d %s",
(time.time() - time_request) * 1000, length,
response.status, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s status:%d", response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
response.close()
xlog.info("DIRECT chucked t:%d s:%d %d %s %s", (time.time()-time_request)*1000, length, response.status, host, url)
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
time_last_read = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
xlog.info("DIRECT t:%d s:%d %d %s %s", (time.time()-time_request)*1000, length, response.status, host, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - time_last_read > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s %s", (time.time()-time_request)*1000, length, (end-start), host, url)
return
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
else:
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
send_to_browser = False
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
if "Connection" in headers and headers["Connection"] == "close":
del headers["Connection"]
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s status:%d", response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
response.close()
return
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
def _create_ssl_connection(ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
try:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
check_ssl_cert(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s", ip, e)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ""
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(" certficate is none")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "")
if not issuer_commonname.startswith("Google"):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
if "Connection" in headers and headers["Connection"] == "close":
del headers["Connection"]
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s status:%d", response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
except Exception as e:
google_ip.report_connect_closed(response.ssl_sock.ip, "receive fail")
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
response.close()
return
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
if response.status > 400:
server_type = response.getheader('Server', "")
if "gws" not in server_type:
xlog.warn(
"IP:%s not support GAE, server type:%s status:%d",
response.ssl_sock.ip, server_type, response.status)
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
break
except OpenSSL.SSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e,
host, url)
try:
send_to_browser = True
try:
response_headers = dict(
(k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" %
(response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time() - time_request
xlog.warn(
"direct_handler.handler send response fail. t:%d e:%r %s%s",
wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s",
(time.time() - time_request) * 1000, response.status,
host, url)
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn(
"direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s",
time.time() - time_request, e, host, url)
else:
if not data:
break
response.close()
xlog.info("DIRECT chucked t:%d s:%d %d %s %s",
(time.time() - time_request) * 1000, length,
response.status, host, url)
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
time_last_read = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock, host)
xlog.info("DIRECT t:%d s:%d %d %s %s",
(time.time() - time_request) * 1000, length,
response.status, host, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - time_last_read > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s %s",
(time.time() - time_request) * 1000, length,
(end - start), host, url)
return
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn(
'direct_handler send to browser return %r %s %r',
e_b, host, url)
else:
xlog.warn(
'direct_handler send to browser return %r %s %r',
e_b, host, url)
send_to_browser = False
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.debug("fetch gae status:%s url:%s", response.app_status, url)
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 405: #Method not allowed
logging.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
logging.exception('gae_handler.handler %r %s , retry...', e, url)
if len(errors) == max_retry:
if response and response.app_status >= 500:
status = response.app_status
headers = dict(response.getheaders())
content = response.read()
else:
status = 502
headers = {'Content-Type': 'text/html'}
content = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % url, '<br>'.join(repr(x) for x in errors))
if response:
response.close()
send_response(wfile, status, headers, content.encode('utf-8'))
logging.warn("GAE %d %s %s", status, method, url)
return
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
time_start = time.time()
send_to_broswer = True
while True:
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
logging.warn('gae_handler send to browser return %r %r', e_b, url)
else:
logging.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s", (time.time()-time_request)*1000, length, response.status, url)
return
except NetWorkIOError as e:
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
logging.warn("gae_handler err:%r %s ", e, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)
def _create_ssl_connection(self, ip_port):
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
#server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
#if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
return ssl_sock
except Exception as e:
logging.debug("create_ssl %s fail:%s c:%d h:%d", ip, e, connect_time, handshake_time)
google_ip.report_connect_fail(ip)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
html = generate_message_html('504 GoAgent Proxy Time out',
u'GoAgent代理处理超时，请查看日志！')
send_response(wfile, 504, body=html.encode('utf-8'))
return
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.debug("fetch gae status:%s url:%s",
response.app_status, url)
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 405: #Method not allowed
logging.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%s %r", e, url)
except Exception as e:
errors.append(e)
logging.exception('gae_handler.handler %r %s , retry...', e, url)
if len(errors) == max_retry:
if response and response.app_status >= 500:
status = response.app_status
headers = dict(response.getheaders())
content = response.read()
else:
status = 502
headers = {'Content-Type': 'text/html'}
content = generate_message_html(
'502 URLFetch failed', 'Local URLFetch %r failed' % url,
'<br>'.join(repr(x) for x in errors))
if response:
response.close()
send_response(wfile, status, headers, content.encode('utf-8'))
logging.warn("GAE %d %s %s", status, method, url)
return
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
time_start = time.time()
send_to_broswer = True
while True:
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d",
ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
logging.warn(
'gae_handler send to browser return %r %r', e_b,
url)
else:
logging.warn(
'gae_handler send to browser return %r %r', e_b,
url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s",
(time.time() - time_request) * 1000, length,
response.status, url)
return
except NetWorkIOError as e:
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
logging.warn("gae_handler err:%r %s ", e, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)
