def try_sort_gws_ip(self, force=False):
if time.time() - self.last_sort_time_for_gws < 10 and not force:
return
self.ip_lock.acquire()
self.last_sort_time_for_gws = time.time()
try:
self.good_ip_num = 0
ip_rate = {}
for ip_str in self.ip_dict:
if "gws" not in self.ip_dict[ip_str]["server"]:
continue
ip_rate[ip_str] = self.ip_dict[ip_str]["handshake_time"] + (self.ip_dict[ip_str]["fail_times"] * 1000)
if self.ip_dict[ip_str]["fail_times"] == 0:
self.good_ip_num += 1
ip_time = sorted(ip_rate.items(), key=operator.itemgetter(1))
self.gws_ip_list = [ip_str for ip_str, rate in ip_time]
except Exception as e:
xlog.error("try_sort_ip_by_handshake_time:%s", e)
finally:
self.ip_lock.release()
time_cost = (time.time() - self.last_sort_time_for_gws) * 1000
if time_cost > 30:
xlog.debug("sort ip time:%dms", time_cost) # 5ms for 1000 ip. 70~150ms for 30000 ip.
self.adjust_scan_thread_num()
def try_sort_ip_by_handshake_time(self, force=False):
if time.time() - self.last_sort_time_for_gws < 10 and not force:
return
self.last_sort_time_for_gws = time.time()
self.ip_lock.acquire()
try:
ip_dict_handshake_time = {}
for ip_str in self.ip_dict:
if 'gws' not in self.ip_dict[ip_str]['server']:
continue
ip_dict_handshake_time[ip_str] = self.ip_dict[ip_str]['handshake_time']
ip_time = sorted(ip_dict_handshake_time.items(), key=operator.itemgetter(1))
self.gws_ip_list = [ip_str for ip_str,handshake_time in ip_time]
except Exception as e:
xlog.error("try_sort_ip_by_handshake_time:%s", e)
finally:
self.ip_lock.release()
time_cost = (( time.time() - self.last_sort_time_for_gws) * 1000)
if time_cost > 30:
xlog.debug("sort ip time:%dms", time_cost) # 5ms for 1000 ip. 70~150ms for 30000 ip.
self.adjust_scan_thread_num()
def save_ip_list(self, force=False):
if not force:
if self.iplist_need_save == 0:
return
if time.time() - self.iplist_saved_time < 10:
return
self.iplist_saved_time = time.time()
try:
self.ip_lock.acquire()
ip_dict = sorted(self.ip_dict.items(), key=lambda x: x[1]['handshake_time'])
with open(self.good_ip_file, "w") as fd:
for ip_str, property in ip_dict:
fd.write( "%s %s %s %d\n" % (ip_str, property['domain'], property['server'], property['handshake_time']) )
with open(self.bad_ip_file, "w") as fd:
for ip in self.bad_ip_pool:
xlog.debug("save bad ip:%s", ip)
fd.write("%s\n" % (ip))
self.iplist_need_save = 0
except Exception as e:
xlog.error("save good_ip.txt fail %s", e)
finally:
self.ip_lock.release()
def remove_ip_process(self):
try:
while True:
try:
ip_str = self.to_remove_ip_list.get_nowait()
except:
break
result = check_ip.test(ip_str)
if result and result.appspot_ok:
self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type)
xlog.debug("remove ip process, restore ip:%s", ip_str)
continue
if not check_ip.network_is_ok():
self.to_remove_ip_list.put(ip_str)
xlog.warn("network is unreachable. check your network connection.")
return
xlog.info("real remove ip:%s ", ip_str)
self.iplist_need_save = 1
finally:
self.remove_ip_thread_num_lock.acquire()
self.remove_ip_thread_num -= 1
self.remove_ip_thread_num_lock.release()
def get_host_ip(self, host):
self.try_sort_ip_by_handshake_time()
self.ip_lock.acquire()
try:
ip_num = len(self.ip_dict)
if ip_num == 0:
#logging.warning("no gws ip")
time.sleep(1)
return None
for ip_str in self.ip_dict:
domain = self.ip_dict[ip_str]["domain"]
if domain != host:
continue
get_time = self.ip_dict[ip_str]["get_time"]
if time.time() - get_time < 10:
continue
handshake_time = self.ip_dict[ip_str]["handshake_time"]
fail_time = self.ip_dict[ip_str]["fail_time"]
if time.time() - fail_time < 300:
continue
xlog.debug("get host:%s ip:%s t:%d", host, ip_str, handshake_time)
self.ip_dict[ip_str]['history'].append([time.time(), "get"])
self.ip_dict[ip_str]['get_time'] = time.time()
return ip_str
except Exception as e:
xlog.error("get_gws_ip fail:%s", e)
traceback.print_exc()
finally:
self.ip_lock.release()
def report_bad_ip(self, ip_str):
xlog.debug("report_bad_ip %s", ip_str)
if not ip_utils.check_ip_valid(ip_str):
return
self.bad_ip_pool.add(ip_str)
self.save_ip_list(force=True)
def test(ip_str, loop=1):
xlog.info("==>%s", ip_str)
check = Check_frame(ip_str, check_cert=False)
for i in range(loop):
result = check.check(callback=test_app_head)
if not result:
if "gws" in check.result.server_type:
xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type)
xlog.warn("check fail")
#continue
else:
xlog.debug("=======app check ok: %s", ip_str)
check.result.appspot_ok = result
result = check.check(callback=test_server_type, check_ca=True)
if not result:
xlog.debug("test server type fail")
continue
check.result.server_type = result
xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type,
check.result.domain, check.result.handshake_time)
return check.result
def do_POST(self):
try:
refer = self.headers.getheader('Referer')
netloc = urlparse.urlparse(refer).netloc
if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"):
xlog.warn("web control ref:%s refuse", netloc)
return
except:
pass
xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path)
try:
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
if ctype == 'multipart/form-data':
self.postvars = cgi.parse_multipart(self.rfile, pdict)
elif ctype == 'application/x-www-form-urlencoded':
length = int(self.headers.getheader('content-length'))
self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1)
else:
self.postvars = {}
except:
self.postvars = {}
path = urlparse.urlparse(self.path).path
if path == '/deploy':
return self.req_deploy_handler()
elif path == "/config":
return self.req_config_handler()
elif path == "/scan_ip":
return self.req_scan_ip_handler()
elif path.startswith("/importip"):
return self.req_importip_handler()
else:
self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')
xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)
def test2(self):
work_ciphers = ["AES128-SHA"]
for cipher in self.cipher_list:
if cipher in work_ciphers:
continue
else:
work_ciphers.append(cipher)
xlog.debug("%s", cipher)
cipher_suites = (work_ciphers)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
if "gws" not in server_type:
work_ciphers.remove(cipher)
except Exception as e:
xlog.warn("err:%s", e)
try:
work_ciphers.remove(cipher)
except:
pass
work_str = ""
for cipher in work_ciphers:
work_str += cipher + ":"
xlog.info("work ciphers:%s", work_str)
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def create_connection_worker(self):
try:
while connect_control.keep_running:
if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
#xlog.debug("get enough conn")
break
if self.thread_num > self.connection_pool_min_num - self.new_conn_pool.qsize():
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no gws ip")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
connect_control.start_connect_register(True)
ssl_sock = self._create_ssl_connection( (ip_str, port) )
connect_control.end_connect_register(True)
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
xlog.debug("create_connection_worker, control not allow")
time.sleep(10)
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def report_connect_fail(self, ip_str, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip_str in self.ip_dict:
return
self.ip_dict[ip_str]["links"] -= 1
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
# connect_control.fall_into_honeypot()
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time_now - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
self.ip_dict[ip_str]["handshake_time"] += 300
if self.ip_dict[ip_str]["fail_times"] == 0:
self.good_ip_num -= 1
self.ip_dict[ip_str]["fail_times"] += 1
self.append_ip_history(ip_str, "fail")
self.ip_dict[ip_str]["fail_time"] = time_now
if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50:
property = self.ip_dict[ip_str]
server = property["server"]
del self.ip_dict[ip_str]
if "gws" in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
xlog.info(
"remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
else:
xlog.info(
"remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)
)
if self.good_ip_num > len(self.ip_dict):
self.good_ip_num = len(self.ip_dict)
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug("create_ssl_connection fail")
continue
if ssl_sock.host == "":
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
raise GAE_Exception(1, "no appid can use")
headers["Host"] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers["Host"]
else:
headers["Host"] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.warn("request failed:%s", e)
if ssl_sock:
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def get_gws_ip(self):
self.try_sort_gws_ip()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
if ip_num == 0:
#logging.warning("no gws ip")
time.sleep(10)
return None
for i in range(ip_num):
if self.gws_ip_pointer >= ip_num:
if time.time() - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time.time()
elif self.gws_ip_pointer > 0 and time.time() - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time.time()
ip_str = self.gws_ip_list[self.gws_ip_pointer]
get_time = self.ip_dict[ip_str]["get_time"]
if time.time() - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
if time.time() - self.ip_dict[ip_str]['success_time'] > 300: # 5 min
fail_connect_interval = 1800 # 30 min
else:
fail_connect_interval = 120 # 2 min
fail_time = self.ip_dict[ip_str]["fail_time"]
if time.time() - fail_time < fail_connect_interval:
self.gws_ip_pointer += 1
continue
time_now = time.time()
active_time = self.ip_dict[ip_str]['data_active']
transfered_data = self.ip_dict[ip_str]['transfered_data'] - ((time_now - active_time) * config.ip_traffic_quota)
if transfered_data > config.ip_traffic_quota_base:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip_str]["handshake_time"]
xlog.debug("get ip:%s t:%d", ip_str, handshake_time)
self.ip_dict[ip_str]['history'].append([time.time(), "get"])
self.ip_dict[ip_str]['get_time'] = time.time()
self.gws_ip_pointer += 1
return ip_str
except Exception as e:
xlog.error("get_gws_ip fail:%s", e)
traceback.print_exc()
finally:
self.ip_lock.release()
def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
# change top domain to xx-net.net
# this domain bypass the cloudflare front for ipv4
#p = host.find(".")
#host_sub = host[:p]
#host = host_sub + ".xx-net.net"
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}
heroku_host = ""
content, status, response = self._request(
"POST", heroku_host, "/2/",
request_headers, request_body, timeout)
# xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# xlog.debug("status:%d", status)
if status == 200:
xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace())
self.last_success_time = time.time()
self.continue_fail_num = 0
self.success_num += 1
else:
if status == 404:
heroku_host = response.ssl_sock.host
xlog.warn("heroku:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
self.last_fail_time = time.time()
self.continue_fail_num += 1
self.fail_num += 1
try:
res = simple_http_client.TxtResponse(content)
except:
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def main():
global ready
connect_control.keep_running = True
config.load()
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
xlog.basicConfig(level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]')
pre_start()
log_info()
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True # checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False # checked by launcher.module_init
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
if do_profile:
pr.disable()
pr.print_stats()
def test(self):
for cipher in self.cipher_list:
xlog.debug("%s", cipher)
openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,))
try:
ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
server_type = test_server_type(ssl, self.ip)
xlog.debug("%s", server_type)
except Exception as e:
xlog.warn("err:%s", e)
def report_connect_closed(self, ip_str, reason=""):
xlog.debug("%s close:%s", ip_str, reason)
self.ip_lock.acquire()
try:
if ip_str in self.ip_dict:
self.ip_dict[ip_str]["links"] -= 1
self.append_ip_history(ip_str, "C[%s]" % reason)
except Exception as e:
xlog.error("report_connect_closed err:%s", e)
finally:
self.ip_lock.release()
def check(self, callback=None, check_ca=True, close_ssl=True):
ssl_sock = None
try:
ssl_sock,self.result.connct_time,self.result.handshake_time = connect_ssl(self.ip, timeout=self.timeout, openssl_context=self.openssl_context)
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#raise HoneypotError(' certficate is none')
raise SSLError("no cert")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if self.check_cert and not issuer_commonname.startswith('Google'):
raise HoneypotError(' certficate is issued by %r, not Google' % ( issuer_commonname))
ssl_cert = cert_util.SSLCert(cert)
xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
self.result.domain = ssl_cert.cn
if check_ca:
check_ssl_cert(ssl_sock)
if callback:
return callback(ssl_sock, self.ip)
return True
except HoneypotError as e:
xlog.warn("honeypot %s", self.ip)
raise e
except SSLError as e:
xlog.debug("Check_appengine %s SSLError:%s", self.ip, e)
pass
except IOError as e:
xlog.warn("Check %s IOError:%s", self.ip, e)
pass
except httplib.BadStatusLine:
#logging.debug('Check_appengine http.bad status line ip:%s', ip)
#import traceback
#traceback.print_exc()
pass
except Exception as e:
if len(e.args)>0:
errno_str = e.args[0]
else:
errno_str = e.message
xlog.exception('check_appengine %s %s err:%s', self.ip, errno_str, e)
finally:
if ssl_sock and close_ssl:
ssl_sock.close()
return False
def _create_connection(ip_port, delay=0):
connect_control.start_connect_register(True)
time.sleep(delay)
ip = ip_port[0]
sock = None
# start connection time record
start_time = time.time()
conn_time = 0
try:
# create a ipv4/ipv6 socket object
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ":" not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ":" not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
# TCP connect
sock.connect(ip_port)
# record TCP connection time
conn_time = time.time() - start_time
xlog.debug("tcp conn %s time:%d", ip, conn_time * 1000)
google_ip.update_ip(ip, conn_time * 2000)
# logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000)
# put ssl socket object to output queobj
# sock.ip = ip
self.tcp_connection_cache.put((time.time(), sock))
except Exception as e:
conn_time = int((time.time() - start_time) * 1000)
xlog.debug("tcp conn %s fail t:%d", ip, conn_time)
google_ip.report_connect_fail(ip)
# logging.info("create_tcp report fail ip:%s", ip)
if sock:
sock.close()
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
connect_control.end_connect_register(True)
def get_gws_ip(self):
self.try_sort_ip_by_handshake_time()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
for i in range(ip_num):
if ip_num == 0:
#logging.warning("no gws ip")
time.sleep(1)
return None
if self.gws_ip_pointer >= ip_num:
if time.time() - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time.time()
elif self.gws_ip_pointer > 0 and time.time() - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time.time()
ip_str = self.gws_ip_list[self.gws_ip_pointer]
if self.is_bad_ip(ip_str):
self.gws_ip_pointer += 1
continue
get_time = self.ip_dict[ip_str]["get_time"]
if time.time() - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip_str]["handshake_time"]
fail_time = self.ip_dict[ip_str]["fail_time"]
if time.time() - fail_time < 300:
self.gws_ip_pointer += 1
continue
xlog.debug("get ip:%s t:%d", ip_str, handshake_time)
self.ip_dict[ip_str]['history'].append([time.time(), "get"])
self.ip_dict[ip_str]['get_time'] = time.time()
self.gws_ip_pointer += 1
return ip_str
except Exception as e:
xlog.error("get_gws_ip fail:%s", e)
traceback.print_exc()
finally:
self.ip_lock.release()
def head_request(self, ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
xlog.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
# public appid don't keep alive, for quota limit.
if ssl_sock.appid.startswith("xxnet-") and ssl_sock.appid[7:].isdigit():
#logging.info("public appid don't keep alive")
#self.keep_alive = 0
return False
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
response = None
try:
ssl_sock.settimeout(3)
ssl_sock.sock.settimeout(3)
data = request_data.encode()
ret = ssl_sock.send(data)
if ret != len(data):
xlog.warn("head send len:%d %d", ret, len(data))
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
xlog.debug("app head fail status:%d", status)
raise Exception("app check fail")
return True
except httplib.BadStatusLine as e:
return False
except Exception as e:
xlog.debug("head request fail:%r", e)
return False
finally:
if response:
response.close()
def report_connect_fail(self, ip_str, force_remove=False):
# ignore if system network is disconnected.
if not force_remove:
if not check_ip.network_is_ok():
xlog.debug("report_connect_fail network fail")
return
self.ip_lock.acquire()
try:
if not ip_str in self.ip_dict:
return
fail_time = self.ip_dict[ip_str]["fail_time"]
if not force_remove and time.time() - fail_time < 1:
xlog.debug("fail time too near")
return
# increase handshake_time to make it can be used in lower probability
self.ip_dict[ip_str]['handshake_time'] += 300
self.ip_dict[ip_str]['timeout'] += 1
self.ip_dict[ip_str]['history'].append([time.time(), "fail"])
self.ip_dict[ip_str]["fail_time"] = time.time()
if force_remove or self.ip_dict[ip_str]['timeout'] >= 5:
property = self.ip_dict[ip_str]
server = property['server']
del self.ip_dict[ip_str]
if 'gws' in server and ip_str in self.gws_ip_list:
self.gws_ip_list.remove(ip_str)
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list))
if not force_remove:
self.to_remove_ip_list.put(ip_str)
self.try_remove_thread()
self.iplist_need_save = 1
except Exception as e:
xlog.exception("set_ip err:%s", e)
finally:
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def connect_ssl(ip, port=443, timeout=5, openssl_context=None):
import struct
ip_port = (ip, port)
if not openssl_context:
openssl_context = SSLConnection.context_builder()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET)
else:
sock = socket.socket(socket.AF_INET)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(timeout)
ssl_sock = SSLConnection(openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
# server_hostname = random_hostname() if (cache_key or '').startswith('google_') or hostname.endswith('.appspot.com') else None
# if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
# ssl_sock.set_tlsext_host_name(server_hostname)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.sock = sock
return ssl_sock, connct_time, handshake_time
def checker(self):
while True:
try:
ip = self.get_ip()
except Exception as e:
xlog.info("no ip left")
return
try:
res = self.check_ip.check_ip(ip, sni=host, host=host)
except Exception as e:
xlog.warn("check fail:%s except:%r", e)
continue
if not res or not res.ok:
xlog.debug("check fail:%s fail", ip)
continue
self.write_ip(ip, res.domain, res.handshake_time)
def test_app_head(ssl_sock, ip):
appid = appid_manager.get_appid()
request_data = 'GET / HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
content = response.read()
if "GoAgent" not in content:
xlog.debug("app check %s content:%s", ip, content)
raise Exception("content fail")
except Exception as e:
xlog.exception("test_app_head except:%r", e)
return False
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start)*1000
xlog.debug("app check time:%d", time_cost)
return True
def do_CONNECT_FWD(self):
"""socket forward for http CONNECT command"""
host, _, port = self.path.rpartition(':')
port = int(port)
xlog.info('FWD %s %s:%d ', self.command, host, port)
if host == "appengine.google.com" or host == "www.google.com":
connected_in_s = 5 # gae_proxy upload to appengine is slow, it need more 'fresh' connection.
else:
connected_in_s = 10 # gws connect can be used after tcp connection created 15 s
try:
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
data = self.connection.recv(1024)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port, e)
self.connection.close()
return
remote = forwork_manager.create_connection(host=host, port=port, sock_life=connected_in_s)
if remote is None:
self.connection.close()
xlog.warn('FWD %s %s:%d create_connection fail', self.command, host, port)
return
try:
if data:
remote.send(data)
except Exception as e:
xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port, e)
self.connection.close()
remote.close()
return
# reset timeout default to avoid long http upload failure, but it will delay timeout retry :(
remote.settimeout(None)
forwork_manager.forward_socket(self.connection, remote, bufsize=self.bufsize)
xlog.debug('FWD %s %s:%d with closed', self.command, host, port)
def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192):
try:
timecount = timeout
while 1:
timecount -= tick
if timecount <= 0:
break
(ins, _, errors) = select.select([local, remote], [], [local, remote], tick)
if errors:
break
if not ins:
continue
for sock in ins:
data = sock.recv(bufsize)
if not data:
if sock is remote:
xlog.debug("forward remote disconnected.")
else:
xlog.debug("forward local disconnected.")
return
if sock is remote:
local.sendall(data)
timecount = timeout
else:
remote.sendall(data)
timecount = timeout
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE):
xlog.exception("forward except:%s.", e)
finally:
if local:
local.close()
if remote:
remote.close()
def get_ssl_connection(self, host=""):
ssl_sock = None
if host:
if host in self.host_conn_pool:
while True:
ret = self.host_conn_pool[host].get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if (
time.time() - ssl_sock.last_use_time < self.keep_alive
): # gws ssl connection can keep for 60s after created
xlog.debug("host_conn_pool %s get:%s handshake:%d", host, ssl_sock.ip, handshake_time)
break
else:
ssl_sock.close()
continue
else:
while True:
ret = self.gae_conn_pool.get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if (
time.time() - ssl_sock.last_use_time < self.keep_alive
): # gws ssl connection can keep for 60s after created
xlog.debug("ssl_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
break
else:
ssl_sock.close()
continue
if host:
self.create_more_connection(type="host")
else:
self.create_more_connection(type="gae")
if ssl_sock:
return ssl_sock
else:
# xlog.debug("get_ssl wait")
ret = self.new_conn_pool.get(True, self.max_timeout)
if ret:
# xlog.debug("get_ssl ok")
handshake_time, ssl_sock = ret
return ssl_sock
else:
xlog.debug("create ssl timeout fail.")
return None
def head_request(self, ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
xlog.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
# public appid don't keep alive, for quota limit.
if ssl_sock.appid.startswith("xxnet-") and ssl_sock.appid[7:].isdigit():
#logging.info("public appid don't keep alive")
#self.keep_alive = 0
return False
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
response = None
try:
ssl_sock.settimeout(3)
ssl_sock.sock.settimeout(3)
data = request_data.encode()
ret = ssl_sock.send(data)
if ret != len(data):
xlog.warn("head send len:%d %d", ret, len(data))
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
xlog.debug("app head fail status:%d", status)
raise Exception("app check fail")
return True
except httplib.BadStatusLine as e:
xlog.debug("head request BadStatusLine fail:%r", e)
return False
except Exception as e:
xlog.debug("head request fail:%r", e)
return False
finally:
if response:
response.close()
def get_ssl_connection(self, host=''):
ssl_sock = None
if host:
if host in self.host_conn_pool:
while True:
ret = self.host_conn_pool[host].get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if time.time() - ssl_sock.last_use_time < self.keep_alive+1:
xlog.debug("host_conn_pool %s get:%s handshake:%d", host, ssl_sock.ip, handshake_time)
break
else:
google_ip.report_connect_closed(ssl_sock.ip, "get_timeout")
ssl_sock.close()
continue
else:
while True:
ret = self.gae_conn_pool.get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if time.time() - ssl_sock.last_use_time < self.keep_alive+1:
xlog.debug("ssl_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
break
else:
google_ip.report_connect_closed(ssl_sock.ip, "get_timeout")
ssl_sock.close()
continue
self.create_more_connection()
if ssl_sock:
return ssl_sock
else:
ret = self.new_conn_pool.get(True, self.max_timeout)
if ret:
handshake_time, ssl_sock = ret
return ssl_sock
else:
xlog.debug("create ssl timeout fail.")
return None
def network_is_ok():
global checking_lock, checking_num, network_ok, last_check_time, check_network_interval
if time.time() - last_check_time < check_network_interval:
return network_ok
if checking_num > 0:
return network_ok
if config.PROXY_ENABLE:
socket.socket = socks.socksocket
xlog.debug("patch socks")
checking_lock.acquire()
checking_num += 1
checking_lock.release()
try:
conn = httplib.HTTPSConnection("code.jquery.com", 443, timeout=30)
header = {
"user-agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, sdch",
"accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
"connection": "keep-alive"
}
conn.request("HEAD", "/", headers=header)
response = conn.getresponse()
if response.status:
xlog.debug("network is ok")
network_ok = True
last_check_time = time.time()
return True
except:
pass
finally:
checking_lock.acquire()
checking_num -= 1
checking_lock.release()
if config.PROXY_ENABLE:
socket.socket = default_socket
xlog.debug("restore socket")
xlog.warn("network fail.")
network_ok = False
last_check_time = time.time()
return False
def create_ssl_connection(self, host=''):
ssl_sock = None
if host:
if host in self.host_conn_pool:
while True:
ret = self.host_conn_pool[host].get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if time.time() - ssl_sock.last_use_time < self.keep_alive+1: # gws ssl connection can keep for 230s after created
xlog.debug("host_conn_pool %s get:%s handshake:%d", host, ssl_sock.ip, handshake_time)
return ssl_sock
break
else:
ssl_sock.close()
continue
else:
while True:
ret = self.gae_conn_pool.get_nowait()
if ret:
handshake_time, ssl_sock = ret
else:
ssl_sock = None
break
if time.time() - ssl_sock.last_use_time < self.keep_alive+1: # gws ssl connection can keep for 230s after created
xlog.debug("ssl_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time)
break
else:
ssl_sock.close()
continue
self.create_more_connection()
if ssl_sock:
return ssl_sock
else:
ret = self.new_conn_pool.get(True, self.max_timeout)
if ret:
handshake_time, ssl_sock = ret
return ssl_sock
else:
xlog.debug("create ssl timeout fail.")
return None
def test_app_check(ssl_sock, ip):
request_data = 'GET /check HTTP/1.1\r\nHost: xxnet-check.appspot.com\r\n\r\n'
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
content = response.read()
if not content == "CHECK_OK":
xlog.debug("app check %s content:%s", ip, content)
raise Exception("content fail")
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start) * 1000
xlog.debug("app check time:%d", time_cost)
return True
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status,
url)
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s",
time.time() - time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
last_read_time = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
xlog.info("GAE t:%d s:%d %d %s",
(time.time() - time_request) * 1000, length,
response.status, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def fetch(self):
response_headers = dict(
(k.title(), v) for k, v in self.response.getheaders())
content_range = response_headers['Content-Range']
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
if start == 0:
response_headers['Content-Length'] = str(length)
del response_headers['Content-Range']
else:
response_headers['Content-Range'] = 'bytes %s-%s/%s' % (start, end,
length)
response_headers['Content-Length'] = str(length - start)
xlog.info('>>>>>>>>>>>>>>> RangeFetch started(%r) %d-%d', self.url,
start, end)
try:
self.wfile.write("HTTP/1.1 200 OK\r\n")
for key in response_headers:
if key == 'Transfer-Encoding':
continue
if key == 'X-Head-Content-Length':
continue
if key in skip_headers:
continue
value = response_headers[key]
#logging.debug("Head %s: %s", key.title(), value)
send_header(self.wfile, key, value)
self.wfile.write("\r\n")
except Exception as e:
self._stopped = True
xlog.warn("RangeFetch send response fail:%r %s", e, self.url)
return
data_queue = Queue.PriorityQueue()
range_queue = Queue.PriorityQueue()
range_queue.put((start, end, self.response))
self.expect_begin = start
for begin in range(end + 1, length, self.maxsize):
range_queue.put((begin, min(begin + self.maxsize - 1,
length - 1), None))
for i in xrange(0, self.threads):
range_delay_size = i * self.maxsize
spawn_later(
float(range_delay_size) / self.waitsize, self.__fetchlet,
range_queue, data_queue, range_delay_size)
has_peek = hasattr(data_queue, 'peek')
peek_timeout = 120
while self.expect_begin < length - 1:
try:
if has_peek:
begin, data = data_queue.peek(timeout=peek_timeout)
if self.expect_begin == begin:
data_queue.get()
elif self.expect_begin < begin:
time.sleep(0.1)
continue
else:
xlog.error(
'RangeFetch Error: begin(%r) < expect_begin(%r), quit.',
begin, self.expect_begin)
break
else:
begin, data = data_queue.get(timeout=peek_timeout)
if self.expect_begin == begin:
pass
elif self.expect_begin < begin:
data_queue.put((begin, data))
time.sleep(0.1)
continue
else:
xlog.error(
'RangeFetch Error: begin(%r) < expect_begin(%r), quit.',
begin, self.expect_begin)
break
except Queue.Empty:
xlog.error('data_queue peek timeout, break')
break
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d, retry",
ret)
ret = self.wfile.write(data)
xlog.debug("send to browser wfile.write ret:%d", ret)
self.expect_begin += len(data)
del data
except Exception as e:
xlog.warn('RangeFetch client closed(%s). %s', e, self.url)
break
self._stopped = True
xlog.warn("read timeout t:%d len:%d left:%d %s %s",
(time.time() - time_request) * 1000, length,
(end - start), host, url)
return
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn(
'direct_handler send to browser return %r %s %r',
e_b, host, url)
else:
xlog.warn(
'direct_handler send to browser return %r %s %r',
e_b, host, url)
send_to_browser = False
except NetWorkIOError as e:
def __init__(self, type, message):
xlog.debug("GAE_Exception %r %r", type, message)
self.type = type
self.message = message
def main():
global ready
connect_control.keep_running = True
config.load()
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s",
connect_control.keep_running)
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
xlog.basicConfig(
level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO,
format='%(levelname)s - %(asctime)s %(message)s',
datefmt='[%b %d %H:%M:%S]')
pre_start()
log_info()
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer(
(config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer(
(config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True # checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False # checked by launcher.module_init
xlog.debug("## GAEProxy set keep_running: %s",
connect_control.keep_running)
if do_profile:
pr.disable()
pr.print_stats()
def do_GET(self):
try:
refer = self.headers.getheader('Referer')
netloc = urlparse.urlparse(refer).netloc
if not netloc.startswith("127.0.0.1") and not netloc.startswitch(
"localhost"):
xlog.warn("web control ref:%s refuse", netloc)
return
except:
pass
path = urlparse.urlparse(self.path).path
if path == "/log":
return self.req_log_handler()
elif path == "/status":
return self.req_status_handler()
else:
xlog.debug('GAEProxy Web_control %s %s %s ', self.address_string(),
self.command, self.path)
if path == '/deploy':
return self.req_deploy_handler()
elif path == "/config":
return self.req_config_handler()
elif path == "/ip_list":
return self.req_ip_list_handler()
elif path == "/scan_ip":
return self.req_scan_ip_handler()
elif path == "/ssl_pool":
return self.req_ssl_pool_handler()
elif path == "/download_cert":
return self.req_download_cert_handler()
elif path == "/is_ready":
return self.req_is_ready_handler()
elif path == "/test_ip":
return self.req_test_ip_handler()
elif path == "/quit":
connect_control.keep_running = False
data = "Quit"
self.wfile.write((
'HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n'
% ('text/plain', len(data))).encode())
self.wfile.write(data)
#sys.exit(0)
#quit()
#os._exit(0)
return
elif path.startswith("/wizard/"):
file_path = os.path.abspath(
os.path.join(web_ui_path, '/'.join(path.split('/')[1:])))
if not os.path.isfile(file_path):
self.wfile.write(b'HTTP/1.1 404 Not Found\r\n\r\n')
xlog.warn('%s %s %s wizard file %s not found',
self.address_string(), self.command, self.path,
file_path)
return
if file_path.endswith('.html'):
mimetype = 'text/html'
elif file_path.endswith('.png'):
mimetype = 'image/png'
elif file_path.endswith('.jpg') or file_path.endswith('.jpeg'):
mimetype = 'image/jpeg'
else:
mimetype = 'application/octet-stream'
self.send_file(file_path, mimetype)
return
else:
xlog.warn('Control Req %s %s %s ', self.address_string(),
self.command, self.path)
# check for '..', which will leak file
if re.search(r'(\.{2})', self.path) is not None:
self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
xlog.warn('%s %s %s haking', self.address_string(), self.command,
self.path)
return
filename = os.path.normpath('./' + path)
if self.path.startswith(('http://', 'https://')):
data = b'HTTP/1.1 200\r\nCache-Control: max-age=86400\r\nExpires:Oct, 01 Aug 2100 00:00:00 GMT\r\nConnection: close\r\n'
data += b'\r\n'
self.wfile.write(data)
xlog.info('%s "%s %s HTTP/1.1" 200 -', self.address_string(),
self.command, self.path)
elif os.path.isfile(filename):
if filename.endswith('.pac'):
mimetype = 'text/plain'
else:
mimetype = 'application/octet-stream'
#self.send_file(filename, mimetype)
else:
self.wfile.write(
b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found'
)
xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(),
self.command, self.path)
def request(self,
method,
host,
schema="http",
path="/",
headers={},
data="",
timeout=40):
# change top domain to xx-net.net
# this domain bypass the cloudflare front for ipv4
p = host.find(".")
host_sub = host[:p]
host = host_sub + ".xx-net.net"
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {
'Content-Length': len(data),
'Content-Type': 'application/octet-stream'
}
heroku_host = ""
content, status, response = self._request("POST", heroku_host, "/2/",
request_headers,
request_body, timeout)
# xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# xlog.debug("status:%d", status)
if status == 200:
xlog.debug("%s %s%s trace:%s", method, host, path,
response.task.get_trace())
self.last_success_time = time.time()
self.continue_fail_num = 0
self.success_num += 1
else:
if status == 404:
heroku_host = response.ssl_sock.host
xlog.warn("heroku:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
self.last_fail_time = time.time()
self.continue_fail_num += 1
self.fail_num += 1
try:
res = simple_http_client.TxtResponse(content)
except:
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def terminate():
xlog.info("start to terminate GAE_Proxy")
connect_control.keep_running = False
xlog.debug("## Set keep_running: %s", connect_control.keep_running)
def handler(method, host, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30:
return return_fail_message(wfile)
try:
response = fetch(method, host, url, headers, body)
if response:
break
except OpenSSL.SysCallError as e:
errors.append(e)
xlog.warn("direct_handler.handler err:%r %s/%s", e, host, url)
except Exception as e:
errors.append(e)
xlog.exception('direct_handler.handler %r %s %s , retry...', e, host, url)
try:
send_to_browser = True
try:
response_headers = dict((k.title(), v) for k, v in response.getheaders())
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response.getheaders():
send_header(wfile, key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
wait_time = time.time()-time_request
xlog.warn("direct_handler.handler send response fail. t:%d e:%r %s%s", wait_time, e, host, url)
if method == 'HEAD' or response.status in (204, 304):
xlog.info("DIRECT t:%d %d %s %s", (time.time()-time_request)*1000, response.status, host, url)
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
response.close()
return
if 'Transfer-Encoding' in response_headers:
length = 0
while True:
try:
data = response.read(8192)
except httplib.IncompleteRead, e:
data = e.partial
if send_to_browser:
try:
if not data:
wfile.write('0\r\n\r\n')
break
length += len(data)
wfile.write('%x\r\n' % len(data))
wfile.write(data)
wfile.write('\r\n')
except Exception as e:
send_to_browser = False
xlog.warn("direct_handler.handler send Transfer-Encoding t:%d e:%r %s/%s", time.time()-time_request, e, host, url)
else:
if not data:
break
response.close()
xlog.info("DIRECT chucked t:%d s:%d %d %s %s", (time.time()-time_request)*1000, length, response.status, host, url)
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
time_last_read = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, host)
xlog.info("DIRECT t:%d s:%d %d %s %s", (time.time()-time_request)*1000, length, response.status, host, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - time_last_read > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s %s", (time.time()-time_request)*1000, length, (end-start), host, url)
return
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
else:
xlog.warn('direct_handler send to browser return %r %s %r', e_b, host, url)
send_to_browser = False
def do_CONNECT_DIRECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
if len(self.parsed_url[4]):
path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
path = self.parsed_url[2]
request_headers = dict(
(k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
#logging.debug("payload_len:%d %s %s", payload_len, self.command, self.path)
payload = self.rfile.read(payload_len)
except NetWorkIOError as e:
xlog.error('handle_method_urlfetch read payload failed:%s',
e)
return
direct_handler.handler(self.command, host, path, request_headers,
payload, self.wfile)
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
xlog.warn("read request line len:%d",
len(self.raw_requestline))
return
if not self.raw_requestline:
xlog.warn("read request line empty")
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(
config.HOSTS_FWD_ENDSWITH) or host.endswith(
config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_FWD:
fwd_set = [s for s in config.HOSTS_FWD]
fwd_set.append(host)
config.HOSTS_FWD = tuple(fwd_set)
xlog.warn(
"Method %s not support in GAE, Redirect to FWD for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
self.path).encode())
else:
xlog.warn("Method %s not support in GAEProxy for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def get_gws_ip(self):
self.try_sort_gws_ip()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
if ip_num == 0:
#logging.warning("no gws ip")
time.sleep(10)
return None
for i in range(ip_num):
time_now = time.time()
if self.gws_ip_pointer >= ip_num:
if time_now - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
elif self.gws_ip_pointer > 0 and time_now - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
ip_str = self.gws_ip_list[self.gws_ip_pointer]
get_time = self.ip_dict[ip_str]["get_time"]
if time_now - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
if time_now - self.ip_dict[ip_str][
'success_time'] > 300: # 5 min
fail_connect_interval = 1800 # 30 min
else:
fail_connect_interval = 120 # 2 min
fail_time = self.ip_dict[ip_str]["fail_time"]
if time_now - fail_time < fail_connect_interval:
self.gws_ip_pointer += 1
continue
if self.trafic_control: # not check now
active_time = self.ip_dict[ip_str]['data_active']
transfered_data = self.ip_dict[ip_str][
'transfered_data'] - (
(time_now - active_time) * config.ip_traffic_quota)
if transfered_data > config.ip_traffic_quota_base:
self.gws_ip_pointer += 1
continue
if self.ip_dict[ip_str]['links'] >= config.max_links_per_ip:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip_str]["handshake_time"]
xlog.debug("get ip:%s t:%d", ip_str, handshake_time)
self.append_ip_history(ip_str, "get")
self.ip_dict[ip_str]['get_time'] = time_now
self.ip_dict[ip_str]['links'] += 1
self.gws_ip_pointer += 1
return ip_str
except Exception as e:
xlog.error("get_gws_ip fail:%s", e)
traceback.print_exc()
finally:
self.ip_lock.release()
def check(self, callback=None, check_ca=True, close_ssl=True):
ssl_sock = None
try:
ssl_sock, self.result.connct_time, self.result.handshake_time = connect_ssl(
self.ip,
timeout=self.timeout,
openssl_context=self.openssl_context)
# verify SSL certificate issuer.
def check_ssl_cert(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#raise HoneypotError(' certficate is none')
raise SSLError("no cert")
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if self.check_cert and not issuer_commonname.startswith(
'Google'):
raise HoneypotError(
' certficate is issued by %r, not Google' %
(issuer_commonname))
ssl_cert = cert_util.SSLCert(cert)
xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
self.result.domain = ssl_cert.cn
if check_ca:
check_ssl_cert(ssl_sock)
if callback:
return callback(ssl_sock, self.ip)
return True
except HoneypotError as e:
xlog.warn("honeypot %s", self.ip)
raise e
except SSLError as e:
xlog.debug("Check_appengine %s SSLError:%s", self.ip, e)
pass
except IOError as e:
xlog.warn("Check %s IOError:%s", self.ip, e)
pass
except httplib.BadStatusLine:
#logging.debug('Check_appengine http.bad status line ip:%s', ip)
#import traceback
#traceback.print_exc()
pass
except Exception as e:
if len(e.args) > 0:
errno_str = e.args[0]
else:
errno_str = e.message
xlog.exception('check_appengine %s %s err:%s', self.ip, errno_str,
e)
finally:
if ssl_sock and close_ssl:
ssl_sock.close()
return False