def do_CONNECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=CertUtil.cert_keyfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.parse_request()
self.do_METHOD()
def do_CONNECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port not in (80, 443):
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
self.wfile.write(b'HTTP/1.1 200 Connection Established\r\n\r\n')
#self.conntunnel = True
leadbyte = self.connection.recv(1, socket.MSG_PEEK)
if leadbyte in ('\x80', '\x16'):
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=CertUtil.cert_keyfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.close_connection = 0
def main():
pre_start()
connect_control.keep_running = True
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
try:
urllib2.urlopen('http://127.0.0.1:%d/%s' % (config.PAC_PORT, config.PAC_FILE))
except:
pass
while connect_control.keep_running:
time.sleep(1)
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
def main(args):
global ready, proxy_server
check_create_data_path()
log_info()
CertUtil.init_ca()
allow_remote = args.get("allow_remote", 0)
listen_ips = front.config.listen_ip
if isinstance(listen_ips, basestring):
listen_ips = [listen_ips]
else:
listen_ips = list(listen_ips)
if allow_remote and ("0.0.0.0" not in listen_ips or "::" not in listen_ips):
listen_ips.append("0.0.0.0")
addresses = [(listen_ip, front.config.listen_port) for listen_ip in listen_ips]
front.start()
direct_front.start()
proxy_server = simple_http_server.HTTPServer(
addresses, proxy_handler.GAEProxyHandler, logger=xlog)
ready = True # checked by launcher.module_init
proxy_server.serve_forever()
def main():
global ready
connect_control.keep_running = True
config.load()
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = config.do_profile
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
#xlog.basicConfig(level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]')
pre_start()
log_info()
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True # checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False # checked by launcher.module_init
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
if do_profile:
pr.disable()
pr.print_stats()
def main():
global ready
# to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, "readlink", lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
logging.basicConfig(
level=logging.DEBUG if config.LISTEN_DEBUGINFO else logging.INFO,
format="%(levelname)s - %(asctime)s %(message)s",
datefmt="[%b %d %H:%M:%S]",
)
pre_start()
logging.info(config.info())
CertUtil.init_ca()
proxy_daemon = LocalProxyServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = LocalProxyServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
ready = True
while config.keep_run:
time.sleep(1)
logging.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False
logging.info("Finished Exiting gae_proxy module...")
if do_profile:
pr.disable()
pr.print_stats()
def do_CONNECT_AGENT(self):
"""send fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
# xlog.info('https GAE %s %s:%d ', self.command, host, port)
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=CertUtil.cert_keyfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.parse_request()
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
if self.path == "https://%s/xxnet" % self.fake_host:
# for web_ui status page
# auto detect browser proxy setting is work
xlog.debug("CONNECT %s %s", self.command, self.path)
return self.wfile.write(self.self_check_response_data)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
def main():
global ready
for i in range(50):
check_network = simple_check_worker()
if check_network:
break
else:
time.sleep(3)
if (i+1)%5 == 0: xlog.error('Failed to connect to network, please check!')
if i+1 == 50: return
pre_start()
ip_range.load()
google_ip.check()
connect_control.keep_running = True
connect_manager.https_manager.load_config()
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
CertUtil.init_ca()
proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
proxy_thread = threading.Thread(target=proxy_daemon.serve_forever)
proxy_thread.setDaemon(True)
proxy_thread.start()
if config.PAC_ENABLE:
pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), PACServerHandler)
pac_thread = threading.Thread(target=pac_daemon.serve_forever)
pac_thread.setDaemon(True)
pac_thread.start()
urllib2.urlopen('http://127.0.0.1:%d/%s' % (config.PAC_PORT, config.PAC_FILE))
#PACServerHandler.do_GET()
ready = True # checked by launcher.module_init
while connect_control.keep_running:
time.sleep(1)
xlog.info("Exiting gae_proxy module...")
proxy_daemon.shutdown()
proxy_daemon.server_close()
proxy_thread.join()
if config.PAC_ENABLE:
pac_daemon.shutdown()
pac_daemon.server_close()
pac_thread.join()
ready = False # checked by launcher.module_init
xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running)
def redirect_handler(sock, host, port, client_address):
leadbyte = sock.recv(1, socket.MSG_PEEK)
if leadbyte in ('\x80', '\x16'):
server_name = ''
if leadbyte == '\x16':
for _ in xrange(2):
leaddata = sock.recv(1024, socket.MSG_PEEK)
if is_clienthello(leaddata):
try:
server_name = extract_sni_name(leaddata)
finally:
break
try:
certfile = CertUtil.get_cert(server_name or 'www.google.com')
ssl_sock = ssl.wrap_socket(sock, keyfile=CertUtil.cert_keyfile,
certfile=certfile, server_side=True)
except StandardError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('redirect_handler wrap_socket from:%s to:%s:%d sni:%s failed:%r',
client_address, host, port, server_name, e)
return
elif leadbyte in ["G", "P", "D", "O", "H", "T"]:
ssl_sock = sock
else:
xlog.warn("redirect_handler lead byte:%s", leadbyte)
return
handler = GAEProxyHandler(ssl_sock, client_address, None, logger=xlog)
xlog.debug('redirect_handler from:%s to:%s:%d', client_address, host, port)
client_thread = threading.Thread(target=handler.handle)
client_thread.start()
def main():
# to profile goagent, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result.
do_profile = False
if do_profile:
import cProfile, pstats
pr = cProfile.Profile()
pr.enable()
global __file__
__file__ = os.path.abspath(__file__)
if os.path.islink(__file__):
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
logging.basicConfig(level=logging.DEBUG if config.LISTEN_DEBUGINFO else logging.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]')
pre_start()
logging.info(config.info())
CertUtil.init_ca()
server = LocalProxyServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler)
p = threading.Thread(target=server.serve_forever)
p.setDaemon(True)
p.start()
if config.PAC_ENABLE:
server = LocalProxyServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler)
p = threading.Thread(target=server.serve_forever)
p.setDaemon(True)
p.start()
if config.CONTROL_ENABLE:
control_server = LocalProxyServer((config.CONTROL_IP, config.CONTROL_PORT), web_control.RemoteContralServerHandler)
p = threading.Thread(target=control_server.serve_forever)
p.setDaemon(True)
p.start()
while config.keep_run:
time.sleep(1)
if do_profile:
pr.disable()
pr.print_stats()
def main(args):
global ready, proxy_server
check_create_data_path()
log_info()
CertUtil.init_ca()
allow_remote = args.get("allow_remote", 0)
if allow_remote:
listen_ip = "0.0.0.0"
else:
listen_ip = front.config.listen_ip
proxy_server = simple_http_server.HTTPServer(
(listen_ip, front.config.listen_port), proxy_handler.GAEProxyHandler, logger=xlog)
ready = True # checked by launcher.module_init
proxy_server.serve_forever()
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
logging.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
logging.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
logging.debug('GAE CONNECT %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def import_certfile():
CertUtil.init_ca()
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
logging.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
logging.info('ssl error: %s, create full domain cert for host:%s',
e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
logging.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(
config.HOSTS_FWD_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_FWD:
fwd_set = [s for s in config.HOSTS_FWD]
fwd_set.append(host)
config.HOSTS_FWD = tuple(fwd_set)
logging.warn(
"Method %s not support in GAE, Redirect to FWD for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
self.path).encode())
else:
logging.warn("Method %s not support in GoAgent for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT(self):
self.method = "https"
self.host_port = self.path
self.host, _, self.port = self.host_port.rpartition(':')
self.port = int(self.port)
if self.port == 443:
self.host_port = self.host
# xlog.debug('CONNECT %s:%s ', host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
certfile = CertUtil.get_cert(self.host)
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, self.host)
certfile = CertUtil.get_cert(self.host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65535 or not self.raw_requestline:
#xlog.warn("read request line len:%d", len(self.raw_requestline))
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection,
e, self.path, e.args[0])
raise
return
except Exception as e:
xlog.exception("read request line error:%r", e)
return
if self.path[0] != '/':
xlog.warn("CONNECT host:%s path:%s", self.host_port, self.path)
return
self.url = 'https://%s%s' % (self.host_port, self.path)
self.headers = dict((k.title(), v) for k, v in self.headers.items())
self.read_payload()
self.dispatch_request()
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_AGENT(self):
"""send fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
# xlog.info('https GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=CertUtil.cert_keyfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.parse_request()
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
if self.path == "https://%s/xxnet" % self.fake_host:
# for web_ui status page
# auto detect browser proxy setting is work
xlog.debug("CONNECT %s %s", self.command, self.path)
return self.wfile.write(self.self_check_response_data)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def wrap_ssl(sock, host, port, client_address):
certfile = CertUtil.get_cert(host or 'www.google.com')
ssl_sock = ssl.wrap_socket(sock, keyfile=CertUtil.cert_keyfile,
certfile=certfile, server_side=True)
return ssl_sock
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
logging.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
logging.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
logging.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(config.HOSTS_FWD_ENDSWITH) or host.endswith(config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_FWD:
fwd_set = [s for s in config.HOSTS_FWD]
fwd_set.append(host)
config.HOSTS_FWD = tuple(fwd_set)
logging.warn("Method %s not support in GAE, Redirect to FWD for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' % self.path).encode())
else:
logging.warn("Method %s not support in GoAgent for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def wrap_ssl(sock, host, port, client_address):
certfile = CertUtil.get_cert(host or 'www.google.com')
ssl_sock = ssl.wrap_socket(sock, keyfile=CertUtil.cert_keyfile,
certfile=certfile, server_side=True)
return ssl_sock
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
logging.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
logging.info('ssl error: %s, create full domain cert for host:%s',
e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
logging.debug('GAE CONNECT %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
xlog.warn("read request line len:%d",
len(self.raw_requestline))
return
if not self.raw_requestline:
xlog.warn("read request line empty")
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
if self.path == "https://www.twitter.com/xxnet":
# for web_ui status page
# auto detect browser proxy setting is work
xlog.debug("CONNECT %s %s", self.command, self.path)
return self.wfile.write(self.self_check_response_data)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(
config.HOSTS_DIRECT_ENDSWITH) or host.endswith(
config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_DIRECT:
fwd_set = [s for s in config.HOSTS_DIRECT]
fwd_set.append(host)
config.HOSTS_DIRECT = tuple(fwd_set)
xlog.warn(
"Method %s not support in GAE, Redirect to DIRECT for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
self.path).encode())
else:
xlog.warn("Method %s not support in GAEProxy for %s",
self.command, self.path)
return self.wfile.write(
('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_AGENT(self):
"""deploy fake cert to client"""
# GAE supports the following HTTP methods: GET, POST, HEAD, PUT, DELETE, and PATCH
host, _, port = self.path.rpartition(':')
port = int(port)
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
xlog.warn("read request line len:%d", len(self.raw_requestline))
return
if not self.raw_requestline:
xlog.warn("read request line empty")
return
if not self.parse_request():
xlog.warn("parse request fail:%s", self.raw_requestline)
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
if self.path == "https://www.twitter.com/xxnet":
# for web_ui status page
# auto detect browser proxy setting is work
xlog.debug("CONNECT %s %s", self.command, self.path)
return self.wfile.write(self.self_check_response_data)
xlog.debug('GAE CONNECT %s %s', self.command, self.path)
if self.command not in self.gae_support_methods:
if host.endswith(".google.com") or host.endswith(config.HOSTS_DIRECT_ENDSWITH) or host.endswith(config.HOSTS_GAE_ENDSWITH):
if host in config.HOSTS_GAE:
gae_set = [s for s in config.HOSTS_GAE]
gae_set.remove(host)
config.HOSTS_GAE = tuple(gae_set)
if host not in config.HOSTS_DIRECT:
fwd_set = [s for s in config.HOSTS_DIRECT]
fwd_set.append(host)
config.HOSTS_DIRECT = tuple(fwd_set)
xlog.warn("Method %s not support in GAE, Redirect to DIRECT for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 301\r\nLocation: %s\r\nContent-Length: 0\r\n\r\n' % self.path).encode())
else:
xlog.warn("Method %s not support in GAEProxy for %s", self.command, self.path)
return self.wfile.write(('HTTP/1.1 404 Not Found\r\n\r\n').encode())
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
return self.do_AGENT()
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_DIRECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection,
keyfile=certfile,
certfile=certfile,
server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e,
host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception(
'ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s',
self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET,
errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT Direct %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
if len(self.parsed_url[4]):
path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
path = self.parsed_url[2]
request_headers = dict(
(k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
#logging.debug("payload_len:%d %s %s", payload_len, self.command, self.path)
payload = self.rfile.read(payload_len)
except NetWorkIOError as e:
xlog.error('handle_method_urlfetch read payload failed:%s',
e)
return
direct_handler.handler(self.command, host, path, request_headers,
payload, self.wfile)
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT,
errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def do_CONNECT_DIRECT(self):
"""deploy fake cert to client"""
host, _, port = self.path.rpartition(':')
port = int(port)
if port != 443:
xlog.warn("CONNECT %s port:%d not support", host, port)
return
certfile = CertUtil.get_cert(host)
xlog.info('GAE %s %s:%d ', self.command, host, port)
self.__realconnection = None
self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
try:
ssl_sock = ssl.wrap_socket(self.connection, keyfile=certfile, certfile=certfile, server_side=True)
except ssl.SSLError as e:
xlog.info('ssl error: %s, create full domain cert for host:%s', e, host)
certfile = CertUtil.get_cert(host, full_name=True)
return
except Exception as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
xlog.exception('ssl.wrap_socket(self.connection=%r) failed: %s path:%s, errno:%s', self.connection, e, self.path, e.args[0])
return
self.__realconnection = self.connection
self.__realwfile = self.wfile
self.__realrfile = self.rfile
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
try:
self.raw_requestline = self.rfile.readline(65537)
if len(self.raw_requestline) > 65536:
self.requestline = ''
self.request_version = ''
self.command = ''
self.send_error(414)
return
if not self.raw_requestline:
self.close_connection = 1
return
if not self.parse_request():
return
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.EPIPE):
raise
if self.path[0] == '/' and host:
self.path = 'https://%s%s' % (self.headers['Host'], self.path)
xlog.debug('GAE CONNECT Direct %s %s', self.command, self.path)
try:
if self.path[0] == '/' and host:
self.path = 'http://%s%s' % (host, self.path)
elif not host and '://' in self.path:
host = urlparse.urlparse(self.path).netloc
self.parsed_url = urlparse.urlparse(self.path)
if len(self.parsed_url[4]):
path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
path = self.parsed_url[2]
request_headers = dict((k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
#logging.debug("payload_len:%d %s %s", payload_len, self.command, self.path)
payload = self.rfile.read(payload_len)
except NetWorkIOError as e:
xlog.error('handle_method_urlfetch read payload failed:%s', e)
return
direct_handler.handler(self.command, host, path, request_headers, payload, self.wfile)
except NetWorkIOError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ETIMEDOUT, errno.EPIPE):
raise
finally:
if self.__realconnection:
try:
self.__realconnection.shutdown(socket.SHUT_WR)
self.__realconnection.close()
except NetWorkIOError:
pass
finally:
self.__realconnection = None
def import_certfile():
CertUtil.init_ca()
