def smart_route_proxy_socks4():
proxy = "socks4://localhost:8086"
res = simple_http_client.request("GET",
"https://github.com/",
proxy=proxy,
timeout=1000)
return res
def request_gae_server(headers, body, url, timeout):
# process on http protocol
# process status code return by http server
# raise error, let up layer retry.
#response = front.request("POST", None, "/_gh/", headers, body, timeout)
#response = front.request("POST", "127.0.0.1:3003", "/", headers, body, timeout)
response = simple_http_client.request("POST", "http://127.0.0.1:3003/", headers, body)
#print(response.status)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(
response.status, "fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
# content_type = response.getheaders("content-type", "")
#if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
#response.status == 403 or response.status == 405:
#
## some ip can connect, and server type can be gws
## but can't use as GAE server
## so we need remove it immediately
#
#xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers,
#response.status)
#front.ip_manager.recheck_ip(response.ssl_sock.ip)
#response.worker.close("ip not support GAE")
#raise GAE_Exception(602, "ip not support GAE")
#
#appid = response.ssl_sock.host.split(".")[0]
#appid = "appid"
#if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
#front.appid_manager.report_not_exist(
#appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
#response.worker.close("appid not exist:%s" % appid)
#raise GAE_Exception(404, "appid not exist %s")
#if response.status == 503:
##xlog.warning('APPID %r out of Quota, remove it. %s',
##appid, response.ssl_sock.ip)
##front.appid_manager.report_out_of_quota(appid)
## google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
##response.worker.close("appid out of quota:%s" % appid)
#raise GAE_Exception(604, "appid out of quota:%s" % appid)
#
#if response.status > 300:
#raise GAE_Exception(605, "status:%d" % response.status)
#
#if response.status != 200:
#xlog.warn("GAE %s appid:%s status:%d", "ip",
#appid, response.status)
return response
def forward_local(self):
"""
If browser send localhost:xxx request to GAE_proxy,
we forward it to localhost.
"""
request_headers = dict((k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
payload = self.rfile.read(payload_len)
except Exception as e:
xlog.warn('forward_local read payload failed:%s', e)
return
response = simple_http_client.request(self.command, self.path, request_headers, payload)
if not response:
xlog.warn("forward_local fail, command:%s, path:%s, headers: %s, payload: %s",
self.command, self.path, request_headers, payload)
return
out_list = []
out_list.append("HTTP/1.1 %d\r\n" % response.status)
for key in response.headers:
key = key.title()
out_list.append("%s: %s\r\n" % (key, response.headers[key]))
out_list.append("\r\n")
out_list.append(response.text)
self.wfile.write("".join(out_list))
def test_get_fail(self):
headers = {"connection": "close"}
res = simple_http_client.request("GET",
"http://127.0.0.1:2515/ping",
headers=headers,
timeout=0.5)
self.assertIsNone(res)
def forward_local(self):
"""
If browser send localhost:xxx request to GAE_proxy,
we forward it to localhost.
"""
request_headers = dict((k.title(), v) for k, v in self.headers.items())
payload = b''
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
payload = self.rfile.read(payload_len)
except Exception as e:
xlog.warn('forward_local read payload failed:%s', e)
return
response = simple_http_client.request(self.command, self.path, request_headers, payload)
if not response:
xlog.warn("forward_local fail, command:%s, path:%s, headers: %s, payload: %s",
self.command, self.path, request_headers, payload)
return
out_list = []
out_list.append("HTTP/1.1 %d\r\n" % response.status)
for key in response.headers:
key = key.title()
out_list.append("%s: %s\r\n" % (key, response.headers[key]))
out_list.append("\r\n")
out_list.append(response.text)
self.wfile.write("".join(out_list))
def request(method, host, schema="http", path="/", headers={}, data="", timeout=60):
global last_success_time, last_fail_time, continue_fail_num, success_num, fail_num
timeout = 30
# use http to avoid cert fail
url = "http://" + host + path
if data:
headers["Content-Length"] = str(len(data))
# xlog.debug("gae_proxy %s %s", method, url)
try:
response = simple_http_client.request(method, url, headers, data, timeout=timeout)
if response.status != 200:
raise Exception("Direct request fail")
except Exception as e:
fail_num += 1
continue_fail_num += 1
last_fail_time = time.time()
return "", 602, {}
last_success_time = time.time()
continue_fail_num = 0
success_num += 1
response.worker = FakeWorker()
response.task = response.worker
return response.text, response.status, response
def smart_route_proxy_socks5(self):
xlog.info("Start testing SmartRouter SOCKS5 proxy protocol")
proxy = "socks5://localhost:8086"
res = simple_http_client.request("GET",
"https://github.com/",
proxy=proxy,
timeout=15)
self.assertEqual(res.status, 200)
xlog.info("Finished testing SmartRouter SOCKS5 proxy protocol")
def xtunnel_logout(self):
xlog.info("Start testing XTunnel logout")
res = simple_http_client.request(
"POST",
"http://127.0.0.1:8085/module/x_tunnel/control/logout",
timeout=10)
self.assertEqual(res.status, 200)
self.xtunnel_login_status = False
xlog.info("Finished testing XTunnel logout")
def check_web_console(self):
try:
res = simple_http_client.request("GET",
"http://127.0.0.1:8085/",
timeout=1)
return res is not None and res.status in [200, 404]
# 404 is because act running locally may lost some folder. just bypass this error.
except Exception as e:
# xlog.debug("get web_console fail:%r", e)
return False
def req_get_installed_app(self):
if sys_platform.platform != 'android':
# simulate data for developing
data = {
"proxy_by_app":
config.proxy_by_app,
"installed_app_list": [{
"name": "Test",
"package": "com.test"
}, {
"name": "APP",
"package": "com.app"
}, {
"name": "com.google.foundation.Foundation.Application.",
"package": "com.google.application"
}]
}
time.sleep(5)
else:
res = simple_http_client.request(
"GET", "http://localhost:8084/installed_app_list/")
data = json.loads(res.text)
data["proxy_by_app"] = config.proxy_by_app
for app in data["installed_app_list"]:
package = app["package"]
if package in config.enabled_app_list:
app["enable"] = True
else:
app["enable"] = False
if config.proxy_by_app:
# Pass the config in html.
content = '<div id="proxy_by_app_config" checked hidden></div>'
else:
content = '<div id="proxy_by_app_config" hidden></div>'
for app in data["installed_app_list"]:
if app["enable"]:
checked = " checked "
else:
checked = ""
content += '<div class="row-fluid"> <div class="config_label" >'\
+ app["name"] \
+ '</div><div class="config_switch"><input class="app_item" id="'\
+ app['package']\
+ '" type="checkbox" data-toggle="switch" '\
+ checked\
+ '/></div></div>\n'\
# jquery can't work on dynamic insert elements.
# load html content from backend is the best way to make it works.
return self.send_response("text/html", content)
def xtunnel_proxy_socks5(self):
xlog.info("Start testing XTunnel Socks5 proxy protocol")
if not self.xtunnel_login_status:
self.xtunnel_login()
proxy = "socks5://localhost:1080"
res = simple_http_client.request("GET",
"https://github.com/",
proxy=proxy,
timeout=15)
self.assertEqual(res.status, 200)
xlog.info("Finished testing XTunnel Socks5 proxy protocol")
def stop_xxnet(self):
xlog.info("call api to Quit xxnet")
try:
res = simple_http_client.request("GET",
"http://127.0.0.1:8085/quit",
timeout=0.5)
if res:
xlog.info("Quit API res:%s", res.text)
else:
xlog.info("Quit API request failed.")
except Exception as e:
xlog.info("Quit API except:%r", e)
def forward_local(self):
"""
If browser send localhost:xxx request to GAE_proxy,
we forward it to localhost.
"""
request_headers = dict((k.title(), v) for k, v in self.headers.items())
payload = b''
method = None
url = None
if 'Content-Length' in request_headers:
try:
payload_len = int(request_headers.get('Content-Length', 0))
payload = self.rfile.read(payload_len)
#print(payload)
payload_length, = struct.unpack('!h', payload[:2])
#print("len: ", payload_length)
payload = inflate(payload[2:2 + payload_length])
raw_response_line, payload = payload.split('\r\n', 1)
method, url = raw_response_line.split()[:2]
print(method, url)
except Exception as e:
xlog.warn('forward_local read payload failed:%s', e)
return
print(payload)
#response = simple_http_client.request(self.command, self.path, request_headers, payload)
response = simple_http_client.request(method, url, request_headers, payload)
if not response:
xlog.warn("forward_local fail, command:%s, path:%s, headers: %s, payload: %s",
method, url, request_headers, payload)
return
out_list = []
out_list.append("HTTP/1.1 %d\r\n" % response.status)
for key in response.headers:
key = key.title()
out_list.append("%s: %s\r\n" % (key, response.headers[key]))
out_list.append("\r\n")
content = response.text
if isinstance(content, memoryview):
content = content.tobytes()
out_list.append(content)
#print(out_list)
self.wfile.write("".join(out_list))
def _check_one_host(url):
try:
header = {
"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, sdch",
"accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
"connection": "keep-alive"
}
response = simple_http_client.request("HEAD", url, header, "", proxy=proxy, read_payload=False)
if response:
return True
except Exception as e:
if __name__ == "__main__":
xlog.exception("test %s e:%r", url, e)
return False
def _get_os_language():
if sys_platform.platform == "mac":
try:
oot = os.pipe()
p = subprocess.Popen(["/usr/bin/defaults", 'read', 'NSGlobalDomain', 'AppleLanguages'], stdout=oot[1])
p.communicate()
lang_code = os.read(oot[0], 10000)
if b'zh' in lang_code:
return 'zh_CN'
elif b'en' in lang_code:
return 'en_US'
elif b'fa' in lang_code:
return 'fa_IR'
except:
pass
elif sys_platform.platform == "android":
try:
res = request("GET", "http://localhost:8084/env/")
dat = json.loads(res.text)
lang_code = dat["lang_code"]
xlog.debug("lang_code:%s", lang_code)
if 'zh' in lang_code:
return 'zh_CN'
elif 'en' in lang_code:
return 'en_US'
elif 'fa' in lang_code:
return 'fa_IR'
else:
return None
except Exception as e:
xlog.exception("get lang except:%r", e)
return None
else:
try:
lang_code, code_page = locale.getdefaultlocale()
# ('en_GB', 'cp1252'), en_US,
return lang_code
except:
# Mac fail to run this
pass
def __init__(self):
self.xtunnel_login_status = False
self.running = True
self.pth = None
self.log_fp = None
self.log_fn = None
# Lock for one integrate testing at the same time.
# github act running in local will run multi python version in same VM, so we need to lock to avoid conflict.
while True:
try:
res = simple_http_client.request("GET",
"http://127.0.0.1:8888/test")
if res and res.status == 200:
time.sleep(1)
continue
else:
break
except:
break
self.lock_server = simple_http_server.HTTPServer(
('', 8888), simple_http_server.TestHttpServer, ".")
self.lock_server.start()
if self.check_web_console():
xlog.info("APP was running externally.")
self.th = None
return
self.log_fn = os.path.join(root_path, "running.log")
if sys.version_info[0] == 3:
self.log_fp = open(self.log_fn, "wb")
else:
self.log_fp = open(self.log_fn, "w")
self.th = threading.Thread(target=self.start_xxnet)
self.th.start()
def _check_one_host(url):
try:
header = {
"user-agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, sdch",
"accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
"connection": "keep-alive"
}
response = simple_http_client.request("HEAD",
url,
header,
"",
proxy=proxy,
read_payload=False)
if response:
return True
except Exception as e:
if __name__ == "__main__":
xlog.exception("test %s e:%r", url, e)
return False
def xtunnel_login(self):
xlog.info("Start testing XTunnel login")
headers = {"Content-Type": "application/json"}
data = {
"username": [
os.getenv("XTUNNEL_USER"),
],
"password": [
os.getenv("XTUNNEL_PASS"),
],
"is_register": [
0,
]
}
data = json.dumps(data)
res = simple_http_client.request(
"POST",
"http://127.0.0.1:8085/module/x_tunnel/control/login",
headers=headers,
body=data,
timeout=60)
self.assertEqual(res.status, 200)
self.xtunnel_login_status = True
xlog.info("Finished testing XTunnel login")
import os
from front_base.config import ConfigBase
import simple_http_client
current_path = os.path.dirname(os.path.abspath(__file__))
root_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir))
data_path = os.path.abspath(
os.path.join(root_path, os.pardir, os.pardir, 'data'))
module_data_path = os.path.join(data_path, 'gae_proxy')
headers = {"connection": "close"}
fqrouter = simple_http_client.request("GET",
"http://127.0.0.1:2515/ping",
headers=headers,
timeout=0.5)
mobile = fqrouter and "PONG" in fqrouter.text
del headers, fqrouter
class Config(ConfigBase):
def __init__(self, fn):
super(Config, self).__init__(fn)
# globa setting level
# passive < conservative < normal < radical < extreme
self.set_var("setting_level", "normal")
# proxy
self.set_var("listen_ip", "127.0.0.1")
self.set_var("listen_port", 8087)
def do_GET(self):
# self.headers = utils.to_str(self.headers)
self.path = utils.to_str(self.path)
refer = self.headers.get('Referer')
if refer:
refer_loc = urlparse(refer).netloc
host = self.headers.get('Host')
if refer_loc != host:
xlog.warn("web control ref:%s host:%s", refer_loc, host)
return
# check for '..', which will leak file
if re.search(r'(\.{2})', self.path) is not None:
self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
xlog.warn('%s %s %s haking', self.address_string(), self.command,
self.path)
return
url_path = urlparse(self.path).path
if url_path == '/':
return self.req_index_handler()
url_path_list = self.path.split('/')
if len(url_path_list) >= 3 and url_path_list[1] == "module":
module = url_path_list[2]
if len(url_path_list) >= 4 and url_path_list[3] == "control":
if module not in module_init.proc_handler:
xlog.warn("request %s no module in path", url_path)
self.send_not_found()
return
if "imp" not in module_init.proc_handler[module]:
xlog.warn("request module:%s start fail", module)
self.send_not_found()
return
path = '/' + '/'.join(url_path_list[4:])
controler = module_init.proc_handler[module][
"imp"].local.web_control.ControlHandler(
self.client_address, self.headers, self.command, path,
self.rfile, self.wfile)
controler.do_GET()
return
else:
relate_path = '/'.join(url_path_list[3:])
file_path = os.path.join(root_path, module, "web_ui",
relate_path)
if not os.path.isfile(file_path):
return self.send_not_found()
# i18n code lines (Both the locale dir & the template dir are module-dependent)
locale_dir = os.path.abspath(
os.path.join(root_path, module, 'lang'))
content = i18n_translator.render(locale_dir, file_path)
return self.send_response('text/html', content)
else:
file_path = os.path.join(current_path, 'web_ui' + url_path)
if os.path.isfile(file_path):
if file_path.endswith('.js'):
mimetype = 'application/javascript'
elif file_path.endswith('.css'):
mimetype = 'text/css'
elif file_path.endswith('.html'):
mimetype = 'text/html'
elif file_path.endswith('.jpg'):
mimetype = 'image/jpeg'
elif file_path.endswith('.png'):
mimetype = 'image/png'
else:
mimetype = 'text/plain'
self.send_file(file_path, mimetype)
else:
xlog.debug('launcher web_control %s %s %s ', self.address_string(),
self.command, self.path)
if url_path == '/config':
self.req_config_handler()
elif url_path == '/update':
self.req_update_handler()
elif url_path == '/config_proxy':
self.req_config_proxy_handler()
elif url_path == '/installed_app':
self.req_get_installed_app()
elif url_path == '/init_module':
self.req_init_module_handler()
elif url_path == '/quit':
content = b'Exited successfully.'
self.send_response('text/html', content)
self.wfile.flush()
if sys_platform.platform == "android":
simple_http_client.request("GET",
"http://localhost:8084/quit/",
timeout=0.2)
sys_platform.on_quit()
elif url_path == "/debug":
self.req_debug_handler()
elif url_path == '/restart':
self.send_response('text/html', '{"status":"success"}')
update_from_github.restart_xxnet()
else:
self.send_not_found()
xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(),
self.command, self.path)
import os
from front_base.config import ConfigBase
import simple_http_client
current_path = os.path.dirname(os.path.abspath(__file__))
root_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir))
data_path = os.path.abspath(os.path.join(root_path, os.pardir, os.pardir, 'data'))
module_data_path = os.path.join(data_path, 'gae_proxy')
headers = {"connection": "close"}
fqrouter = simple_http_client.request("GET", "http://127.0.0.1:2515/ping", headers=headers, timeout=0.5)
mobile = fqrouter and "PONG" in fqrouter.text
del headers, fqrouter
class Config(ConfigBase):
def __init__(self, fn):
super(Config, self).__init__(fn)
# globa setting level
# passive < conservative < normal < radical < extreme
self.set_var("setting_level", "normal")
# proxy
self.set_var("listen_ip", "127.0.0.1")
self.set_var("listen_port", 8087)
# auto range
self.set_var("AUTORANGE_THREADS", 10)
self.set_var("AUTORANGE_MAXSIZE", 512 * 1024)
