def smart_route_proxy_socks4(): proxy = "socks4://localhost:8086" res = simple_http_client.request("GET", "https://github.com/", proxy=proxy, timeout=1000) return res
def request_gae_server(headers, body, url, timeout): # process on http protocol # process status code return by http server # raise error, let up layer retry. #response = front.request("POST", None, "/_gh/", headers, body, timeout) #response = front.request("POST", "127.0.0.1:3003", "/", headers, body, timeout) response = simple_http_client.request("POST", "http://127.0.0.1:3003/", headers, body) #print(response.status) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") # content_type = response.getheaders("content-type", "") #if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ #response.status == 403 or response.status == 405: # ## some ip can connect, and server type can be gws ## but can't use as GAE server ## so we need remove it immediately # #xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, #response.status) #front.ip_manager.recheck_ip(response.ssl_sock.ip) #response.worker.close("ip not support GAE") #raise GAE_Exception(602, "ip not support GAE") # #appid = response.ssl_sock.host.split(".")[0] #appid = "appid" #if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) #front.appid_manager.report_not_exist( #appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") #response.worker.close("appid not exist:%s" % appid) #raise GAE_Exception(404, "appid not exist %s") #if response.status == 503: ##xlog.warning('APPID %r out of Quota, remove it. %s', ##appid, response.ssl_sock.ip) ##front.appid_manager.report_out_of_quota(appid) ## google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") ##response.worker.close("appid out of quota:%s" % appid) #raise GAE_Exception(604, "appid out of quota:%s" % appid) # #if response.status > 300: #raise GAE_Exception(605, "status:%d" % response.status) # #if response.status != 200: #xlog.warn("GAE %s appid:%s status:%d", "ip", #appid, response.status) return response
def forward_local(self): """ If browser send localhost:xxx request to GAE_proxy, we forward it to localhost. """ request_headers = dict((k.title(), v) for k, v in self.headers.items()) payload = b'' if 'Content-Length' in request_headers: try: payload_len = int(request_headers.get('Content-Length', 0)) payload = self.rfile.read(payload_len) except Exception as e: xlog.warn('forward_local read payload failed:%s', e) return response = simple_http_client.request(self.command, self.path, request_headers, payload) if not response: xlog.warn("forward_local fail, command:%s, path:%s, headers: %s, payload: %s", self.command, self.path, request_headers, payload) return out_list = [] out_list.append("HTTP/1.1 %d\r\n" % response.status) for key in response.headers: key = key.title() out_list.append("%s: %s\r\n" % (key, response.headers[key])) out_list.append("\r\n") out_list.append(response.text) self.wfile.write("".join(out_list))
def test_get_fail(self): headers = {"connection": "close"} res = simple_http_client.request("GET", "http://127.0.0.1:2515/ping", headers=headers, timeout=0.5) self.assertIsNone(res)
def request(method, host, schema="http", path="/", headers={}, data="", timeout=60): global last_success_time, last_fail_time, continue_fail_num, success_num, fail_num timeout = 30 # use http to avoid cert fail url = "http://" + host + path if data: headers["Content-Length"] = str(len(data)) # xlog.debug("gae_proxy %s %s", method, url) try: response = simple_http_client.request(method, url, headers, data, timeout=timeout) if response.status != 200: raise Exception("Direct request fail") except Exception as e: fail_num += 1 continue_fail_num += 1 last_fail_time = time.time() return "", 602, {} last_success_time = time.time() continue_fail_num = 0 success_num += 1 response.worker = FakeWorker() response.task = response.worker return response.text, response.status, response
def smart_route_proxy_socks5(self): xlog.info("Start testing SmartRouter SOCKS5 proxy protocol") proxy = "socks5://localhost:8086" res = simple_http_client.request("GET", "https://github.com/", proxy=proxy, timeout=15) self.assertEqual(res.status, 200) xlog.info("Finished testing SmartRouter SOCKS5 proxy protocol")
def xtunnel_logout(self): xlog.info("Start testing XTunnel logout") res = simple_http_client.request( "POST", "http://127.0.0.1:8085/module/x_tunnel/control/logout", timeout=10) self.assertEqual(res.status, 200) self.xtunnel_login_status = False xlog.info("Finished testing XTunnel logout")
def check_web_console(self): try: res = simple_http_client.request("GET", "http://127.0.0.1:8085/", timeout=1) return res is not None and res.status in [200, 404] # 404 is because act running locally may lost some folder. just bypass this error. except Exception as e: # xlog.debug("get web_console fail:%r", e) return False
def req_get_installed_app(self): if sys_platform.platform != 'android': # simulate data for developing data = { "proxy_by_app": config.proxy_by_app, "installed_app_list": [{ "name": "Test", "package": "com.test" }, { "name": "APP", "package": "com.app" }, { "name": "com.google.foundation.Foundation.Application.", "package": "com.google.application" }] } time.sleep(5) else: res = simple_http_client.request( "GET", "http://localhost:8084/installed_app_list/") data = json.loads(res.text) data["proxy_by_app"] = config.proxy_by_app for app in data["installed_app_list"]: package = app["package"] if package in config.enabled_app_list: app["enable"] = True else: app["enable"] = False if config.proxy_by_app: # Pass the config in html. content = '<div id="proxy_by_app_config" checked hidden></div>' else: content = '<div id="proxy_by_app_config" hidden></div>' for app in data["installed_app_list"]: if app["enable"]: checked = " checked " else: checked = "" content += '<div class="row-fluid"> <div class="config_label" >'\ + app["name"] \ + '</div><div class="config_switch"><input class="app_item" id="'\ + app['package']\ + '" type="checkbox" data-toggle="switch" '\ + checked\ + '/></div></div>\n'\ # jquery can't work on dynamic insert elements. # load html content from backend is the best way to make it works. return self.send_response("text/html", content)
def xtunnel_proxy_socks5(self): xlog.info("Start testing XTunnel Socks5 proxy protocol") if not self.xtunnel_login_status: self.xtunnel_login() proxy = "socks5://localhost:1080" res = simple_http_client.request("GET", "https://github.com/", proxy=proxy, timeout=15) self.assertEqual(res.status, 200) xlog.info("Finished testing XTunnel Socks5 proxy protocol")
def stop_xxnet(self): xlog.info("call api to Quit xxnet") try: res = simple_http_client.request("GET", "http://127.0.0.1:8085/quit", timeout=0.5) if res: xlog.info("Quit API res:%s", res.text) else: xlog.info("Quit API request failed.") except Exception as e: xlog.info("Quit API except:%r", e)
def forward_local(self): """ If browser send localhost:xxx request to GAE_proxy, we forward it to localhost. """ request_headers = dict((k.title(), v) for k, v in self.headers.items()) payload = b'' method = None url = None if 'Content-Length' in request_headers: try: payload_len = int(request_headers.get('Content-Length', 0)) payload = self.rfile.read(payload_len) #print(payload) payload_length, = struct.unpack('!h', payload[:2]) #print("len: ", payload_length) payload = inflate(payload[2:2 + payload_length]) raw_response_line, payload = payload.split('\r\n', 1) method, url = raw_response_line.split()[:2] print(method, url) except Exception as e: xlog.warn('forward_local read payload failed:%s', e) return print(payload) #response = simple_http_client.request(self.command, self.path, request_headers, payload) response = simple_http_client.request(method, url, request_headers, payload) if not response: xlog.warn("forward_local fail, command:%s, path:%s, headers: %s, payload: %s", method, url, request_headers, payload) return out_list = [] out_list.append("HTTP/1.1 %d\r\n" % response.status) for key in response.headers: key = key.title() out_list.append("%s: %s\r\n" % (key, response.headers[key])) out_list.append("\r\n") content = response.text if isinstance(content, memoryview): content = content.tobytes() out_list.append(content) #print(out_list) self.wfile.write("".join(out_list))
def _check_one_host(url): try: header = { "user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36", "accept": "application/json, text/javascript, */*; q=0.01", "accept-encoding": "gzip, deflate, sdch", "accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2', "connection": "keep-alive" } response = simple_http_client.request("HEAD", url, header, "", proxy=proxy, read_payload=False) if response: return True except Exception as e: if __name__ == "__main__": xlog.exception("test %s e:%r", url, e) return False
def _get_os_language(): if sys_platform.platform == "mac": try: oot = os.pipe() p = subprocess.Popen(["/usr/bin/defaults", 'read', 'NSGlobalDomain', 'AppleLanguages'], stdout=oot[1]) p.communicate() lang_code = os.read(oot[0], 10000) if b'zh' in lang_code: return 'zh_CN' elif b'en' in lang_code: return 'en_US' elif b'fa' in lang_code: return 'fa_IR' except: pass elif sys_platform.platform == "android": try: res = request("GET", "http://localhost:8084/env/") dat = json.loads(res.text) lang_code = dat["lang_code"] xlog.debug("lang_code:%s", lang_code) if 'zh' in lang_code: return 'zh_CN' elif 'en' in lang_code: return 'en_US' elif 'fa' in lang_code: return 'fa_IR' else: return None except Exception as e: xlog.exception("get lang except:%r", e) return None else: try: lang_code, code_page = locale.getdefaultlocale() # ('en_GB', 'cp1252'), en_US, return lang_code except: # Mac fail to run this pass
def __init__(self): self.xtunnel_login_status = False self.running = True self.pth = None self.log_fp = None self.log_fn = None # Lock for one integrate testing at the same time. # github act running in local will run multi python version in same VM, so we need to lock to avoid conflict. while True: try: res = simple_http_client.request("GET", "http://127.0.0.1:8888/test") if res and res.status == 200: time.sleep(1) continue else: break except: break self.lock_server = simple_http_server.HTTPServer( ('', 8888), simple_http_server.TestHttpServer, ".") self.lock_server.start() if self.check_web_console(): xlog.info("APP was running externally.") self.th = None return self.log_fn = os.path.join(root_path, "running.log") if sys.version_info[0] == 3: self.log_fp = open(self.log_fn, "wb") else: self.log_fp = open(self.log_fn, "w") self.th = threading.Thread(target=self.start_xxnet) self.th.start()
def xtunnel_login(self): xlog.info("Start testing XTunnel login") headers = {"Content-Type": "application/json"} data = { "username": [ os.getenv("XTUNNEL_USER"), ], "password": [ os.getenv("XTUNNEL_PASS"), ], "is_register": [ 0, ] } data = json.dumps(data) res = simple_http_client.request( "POST", "http://127.0.0.1:8085/module/x_tunnel/control/login", headers=headers, body=data, timeout=60) self.assertEqual(res.status, 200) self.xtunnel_login_status = True xlog.info("Finished testing XTunnel login")
import os from front_base.config import ConfigBase import simple_http_client current_path = os.path.dirname(os.path.abspath(__file__)) root_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir)) data_path = os.path.abspath( os.path.join(root_path, os.pardir, os.pardir, 'data')) module_data_path = os.path.join(data_path, 'gae_proxy') headers = {"connection": "close"} fqrouter = simple_http_client.request("GET", "http://127.0.0.1:2515/ping", headers=headers, timeout=0.5) mobile = fqrouter and "PONG" in fqrouter.text del headers, fqrouter class Config(ConfigBase): def __init__(self, fn): super(Config, self).__init__(fn) # globa setting level # passive < conservative < normal < radical < extreme self.set_var("setting_level", "normal") # proxy self.set_var("listen_ip", "127.0.0.1") self.set_var("listen_port", 8087)
def do_GET(self): # self.headers = utils.to_str(self.headers) self.path = utils.to_str(self.path) refer = self.headers.get('Referer') if refer: refer_loc = urlparse(refer).netloc host = self.headers.get('Host') if refer_loc != host: xlog.warn("web control ref:%s host:%s", refer_loc, host) return # check for '..', which will leak file if re.search(r'(\.{2})', self.path) is not None: self.wfile.write(b'HTTP/1.1 404\r\n\r\n') xlog.warn('%s %s %s haking', self.address_string(), self.command, self.path) return url_path = urlparse(self.path).path if url_path == '/': return self.req_index_handler() url_path_list = self.path.split('/') if len(url_path_list) >= 3 and url_path_list[1] == "module": module = url_path_list[2] if len(url_path_list) >= 4 and url_path_list[3] == "control": if module not in module_init.proc_handler: xlog.warn("request %s no module in path", url_path) self.send_not_found() return if "imp" not in module_init.proc_handler[module]: xlog.warn("request module:%s start fail", module) self.send_not_found() return path = '/' + '/'.join(url_path_list[4:]) controler = module_init.proc_handler[module][ "imp"].local.web_control.ControlHandler( self.client_address, self.headers, self.command, path, self.rfile, self.wfile) controler.do_GET() return else: relate_path = '/'.join(url_path_list[3:]) file_path = os.path.join(root_path, module, "web_ui", relate_path) if not os.path.isfile(file_path): return self.send_not_found() # i18n code lines (Both the locale dir & the template dir are module-dependent) locale_dir = os.path.abspath( os.path.join(root_path, module, 'lang')) content = i18n_translator.render(locale_dir, file_path) return self.send_response('text/html', content) else: file_path = os.path.join(current_path, 'web_ui' + url_path) if os.path.isfile(file_path): if file_path.endswith('.js'): mimetype = 'application/javascript' elif file_path.endswith('.css'): mimetype = 'text/css' elif file_path.endswith('.html'): mimetype = 'text/html' elif file_path.endswith('.jpg'): mimetype = 'image/jpeg' elif file_path.endswith('.png'): mimetype = 'image/png' else: mimetype = 'text/plain' self.send_file(file_path, mimetype) else: xlog.debug('launcher web_control %s %s %s ', self.address_string(), self.command, self.path) if url_path == '/config': self.req_config_handler() elif url_path == '/update': self.req_update_handler() elif url_path == '/config_proxy': self.req_config_proxy_handler() elif url_path == '/installed_app': self.req_get_installed_app() elif url_path == '/init_module': self.req_init_module_handler() elif url_path == '/quit': content = b'Exited successfully.' self.send_response('text/html', content) self.wfile.flush() if sys_platform.platform == "android": simple_http_client.request("GET", "http://localhost:8084/quit/", timeout=0.2) sys_platform.on_quit() elif url_path == "/debug": self.req_debug_handler() elif url_path == '/restart': self.send_response('text/html', '{"status":"success"}') update_from_github.restart_xxnet() else: self.send_not_found() xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)
import os from front_base.config import ConfigBase import simple_http_client current_path = os.path.dirname(os.path.abspath(__file__)) root_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir)) data_path = os.path.abspath(os.path.join(root_path, os.pardir, os.pardir, 'data')) module_data_path = os.path.join(data_path, 'gae_proxy') headers = {"connection": "close"} fqrouter = simple_http_client.request("GET", "http://127.0.0.1:2515/ping", headers=headers, timeout=0.5) mobile = fqrouter and "PONG" in fqrouter.text del headers, fqrouter class Config(ConfigBase): def __init__(self, fn): super(Config, self).__init__(fn) # globa setting level # passive < conservative < normal < radical < extreme self.set_var("setting_level", "normal") # proxy self.set_var("listen_ip", "127.0.0.1") self.set_var("listen_port", 8087) # auto range self.set_var("AUTORANGE_THREADS", 10) self.set_var("AUTORANGE_MAXSIZE", 512 * 1024)