def request(url, retry=0, timeout=30):
if retry == 0:
if int(config.get(["proxy", "enable"], 0)):
client = simple_http_client.Client(proxy={
"type":
config.get(["proxy", "type"], ""),
"host":
config.get(["proxy", "host"], ""),
"port":
int(config.get(["proxy", "port"], 0)),
"user":
config.get(["proxy", "user"], ""),
"pass":
config.get(["proxy", "passwd"], ""),
},
timeout=timeout)
else:
client = simple_http_client.Client(timeout=timeout)
else:
cert = os.path.join(data_root, "gae_proxy", "CA.crt")
client = simple_http_client.Client(proxy={
"type": "http",
"host": "127.0.0.1",
"port": 8087,
"user": None,
"pass": None
},
timeout=timeout,
cert=cert)
res = client.request("GET", url, read_payload=False)
return res
def request(url, retry=0, timeout=30):
if retry == 0:
if config.global_proxy_enable == 1:
client = simple_http_client.Client(proxy={
"type":
config.global_proxy_type,
"host":
config.global_proxy_host,
"port":
config.global_proxy_port,
"user":
config.global_proxy_username,
"pass":
config.global_proxy_password,
},
timeout=timeout)
else:
client = simple_http_client.Client(timeout=timeout)
else:
cert = os.path.join(data_root, "gae_proxy", "CA.crt")
client = simple_http_client.Client(proxy={
"type": "http",
"host": "127.0.0.1",
"port": 8087,
"user": None,
"pass": None
},
timeout=timeout,
cert=cert)
res = client.request("GET", url, read_payload=False)
return res
def update_front_domains(self):
next_update_time = time.time()
while self.front.running:
if time.time() < next_update_time:
time.sleep(4)
continue
try:
timeout = 30
if self.config.PROXY_ENABLE:
client = simple_http_client.Client(proxy={
"type":
self.config.PROXY_TYPE,
"host":
self.config.PROXY_HOST,
"port":
self.config.PROXY_PORT,
"user":
self.config.PROXY_USER,
"pass":
self.config.PROXY_PASSWD,
},
timeout=timeout)
else:
client = simple_http_client.Client(timeout=timeout)
url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
response = client.request("GET", url)
if not response or response.status != 200:
if response:
self.logger.warn("update front domains fail:%d",
response.status)
next_update_time = time.time() + (1800)
continue
content = response.text
if isinstance(content, memoryview):
content = content.tobytes()
need_update = True
front_domains_fn = self.fn
if os.path.exists(front_domains_fn):
with open(front_domains_fn, "r") as fd:
old_content = fd.read()
if content == old_content:
need_update = False
if need_update:
with open(front_domains_fn, "w") as fd:
fd.write(content.decode("utf-8"))
self.load()
self.logger.info(
"updated cloudflare front domains from github.")
next_update_time = time.time() + (4 * 3600)
except Exception as e:
next_update_time = time.time() + (1800)
self.logger.exception(
"updated cloudflare front domains from github fail:%r", e)
def update_front_domains():
next_update_time = time.time()
while connect_control.keep_running:
if time.time() < next_update_time:
time.sleep(4)
continue
try:
timeout = 30
if config.getint("proxy", "enable", 0):
client = simple_http_client.Client(proxy={
"type":
config.get("proxy", "type", ""),
"host":
config.get("proxy", "host", ""),
"port":
int(config.get("proxy", "port", "0")),
"user":
config.get("proxy", "user", ""),
"pass":
config.get("proxy", "passwd", ""),
},
timeout=timeout)
else:
client = simple_http_client.Client(timeout=timeout)
url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
response = client.request("GET", url)
if response.status != 200:
xlog.warn("update front domains fail:%d", response.status)
raise Exception("status:%r", response.status)
need_update = True
front_domains_fn = os.path.join(config.DATA_PATH,
"front_domains.json")
if os.path.exists(front_domains_fn):
with open(front_domains_fn, "r") as fd:
old_content = fd.read()
if response.text == old_content:
need_update = False
if need_update:
with open(front_domains_fn, "w") as fd:
fd.write(response.text)
check_ip.update_front_domains()
next_update_time = time.time() + (4 * 3600)
xlog.info("updated cloudflare front domains from github.")
except Exception as e:
next_update_time = time.time() + (1800)
xlog.debug(
"updated cloudflare front domains from github fail:%r", e)
def test_get(self):
client = simple_http_client.Client(timeout=10)
url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
res = client.request("GET", url)
self.assertEquals(res.status, 200)
content = utils.to_str(res.text)
data = json.loads(content)
print(data)
def test_get2(self):
url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/update_v4.txt"
client = simple_http_client.Client(timeout=10)
res = client.request("GET", url)
self.assertEqual(res.status, 200)
content = utils.to_str(res.text)
print(content)
def get_connection(self):
while len(self.connections):
try:
[client, last_query_time] = self.connections.pop()
if time.time() - last_query_time < self.connection_timeout:
return client
except:
pass
if g.config.PROXY_ENABLE == 1:
return simple_http_client.Client(proxy={
"type": g.config.PROXY_TYPE,
"host": g.config.PROXY_HOST,
"port": g.config.PROXY_PORT,
"user": g.config.PROXY_USER,
"pass": g.config.PROXY_PASSWD,
}, timeout=self.timeout)
else:
return simple_http_client.Client(timeout=self.timeout)
def request(url, retry=0, timeout=30):
cert = os.path.join(data_path, "CA.crt")
client = simple_http_client.Client(proxy={
"type": "http",
"host": "127.0.0.1",
"port": 8087,
"user": None,
"pass": None
}, timeout=timeout, cert=cert)
res = client.request("GET", url, read_payload=False)
return res
def test_get(self):
server = simple_http_server.HTTPServer(
('', 8880), simple_http_server.TestHttpServer, ".")
server.start()
client = simple_http_client.Client(timeout=5)
url = "http://localhost:8880/test"
res = client.request("GET", url)
self.assertEqual(res.status, 200)
content = utils.to_str(res.text)
print(content)
server.shutdown()
def __init__(self, type="IPv4"):
self.type = type
self.urls = []
self._checking_lock = threading.Lock()
self._checking_num = 0
self.network_stat = "unknown"
self.last_check_time = 0
self.continue_fail_count = 0
if config.PROXY_USER:
self.proxy = "%s://%s:%s@%s:%d" % \
(config.PROXY_TYPE, config.PROXY_USER, config.PROXY_PASSWD, config.PROXY_HOST, config.PROXY_PORT)
else:
self.proxy = "%s://%s:%d" % \
(config.PROXY_TYPE, config.PROXY_HOST, config.PROXY_PORT)
self.http_client = simple_http_client.Client(self.proxy, timeout=30)
def test_get_bulk(self):
timeout = 5
client = simple_http_client.Client(timeout=timeout)
url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/update_v4.txt"
start_time = time.time()
req = client.request("GET", url, read_payload=False)
self.assertIsNotNone(req)
tp = tempfile.gettempdir()
filename = os.path.join(tp, "v4.txt")
if req.chunked:
downloaded = 0
with open(filename, 'wb') as fp:
while True:
time_left = timeout - (time.time() - start_time)
if time_left < 0:
raise Exception("time out")
dat = req.read(timeout=time_left)
if not dat:
break
fp.write(dat)
downloaded += len(dat)
return True
else:
file_size = int(req.getheader(b'Content-Length', 0))
left = file_size
downloaded = 0
with open(filename, 'wb') as fp:
while True:
chunk_len = min(65536, left)
if not chunk_len:
break
chunk = req.read(chunk_len)
if not chunk:
break
fp.write(chunk)
downloaded += len(chunk)
left -= len(chunk)
def test_proxy(type, host, port, user, passwd):
if not host:
return False
if host == "127.0.0.1":
if port in [8087, 1080]:
xlog.warn("set LAN Proxy to %s:%d fail.", host, port)
return False
client = simple_http_client.Client(proxy={
"type": type,
"host": host,
"port": int(port),
"user": user if len(user) else None,
"pass": passwd if len(passwd) else None
},
timeout=5)
urls = [
"https://www.microsoft.com", "https://www.apple.com",
"https://code.jquery.com", "https://cdn.bootcss.com",
"https://cdnjs.cloudflare.com"
]
for url in urls:
header = {
"user-agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, sdch",
"accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
"connection": "keep-alive"
}
try:
response = client.request("HEAD", url, header, "")
if response:
return True
except Exception as e:
xlog.exception("test_proxy %s fail:%r", url, e)
pass
return False