def query(self, domain, dns_type=1, history=[]):
domain = utils.to_bytes(domain)
if utils.check_ip_valid(domain):
return [domain]
if not self.is_valid_hostname(domain):
xlog.warn("DNS query:%s not valid, type:%d", domain, dns_type)
return []
ips = g.domain_cache.get_ips(domain, dns_type)
if ips:
return ips
rule = g.user_rules.check_host(domain, 0)
if rule == "black":
# user define black list like advertisement or malware server.
ips = ["127.0.0.1"]
xlog.debug("DNS query:%s in black", domain)
return ips
elif b"." not in domain or g.gfwlist.in_white_list(domain):
ips = self.local_dns_resolve.query(domain, timeout=1)
g.domain_cache.set_ips(domain, ips, dns_type)
return ips
elif g.gfwlist.in_block_list(domain) or rule in [
"gae", "socks"
] or g.config.pac_policy == "all_X-Tunnel":
ips = self.query_blocked_domain(domain, dns_type)
else:
ips = self.query_unknown_domain(domain, dns_type)
if not ips:
ips = self.local_dns_resolve.query(domain, timeout=1)
out_ips = []
for ip in ips:
if not utils.check_ip_valid(ip):
if ip == domain:
continue
if ip in history:
continue
ip_ips = self.query(ip, dns_type, history.append(ip))
for ip in ip_ips:
out_ips.append(ip)
elif ip not in ips:
out_ips.append(ip)
return out_ips
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == "www.twitter.com":
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port, client_address, left_buf)
record = g.domain_cache.get(host)
if record:
rule = record["r"]
if rule == "gae":
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.gfwlist.check(host):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def direct_connect(self, host, port):
connect_timeout = 30
if ':' in host:
info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port,
0, 0))]
elif utils.check_ip_valid(host):
info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))]
else:
try:
info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
except socket.gaierror:
info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host,
port))]
for res in info:
af, socktype, proto, canonname, sa = res
s = None
try:
s = socket.socket(af, socktype, proto)
# See http://groups.google.com/group/cherrypy-users/
# browse_frm/thread/bbfe5eb39c904fe0
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
s.settimeout(connect_timeout)
s.connect((host, port))
return s
except socket.error:
if s:
s.close()
return None
def smart_route_dns_query(self):
xlog.info("Start testing SmartRouter DNS Query")
domain = "appsec.hicloud.com"
d = DNSRecord(DNSHeader(123))
d.add_question(DNSQuestion(domain, 1))
req4_pack = d.pack()
for port in [8053, 53]:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(req4_pack, ("127.0.0.1", port))
sock.settimeout(5)
try:
response, server = sock.recvfrom(8192)
except Exception as e:
xlog.warn("recv fail for port:%s e:%r", port, e)
continue
p = DNSRecord.parse(response)
for r in p.rr:
ip = utils.to_bytes(str(r.rdata))
xlog.info("IP:%s" % ip)
self.assertEqual(utils.check_ip_valid(ip), True)
xlog.info("Finished testing SmartRouter DNS Query")
return
def req_importip_handler(self):
req = urlparse.urlparse(self.path).query
reqs = urlparse.parse_qs(req, keep_blank_values=True)
data = ''
if reqs['cmd'] == ['importip']:
count = 0
ip_list = self.postvars['ipList'][0]
lines = ip_list.split("\n")
for line in lines:
addresses = line.split('|')
for ip in addresses:
ip = ip.strip()
if not utils.check_ip_valid(ip):
continue
if front.ip_manager.add_ip(ip, 100, "google.com", "gws"):
count += 1
data = '{"res":"%s"}' % count
front.ip_manager.save(force=True)
elif reqs['cmd'] == ['exportip']:
data = '{"res":"'
for ip in front.ip_manager.ip_list:
if front.ip_manager.ip_dict[ip]['fail_times'] > 0:
continue
data += "%s|" % ip
data = data[0: len(data) - 1]
data += '"}'
self.send_response_nc('text/html', data)
def query(self, domain, dns_type=1):
domain = utils.to_bytes(domain)
if utils.check_ip_valid(domain):
return [domain]
if not self.is_valid_hostname(domain):
xlog.warn("DNS query:%s not valid, type:%d", domain, dns_type)
return []
ips = g.domain_cache.get_ips(domain, dns_type)
if ips:
return ips
rule = g.user_rules.check_host(domain, 0)
if rule == "black":
# user define black list like advertisement or malware server.
ips = ["127.0.0.1"]
xlog.debug("DNS query:%s in black", domain)
return ips
elif b"." not in domain or g.gfwlist.in_white_list(domain):
ips = self.local_dns_resolve.query(domain, timeout=1)
g.domain_cache.set_ips(domain, ips, dns_type)
return ips
elif g.gfwlist.in_block_list(domain) or rule in ["gae", "socks"]:
ips = self.query_blocked_domain(domain, dns_type)
else:
ips = self.query_unknown_domain(domain, dns_type)
if not ips:
ips = self.local_dns_resolve.query(domain, timeout=1)
return ips
def direct_connect(self, host, port):
connect_timeout = 30
if ':' in host:
info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))]
elif utils.check_ip_valid(host):
info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))]
else:
try:
info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
except socket.gaierror:
info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))]
for res in info:
af, socktype, proto, canonname, sa = res
s = None
try:
s = socket.socket(af, socktype, proto)
# See http://groups.google.com/group/cherrypy-users/
# browse_frm/thread/bbfe5eb39c904fe0
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
s.settimeout(connect_timeout)
s.connect((host, port))
return s
except socket.error:
if s:
s.close()
return None
def get_cert(commonname, sans=None, full_name=False):
commonname = utils.to_bytes(commonname)
isip = check_ip_valid(commonname)
with ca_lock:
if not isip and not full_name and commonname.count(b'.') >= 2 and [
len(x) for x in reversed(commonname.split(b'.'))
] > [2, 4]:
commonname = commonname.partition(b'.')[-1]
certfile = os.path.join(ca_certdir, utils.to_str(commonname) + '.crt')
if os.path.exists(certfile):
return certfile
else:
commonname = utils.to_str(commonname)
cfile = os.path.join(ca_certdir, commonname + '.config')
rfile = os.path.join(ca_certdir, commonname + '.request')
config = open(cfile, 'w')
config.write(OPENSSL_CONFIG_TEMPLATE % {'commonname': commonname})
config.close()
openssl_c('req', '-new', '-key', cert_keyfile, '-out', rfile,
'-config', cfile)
openssl_c(
'x509', '-req', '-days', str(days), '-in', rfile, '-CA',
ca_cert, '-CAkey', ca_key, '-set_serial', '0x%s' % hashlib.md5(
(str(commonname) + str(
datetime.datetime.now())).encode("utf-8")).hexdigest(),
'-out', certfile, '-extensions', 'v3_req', '-extfile', cfile,
*X509_EXTRA_ARGS)
return certfile
def query(self, domain, type=None):
if utils.check_ip_valid(domain):
return [domain]
ips = g.domain_cache.get_ordered_ips(domain, type)
if ips:
return ips
if g.user_rules.check_host(domain, 0) == "direct" or \
(g.config.auto_direct and not g.gfwlist.check(domain)):
ips = g.dns_client.query(domain, timeout=1)
if not ips or not self.in_country(ips):
if "." in domain:
ips = remote_query_dns(domain, type)
g.domain_cache.set_ips(domain, ips, type)
if not ips or self.in_country(ips):
if g.config.auto_direct or g.user_rules.check_host(domain, 0) == "direct":
ips = g.dns_client.query(domain)
else:
ips = ["127.0.0.1|XX"]
g.domain_cache.set_ips(domain, ips, type)
return ips
def req_importip_handler(self):
req = urllib.parse.urlparse(self.path).query
reqs = urllib.parse.parse_qs(req, keep_blank_values=True)
data = ''
if reqs['cmd'] == ['importip']:
count = 0
ip_list = self.postvars['ipList'][0]
lines = ip_list.split("\n")
for line in lines:
addresses = line.split('|')
for ip in addresses:
ip = ip.strip()
if not utils.check_ip_valid(ip):
continue
if front.ip_manager.add_ip(ip, 100, "google.com", "gws"):
count += 1
data = '{"res":"%s"}' % count
front.ip_manager.save(force=True)
elif reqs['cmd'] == ['exportip']:
data = '{"res":"'
for ip in front.ip_manager.ip_list:
if front.ip_manager.ip_dict[ip]['fail_times'] > 0:
continue
data += "%s|" % ip
data = data[0:len(data) - 1]
data += '"}'
self.send_response_nc('text/html', data)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
global fake_host
if not fake_host and g.gae_proxy:
fake_host = g.gae_proxy.web_control.get_fake_host()
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == fake_host:
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port,
client_address, left_buf)
record = g.domain_cache.get(host)
ips = g.dns_srv.query(host)
if check_local_network.IPv6.is_ok() and have_ipv6(ips):
rule_list = ["direct", "gae", "socks", "redirect_https"]
elif record:
rule = record["r"]
if rule == "gae" or not g.ip_region.check_ips(record["ip"]):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.gfwlist.check(host):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
ips = g.dns_srv.query(host)
if g.ip_region.check_ips(ips):
rule_list = ["direct", "gae", "socks", "redirect_https"]
else:
rule_list = ["gae", "socks", "redirect_https", "direct"]
if not g.config.auto_direct and "direct" in rule_list:
try:
rule_list.remove("direct")
rule_list.remove("redirect_https")
except:
pass
if not g.config.auto_gae and "gae" in rule_list:
try:
rule_list.remove("gae")
except:
pass
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def is_valid_ip(args):
hosts = []
vips = []
if args.host != None:
hosts = args.host.split(",")
if args.vip != None:
vips = args.vip.split(",")
if len(hosts):
for h in hosts:
check_ip_valid(h)
if len(vips):
for v in vips:
check_ip_valid(v)
return True
def ucarpconf(self, groups):
local_clusterid = ""
dst_clusterid = ""
try:
cmd = "%s | grep 'cluster id' | awk -F':' '{print $2}'| awk '{print $1}'" % (
self.config.uss_mdstat)
out, err = exec_shell(cmd, need_return=True, p=False, timeout=30)
local_clusterid = out.split("\n")[0]
except Exception as e:
raise Exp(1, "get local clusterid fail!")
for group in groups:
hosts = group.split(":")[0].split(",")
vip_info = group.split(":")[1].strip()
vip = vip_info.split("/")[0]
master_vip = vip_info.split("/")[1]
master_vip_found = 0
check_ip_valid(vip)
for host in hosts:
try:
cmd = "%s | grep 'cluster id' | awk -F':' '{print $2}'| awk '{print $1}'" % (
self.config.uss_mdstat)
x, y = exec_remote(host, cmd)
dst_clusterid = x.split('\n')[0]
except Exception as e:
raise Exp(1, "get host %s cluster id fail!" % (host))
if dst_clusterid != local_clusterid:
raise Exp(errno.EINVAL,
"host %s not in cluster !" % (host))
if master_vip == host:
master_vip_found = 1
if master_vip_found == 0:
raise Exp(
errno.EINVAL,
"master vip %s not avaliable, please check it !" %
(master_vip))
self._ucarp_conf_group(hosts, vip, master_vip)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
global fake_host
if not fake_host and g.gae_proxy:
fake_host = g.gae_proxy.web_control.get_fake_host()
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == fake_host:
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port, client_address, left_buf)
record = g.domain_cache.get(host)
ips = g.dns_srv.query(host)
if check_local_network.IPv6.is_ok() and have_ipv6(ips):
rule_list = ["direct", "gae", "socks", "redirect_https"]
elif record:
rule = record["r"]
if rule == "gae" or not g.ip_region.check_ips(record["ip"]):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.gfwlist.check(host):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
ips = g.dns_srv.query(host)
if g.ip_region.check_ips(ips):
rule_list = ["direct", "gae", "socks", "redirect_https"]
else:
rule_list = ["gae", "socks", "redirect_https", "direct"]
if not g.config.auto_direct and "direct" in rule_list:
try:
rule_list.remove("direct")
rule_list.remove("redirect_https")
except:
pass
if not g.config.auto_gae and "gae" in rule_list:
try:
rule_list.remove("gae")
except:
pass
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
global fake_host
if not fake_host and g.gae_proxy:
fake_host = g.gae_proxy.web_control.get_fake_host()
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == fake_host:
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port,
client_address, left_buf)
if g.config.block_advertisement and g.gfwlist.is_advertisement(host):
xlog.info("block advertisement %s:%d", host, port)
sock.close()
return
#ips = g.dns_srv.query(host)
#if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443:
# rule_list = ["direct", "gae", "socks", "redirect_https"]
# gae is more faster then direct.
record = g.domain_cache.get(host)
if record and record["r"] != "unknown":
rule = record["r"]
if rule == "gae":
rule_list = ["gae", "socks", "redirect_https", "direct"]
elif rule == "socks":
rule_list = ["socks", "gae", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.gfwlist.is_white(host):
rule_list = ["direct", "gae", "socks", "redirect_https"]
elif g.gfwlist.check(host):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
ips = g.dns_srv.query(host)
if g.ip_region.check_ips(ips):
rule_list = ["direct", "socks", "redirect_https"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def query(self, domain, dns_type=1):
t0 = time.time()
try:
sock = self.get_connection()
if not sock:
xlog.warn("query_over_tcp %s type:%s connect fail.", domain,
dns_type)
return []
d = DNSRecord(DNSHeader())
d.add_question(DNSQuestion(domain, dns_type))
data = d.pack()
data = struct.pack("!H", len(data)) + data
sock.sendall(data)
response = sock.recv(8192)
if not response:
return []
length = struct.unpack("!H", bytes(response[:2]))[0]
while len(response) - 2 < length:
response += sock.recv(8192)
t2 = time.time()
p = DNSRecord.parse(response[2:])
if len(p.rr) == 0:
xlog.warn("query_over_tcp for %s type:%d return none, cost:%f",
domain, dns_type, t2 - t0)
ips = []
for r in p.rr:
ip = utils.to_bytes(str(r.rdata))
if not utils.check_ip_valid(ip):
if ip == domain:
continue
ip_ips = self.query(ip, dns_type)
ips += ip_ips
else:
ips.append(ip)
xlog.debug("DNS %s %s return %s t:%f", self.protocol, domain, ips,
t2 - t0)
self.connections.append([sock, time.time()])
return ips
except socket.timeout:
xlog.warn("query_over_tcp %s type:%s timeout", domain, dns_type)
return []
except Exception as e:
xlog.exception("query_over_tcp %s type:%s except:%r", domain,
dns_type, e)
return []
def query(self, domain, type=None):
if utils.check_ip_valid(domain):
return [domain]
ips = g.domain_cache.get_ordered_ips(domain, type)
if not ips:
if "." in domain:
ips = remote_query_dns(domain, type)
if not ips or self.in_country(ips):
ips = self.query_local_dns(domain)
g.domain_cache.set_ips(domain, ips, type)
return ips
def query(self, domain, dns_type=1):
t0 = time.time()
try:
client = self.get_connection()
url = self.server
d = DNSRecord(DNSHeader())
d.add_question(DNSQuestion(domain, dns_type))
data = d.pack()
r = client.request("POST",
url,
headers={
"accept": "application/dns-message",
"content-type": "application/dns-message"
},
body=data)
t2 = time.time()
ips = []
if not r:
xlog.warn("DNS s:%s query:%s fail t:%f", self.server, domain,
t2 - t0)
return ips
p = DNSRecord.parse(r.text)
self.connections.append([client, time.time()])
for r in p.rr:
ip = utils.to_bytes(str(r.rdata))
if not utils.check_ip_valid(ip):
if ip == domain:
continue
ip_ips = self.query(ip, dns_type)
ips += ip_ips
else:
ips.append(ip)
xlog.debug("DNS %s %s return %s t:%f", self.protocol, domain, ips,
t2 - t0)
return ips
except Exception as e:
t1 = time.time()
t = t1 - t0
xlog.exception("DnsOverHttpsQuery query %s cost:%f fail:%r",
domain, t, e)
return []
def get_cert(commonname, sans=None, full_name=False):
isip = check_ip_valid(commonname)
with CertUtil.ca_lock:
certfile = CertUtil._get_old_cert(commonname)
if certfile:
return certfile
# some site need full name cert
# like https://about.twitter.com in Google Chrome
if not isip and not full_name and commonname.count('.') >= 2 and [len(x) for x in reversed(commonname.split('.'))] > [2, 4]:
commonname = commonname.partition('.')[-1]
certfile = CertUtil._get_old_cert(commonname)
if certfile:
return certfile
return CertUtil._get_cert(commonname, isip, sans)
def query_recursively(self, domain, dns_type=None):
if not dns_type:
dns_types = [1, 28]
else:
dns_types = [dns_type]
ips_out = []
for dns_type in dns_types:
ips = self.query(domain, dns_type)
for ip in ips:
if utils.check_ip_valid(ip):
ips_out.append(ip)
else:
ips_s = self.query_recursively(ip, dns_type)
ips_out += ips_s
return ips_out
def smart_route_dns_query(self):
xlog.info("Start testing SmartRouter DNS Query")
domain = "appsec.hicloud.com"
d = DNSRecord(DNSHeader(123))
d.add_question(DNSQuestion(domain, 1))
req4_pack = d.pack()
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(req4_pack, ("127.0.0.1", 53))
sock.sendto(req4_pack, ("127.0.0.1", 8053))
response, server = sock.recvfrom(8192)
p = DNSRecord.parse(response)
for r in p.rr:
ip = utils.to_bytes(str(r.rdata))
xlog.info("IP:%s" % ip)
self.assertEqual(utils.check_ip_valid(ip), True)
xlog.info("Finished testing SmartRouter DNS Query")
def do_connect(self, host, port):
self.xlog.info("session_id:%s create_conn %d %s:%d",
self.session.session_id, self.conn_id, host, port)
connect_timeout = 30
sock = None
# start_time = time.time()
ip = ""
try:
if ':' in host:
# IPV6
ip = host
elif utils.check_ip_valid(host):
# IPV4
ip = host
else:
# xlog.debug("getting ip of %s", host)
ip = socket.gethostbyname(host)
# xlog.debug("resolve %s to %s", host, ip)
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(connect_timeout)
sock.connect((ip, port))
# record TCP connection time
# conn_time = time.time() - start_time
# xlog.debug("tcp conn %s %s time:%d", host, ip, conn_time * 1000)
return sock, True
except Exception as e:
# conn_time = int((time.time() - start_time) * 1000)
# xlog.debug("tcp conn host:%s %s:%d fail t:%d %r", host, ip, port, conn_time, e)
if sock:
sock.close()
return e, False
def query(self, domain, type=None):
if utils.check_ip_valid(domain):
return [domain]
if not is_valid_hostname(domain):
xlog.warn("DNS query:%s not valid, type:%d", domain, type)
return []
ips = g.domain_cache.get_ordered_ips(domain, type)
if ips:
return ips
rule = g.user_rules.check_host(domain, 0)
if rule == "black":
ips = ["127.0.0.1|XX"]
xlog.debug("DNS query:%s in black", domain)
return ips
if g.gfwlist.check(domain) or rule in ["gae", "socks"]:
ips = remote_query_dns(domain, type)
if not ips and g.config.auto_direct:
ips = g.dns_client.query_over_tcp(domain, type)
return ips
if "." not in domain:
ips = g.dns_client.query(domain, timeout=1, use_local=True)
if not ips:
ips = ["127.0.0.1|XX"]
g.domain_cache.set_ips(domain, ips, type)
return ips
# case: normal domain, not in black list
if g.config.auto_direct:
ips = g.dns_client.query(domain, timeout=1)
if not ips:
ips = g.dns_client.query(domain, timeout=1, use_local=True)
if not ips:
ips = remote_query_dns(domain, type)
return ips
def query(self, domain, type=None):
if utils.check_ip_valid(domain):
return [domain]
if not is_valid_hostname(domain):
xlog.warn("DNS query:%s not valid, type:%d", domain, type)
return []
ips = g.domain_cache.get_ordered_ips(domain, type)
if ips:
return ips
rule = g.user_rules.check_host(domain, 0)
if rule == "black":
ips = ["127.0.0.1|XX"]
xlog.debug("DNS query:%s in black", domain)
return ips
if g.gfwlist.check(domain) or rule in ["gae", "socks"]:
ips = remote_query_dns(domain, type)
if not ips and g.config.auto_direct:
ips = g.dns_client.query_over_tcp(domain, type)
return ips
if "." not in domain:
ips = g.dns_client.query(domain, timeout=1, use_local=True)
if not ips:
ips = ["127.0.0.1|XX"]
g.domain_cache.set_ips(domain, ips, type)
return ips
# case: normal domain, not in black list
if g.config.auto_direct:
ips = g.dns_client.query(domain, timeout=1)
if not ips:
ips = g.dns_client.query(domain, timeout=1, use_local=True)
if not ips:
ips = remote_query_dns(domain, type)
return ips
def do_connect(self, host, port):
self.xlog.info("session_id:%s create_conn %d %s:%d", self.session.session_id, self.conn_id, host, port)
connect_timeout = 30
sock = None
# start_time = time.time()
ip = ""
try:
if ":" in host:
# IPV6
ip = host
elif utils.check_ip_valid(host):
# IPV4
ip = host
else:
# xlog.debug("getting ip of %s", host)
ip = socket.gethostbyname(host)
# xlog.debug("resolve %s to %s", host, ip)
sock = socket.socket(socket.AF_INET if ":" not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(connect_timeout)
sock.connect((ip, port))
# record TCP connection time
# conn_time = time.time() - start_time
# xlog.debug("tcp conn %s %s time:%d", host, ip, conn_time * 1000)
return sock, True
except Exception as e:
# conn_time = int((time.time() - start_time) * 1000)
# xlog.debug("tcp conn host:%s %s:%d fail t:%d %r", host, ip, port, conn_time, e)
if sock:
sock.close()
return e, False
def handle_ip_proxy(sock, ip, port, client_address):
xlog.debug("handle_ip_proxy to %s:%d from:%s:%d", ip, port,
client_address[0], client_address[1])
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
if g.config.pac_policy == "all_X-Tunnel":
rule = "socks"
else:
rule = g.user_rules.check_host(ip, port)
if not rule:
if utils.is_private_ip(ip):
rule = "direct"
if rule:
return try_loop("ip user", [rule], sock, ip, port, client_address)
if port == 443:
try:
host = get_sni(sock)
if host and not utils.check_ip_valid(host):
xlog.debug("ip connect to %s:%d translate to %s", ip, port,
host)
return handle_domain_proxy(sock, host, port, client_address)
except SniNotExist as e:
xlog.debug("ip:%s:%d get sni fail", ip, port)
record = g.ip_cache.get(ip)
if record and record["r"] != "unknown":
rule = record["r"]
if rule == "gae":
rule_list = ["gae", "socks", "direct"]
elif rule == "socks":
rule_list = ["socks", "gae", "direct"]
else:
rule_list = ["direct", "gae", "socks"]
elif g.ip_region.check_ip(ip):
rule_list = ["direct", "socks"]
else:
rule_list = ["direct", "gae", "socks"]
if not g.config.auto_direct:
for rule in ["direct", "redirect_https"]:
try:
rule_list.remove(rule)
except:
pass
elif g.config.auto_direct6 and "direct" in rule_list:
rule_list.insert(rule_list.index("direct"), "direct6")
if not g.config.enable_fake_ca and port == 443 or not g.config.auto_gae:
try:
rule_list.remove("gae")
except:
pass
if g.config.pac_policy == "all_X-Tunnel":
rule_list = [
"socks",
]
try_loop("ip", rule_list, sock, ip, port, client_address)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
start_time = time.time()
rule = g.user_rules.check_host(host, port)
if not rule:
if host == "www.twitter.com":
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
if rule == "direct":
ips = g.dns_srv.query(host)
try:
do_direct(sock, host, ips, port, client_address, left_buf)
xlog.info("host:%s:%d user direct", host, port)
except ConnectFail:
xlog.warn("host:%s:%d user rule:%s connect fail", host, port,
rule)
sock.close()
return
elif rule == "redirect_https":
ips = g.dns_srv.query(host)
try:
do_redirect_https(sock, host, ips, port, client_address,
left_buf)
xlog.info("host:%s:%d user redirect_https", host, port)
except RedirectHttpsFail:
xlog.warn("host:%s:%d user rule:%s connect fail", host, port,
rule)
sock.close()
return
elif rule == "gae":
if not is_gae_workable():
xlog.debug("host:%s:%d user rule:%s, but gae not work", host,
port, rule)
sock.close()
return
try:
host = get_sni(sock)
do_gae(sock, host, port, client_address, left_buf)
xlog.info("host:%s:%d user gae", host, port)
except ssl.SSLError as e:
xlog.warn("host:%s:%d user rule gae, GetReqTimeout:%d e:%r",
host, port, (time.time() - start_time) * 1000, e)
sock.close()
except simple_http_server.GetReqTimeout as e:
# xlog.warn("host:%s:%d user rule gae, GetReqTimeout:%d e:%r", host, port, (time.time()-start_time)*1000, e)
sock.close()
except Exception as e:
xlog.warn("host:%s:%d user rule:%s except:%r", host, port,
rule, e)
sock.close()
return
elif rule == "socks":
do_socks(sock, host, port, client_address, left_buf)
xlog.info("host:%s:%d user rule:socks", host, port)
return
elif rule == "black":
xlog.info("host:%s:%d user rule black", host, port)
sock.close()
return
else:
xlog.error("get rule:%s unknown", rule)
sock.close()
return
record = g.domain_cache.get(host)
if not record:
rule = "direct"
else:
rule = record["r"]
if not rule or rule == "direct":
if g.config.auto_direct:
ips = g.dns_srv.query(host)
try:
if port == 80 and g.gfwlist.check(host):
do_redirect_https(sock, host, ips, port, client_address,
left_buf)
xlog.info("%s:%d redirect_https", host, port)
return
else:
do_direct(sock, host, ips, port, client_address, left_buf)
xlog.info("%s:%d direct", host, port)
return
except (ConnectFail, RedirectHttpsFail) as e:
xlog.debug("%s:%d try direct/redirect fail:%r", host, port, e)
rule = "gae"
else:
rule = "gae"
if rule == "gae":
if g.config.auto_gae and is_gae_workable(
) and g.domain_cache.accept_gae(host, port):
try:
sni_host = get_sni(sock)
do_gae(sock, host, port, client_address, left_buf)
xlog.info("%s:%d gae", host, port)
return
except SniNotExist:
xlog.debug("domain:%s get sni fail", host)
rule = "socks"
except (SslWrapFail, simple_http_server.ParseReqFail) as e:
xlog.warn("domain:%s sni:%s fail:%r", host, sni_host, e)
g.domain_cache.report_gae_deny(host, port)
sock.close()
return
except simple_http_server.GetReqTimeout:
# Happen sometimes, don't known why.
# xlog.warn("host:%s:%d try gae, GetReqTimeout:%d", host, port,
# (time.time() - start_time) * 1000)
sock.close()
return
except Exception as e:
xlog.warn("host:%s:%d cache rule:%s except:%r", host, port,
rule, e)
g.domain_cache.report_gae_deny(host, port)
sock.close()
return
else:
rule = "socks"
if rule == "socks":
do_socks(sock, host, port, client_address, left_buf)
xlog.info("%s:%d socks", host, port)
return
else:
xlog.error("domain:%s get rule:%s unknown", host, rule)
sock.close()
return
def check_all_ip(check_ip):
check = CheckAllIp(check_ip, "scan1.movistar.gq")
check.run()
if __name__ == "__main__":
# case 1: only ip
# case 2: ip + domain
# connect use domain
default_ip = "141.101.120.131"
host = "cloudflare.com"
if len(sys.argv) > 1:
ip = sys.argv[1]
if not utils.check_ip_valid(ip):
ip = default_ip
host = sys.argv[1]
else:
ip = default_ip
print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
xlog.info("test ip:%s", ip)
if len(sys.argv) > 2:
host = sys.argv[2]
xlog.info("host:%s", host)
if len(sys.argv) > 3:
wait_time = int(sys.argv[3])
else:
wait_time = 0
def query_over_tcp(self, domain, type=None, loop_count=0):
if loop_count > 10:
return []
if type is None:
types = [1, 28]
else:
types = [type]
ips = []
for t in types:
query_time = 0
for i in range(0, 3):
server_ip = self.dns_server.get_fastest_public()
if not server_ip:
return []
query_time += 1
if query_time > 3:
break
t0 = time.time()
try:
d = DNSRecord(DNSHeader())
d.add_question(DNSQuestion(domain, t))
a_pkt = d.send(server_ip, 53, tcp=True, timeout=2)
t1 = time.time()
self.dns_server.update_public_server(server_ip, t1-t0)
p = DNSRecord.parse(a_pkt)
if len(p.rr) == 0:
xlog.warn("query_over_tcp for %s type:%d server:%s return none, cost:%f",
domain, t, server_ip, t1-t0)
continue
for r in p.rr:
ip = bytes(r.rdata)
if utils.check_ip_valid(ip):
if b"." in ip and g.ip_region.check_ip(ip):
cn = g.ip_region.cn
else:
cn = b"XX"
if type == 1 and b"." not in ip:
continue
ips.append(ip + b"|" + cn)
else:
# It is domain, loop search it.
ipss = self.query_over_tcp(ip, type, loop_count+1)
if not ipss:
continue
ips += ipss
break
except Exception as e:
t1 = time.time()
self.dns_server.update_public_server(server_ip, t1 - t0)
xlog.warn("query_over_tcp %s type:%s server:%s except:%r", domain, type, server_ip,e)
if ips:
g.domain_cache.set_ips(domain, ips, type)
return ips
# format: [ip] [domain [sni] ]
# case 1: only ip
# case 2: ip domain
# case 3: ip domain sni
# case 4: domain
# case 5: domain sni
ip = "141.101.120.131"
host = "xx-net.net"
sni = host
args = list(sys.argv[1:])
if len(args):
if utils.check_ip_valid(args[0]):
ip = args.pop(0)
if len(args):
host = args.pop(0)
sni = host
if len(args):
sni = args.pop(0)
# print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
xlog.info("test ip:%s", ip)
xlog.info("host:%s", host)
xlog.info("sni:%s", sni)
wait_time = 0
def check_all_ip(check_ip):
check = CheckAllIp(check_ip, "scan1.movistar.gq")
check.run()
if __name__ == "__main__":
# case 1: only ip
# case 2: ip + domain
# connect use domain
default_ip = "141.101.120.131"
host = "xx-net.net"
if len(sys.argv) > 1:
ip = sys.argv[1]
if not utils.check_ip_valid(ip):
ip = default_ip
host = sys.argv[1]
else:
ip = default_ip
print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
xlog.info("test ip:%s", ip)
if len(sys.argv) > 2:
host = sys.argv[2]
xlog.info("host:%s", host)
if len(sys.argv) > 3:
wait_time = int(sys.argv[3])
else:
wait_time = 0
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
global fake_host
if not fake_host and g.gae_proxy:
fake_host = g.gae_proxy.web_control.get_fake_host()
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == fake_host:
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port, client_address, left_buf)
if g.config.block_advertisement and g.gfwlist.is_advertisement(host):
xlog.info("block advertisement %s:%d", host, port)
sock.close()
return
#ips = g.dns_srv.query(host)
#if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443:
# rule_list = ["direct", "gae", "socks", "redirect_https"]
# gae is more faster then direct.
record = g.domain_cache.get(host)
if record and record["r"] != "unknown":
rule = record["r"]
if rule == "gae":
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.gfwlist.is_white(host):
rule_list = ["direct", "gae", "socks", "redirect_https"]
elif g.gfwlist.check(host):
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
ips = g.dns_srv.query(host)
if g.ip_region.check_ips(ips):
rule_list = ["direct", "gae", "socks", "redirect_https"]
else:
rule_list = ["gae", "socks", "redirect_https", "direct"]
if not g.config.auto_direct and "direct" in rule_list:
try:
rule_list.remove("direct")
rule_list.remove("redirect_https")
except:
pass
if not g.config.enable_fake_ca and port == 443 and "gae" in rule_list:
try:
rule_list.remove("gae")
except:
pass
if not g.config.auto_gae and "gae" in rule_list:
try:
rule_list.remove("gae")
except:
pass
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
check.run()
if __name__ == "__main__":
# case 1: only ip
# case 2: ip + domain
# connect use domain
ip = "141.101.120.131"
host = "xx-net.net"
sni = host
args = list(sys.argv[1:])
if len(args):
if utils.check_ip_valid(args[0]):
ip = args.pop(0)
if len(args):
host = args.pop(0)
sni = host
if len(args):
sni = args.pop(0)
# print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
xlog.info("test ip:%s", ip)
xlog.info("host:%s", host)
xlog.info("sni:%s", sni)
wait_time = 0
def get_cert(commonname, sans=None, full_name=False):
commonname = utils.to_bytes(commonname)
isip = check_ip_valid(commonname)
sr = '0x%s' % hashlib.md5(
(str(commonname) +
str(datetime.datetime.now())).encode("utf-8")).hexdigest()
with ca_lock:
if not isip and not full_name and commonname.count(b'.') >= 2 and [
len(x) for x in reversed(commonname.split(b'.'))
] > [2, 4]:
commonname = commonname.partition(b'.')[-1]
certfile = os.path.join(ca_certdir, utils.to_str(commonname) + '.crt')
if os.path.exists(certfile):
return certfile
else:
commonname = utils.to_str(commonname)
if (platform.system() == 'Windows'):
X509_EXTRA_ARGS = ()
cfile = os.path.join(ca_certdir, commonname + '.config')
rfile = os.path.join(ca_certdir, commonname + '.request')
config = open(cfile, 'w')
config.write(OPENSSL_CONFIG_TEMPLATE %
{'commonname': commonname})
config.close()
openssl_c('req', '-new', '-key', cert_keyfile, '-out', rfile,
'-config', cfile)
openssl_c('x509', '-req', '-days', str(days), '-in', rfile,
'-CA', ca_cert, '-CAkey', ca_key, '-set_serial', sr,
'-out', certfile, '-extensions', 'v3_req',
'-extfile', cfile, *X509_EXTRA_ARGS)
return certfile
elif (platform.system() == 'Linux'):
script1 = '''
CONFIG="
prompt = no
distinguished_name = req_distinguished_name
req_extensions = v3_req
[ req_distinguished_name ]
C = CN
ST = Internet
L = Cernet
O = GoAgent-XX-Net-mini-4.5.2
OU = GoAgent-XX-Net-mini-4.5.2 Root
CN = GoAgent-XX-Net-mini-4.5.2 CA
emailAddress = [email protected]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[ alt_names ]
DNS.1 = %(commonname)s
DNS.2 = *.%(commonname)s
"
'''
script1 = script1 % {'commonname': commonname}
script2 = "openssl req -new -key " + cert_keyfile + ''' -config <(echo "$CONFIG")'''
script = script1 + script2
p = subprocess.Popen(['bash', '-c', script],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
os.environ['request'] = stdout.decode()
script3 = '''openssl x509 -req -days 3650 -in <(echo "$request") -CA ''' + ca_cert + ''' -CAkey ''' + ca_key + ''' -set_serial ''' + sr + ''' -extensions v3_req -extfile <(echo "$CONFIG")'''
script4 = script1 + script3
p = subprocess.Popen(['bash', '-c', script4],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
with open(certfile, 'wb') as fp:
fp.write(stdout)
return certfile
else:
pass
def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
global fake_host
if not fake_host and g.gae_proxy:
fake_host = g.gae_proxy.web_control.get_fake_host()
if not isinstance(sock, SocketWrap):
sock = SocketWrap(sock, client_address[0], client_address[1])
# Check user rules
sock.target = "%s:%d" % (host, port)
rule = g.user_rules.check_host(host, port)
if not rule:
if host == fake_host:
rule = "gae"
elif utils.check_ip_valid(host) and utils.is_private_ip(host):
rule = "direct"
if rule:
return try_loop("domain user", [rule], sock, host, port,
client_address, left_buf)
if g.config.block_advertisement and g.gfwlist.is_advertisement(host):
xlog.info("block advertisement %s:%d", host, port)
sock.close()
return
#ips = g.dns_query.query(host)
#if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443:
# rule_list = ["direct", "gae", "socks", "redirect_https"]
# gae is more faster then direct.
rule = g.domain_cache.get_rule(host)
if rule != "unknown":
if rule == "gae":
rule_list = ["gae", "socks", "redirect_https", "direct"]
elif rule == "socks":
rule_list = ["socks", "gae", "redirect_https", "direct"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
if not g.domain_cache.accept_gae(host):
rule_list.remove("gae")
elif g.config.country_code == "CN":
if g.gfwlist.in_white_list(host):
rule_list = ["direct", "gae", "socks", "redirect_https"]
elif g.gfwlist.in_block_list(host):
if g.config.pac_policy == "black_X-Tunnel":
rule_list = ["socks", "redirect_https", "direct", "gae"]
else:
rule_list = ["gae", "socks", "redirect_https", "direct"]
else:
ips = g.dns_query.query_recursively(host, 1)
if g.ip_region.check_ips(ips):
rule_list = ["direct", "socks", "redirect_https"]
else:
rule_list = ["direct", "gae", "socks", "redirect_https"]
else:
rule_list = ["direct", "socks", "gae", "redirect_https"]
# check config.
if not g.config.auto_direct:
for rule in ["direct", "redirect_https"]:
try:
rule_list.remove(rule)
except:
pass
elif g.config.auto_direct6 and "direct" in rule_list:
rule_list.insert(rule_list.index("direct"), "direct6")
if not g.config.enable_fake_ca and port == 443 or not g.config.auto_gae:
try:
rule_list.remove("gae")
except:
pass
xlog.debug("connect to %s:%d from:%s:%d, rule:%s", host, port,
client_address[0], client_address[1], utils.to_str(rule_list))
try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
