def query(self, domain, dns_type=1, history=[]): domain = utils.to_bytes(domain) if utils.check_ip_valid(domain): return [domain] if not self.is_valid_hostname(domain): xlog.warn("DNS query:%s not valid, type:%d", domain, dns_type) return [] ips = g.domain_cache.get_ips(domain, dns_type) if ips: return ips rule = g.user_rules.check_host(domain, 0) if rule == "black": # user define black list like advertisement or malware server. ips = ["127.0.0.1"] xlog.debug("DNS query:%s in black", domain) return ips elif b"." not in domain or g.gfwlist.in_white_list(domain): ips = self.local_dns_resolve.query(domain, timeout=1) g.domain_cache.set_ips(domain, ips, dns_type) return ips elif g.gfwlist.in_block_list(domain) or rule in [ "gae", "socks" ] or g.config.pac_policy == "all_X-Tunnel": ips = self.query_blocked_domain(domain, dns_type) else: ips = self.query_unknown_domain(domain, dns_type) if not ips: ips = self.local_dns_resolve.query(domain, timeout=1) out_ips = [] for ip in ips: if not utils.check_ip_valid(ip): if ip == domain: continue if ip in history: continue ip_ips = self.query(ip, dns_type, history.append(ip)) for ip in ip_ips: out_ips.append(ip) elif ip not in ips: out_ips.append(ip) return out_ips
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == "www.twitter.com": rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) record = g.domain_cache.get(host) if record: rule = record["r"] if rule == "gae": rule_list = ["gae", "socks", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.gfwlist.check(host): rule_list = ["gae", "socks", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def direct_connect(self, host, port): connect_timeout = 30 if ':' in host: info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))] elif utils.check_ip_valid(host): info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] else: try: info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) except socket.gaierror: info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] for res in info: af, socktype, proto, canonname, sa = res s = None try: s = socket.socket(af, socktype, proto) # See http://groups.google.com/group/cherrypy-users/ # browse_frm/thread/bbfe5eb39c904fe0 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) s.settimeout(connect_timeout) s.connect((host, port)) return s except socket.error: if s: s.close() return None
def smart_route_dns_query(self): xlog.info("Start testing SmartRouter DNS Query") domain = "appsec.hicloud.com" d = DNSRecord(DNSHeader(123)) d.add_question(DNSQuestion(domain, 1)) req4_pack = d.pack() for port in [8053, 53]: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(req4_pack, ("127.0.0.1", port)) sock.settimeout(5) try: response, server = sock.recvfrom(8192) except Exception as e: xlog.warn("recv fail for port:%s e:%r", port, e) continue p = DNSRecord.parse(response) for r in p.rr: ip = utils.to_bytes(str(r.rdata)) xlog.info("IP:%s" % ip) self.assertEqual(utils.check_ip_valid(ip), True) xlog.info("Finished testing SmartRouter DNS Query") return
def req_importip_handler(self): req = urlparse.urlparse(self.path).query reqs = urlparse.parse_qs(req, keep_blank_values=True) data = '' if reqs['cmd'] == ['importip']: count = 0 ip_list = self.postvars['ipList'][0] lines = ip_list.split("\n") for line in lines: addresses = line.split('|') for ip in addresses: ip = ip.strip() if not utils.check_ip_valid(ip): continue if front.ip_manager.add_ip(ip, 100, "google.com", "gws"): count += 1 data = '{"res":"%s"}' % count front.ip_manager.save(force=True) elif reqs['cmd'] == ['exportip']: data = '{"res":"' for ip in front.ip_manager.ip_list: if front.ip_manager.ip_dict[ip]['fail_times'] > 0: continue data += "%s|" % ip data = data[0: len(data) - 1] data += '"}' self.send_response_nc('text/html', data)
def query(self, domain, dns_type=1): domain = utils.to_bytes(domain) if utils.check_ip_valid(domain): return [domain] if not self.is_valid_hostname(domain): xlog.warn("DNS query:%s not valid, type:%d", domain, dns_type) return [] ips = g.domain_cache.get_ips(domain, dns_type) if ips: return ips rule = g.user_rules.check_host(domain, 0) if rule == "black": # user define black list like advertisement or malware server. ips = ["127.0.0.1"] xlog.debug("DNS query:%s in black", domain) return ips elif b"." not in domain or g.gfwlist.in_white_list(domain): ips = self.local_dns_resolve.query(domain, timeout=1) g.domain_cache.set_ips(domain, ips, dns_type) return ips elif g.gfwlist.in_block_list(domain) or rule in ["gae", "socks"]: ips = self.query_blocked_domain(domain, dns_type) else: ips = self.query_unknown_domain(domain, dns_type) if not ips: ips = self.local_dns_resolve.query(domain, timeout=1) return ips
def direct_connect(self, host, port): connect_timeout = 30 if ':' in host: info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))] elif utils.check_ip_valid(host): info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] else: try: info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) except socket.gaierror: info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] for res in info: af, socktype, proto, canonname, sa = res s = None try: s = socket.socket(af, socktype, proto) # See http://groups.google.com/group/cherrypy-users/ # browse_frm/thread/bbfe5eb39c904fe0 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) s.settimeout(connect_timeout) s.connect((host, port)) return s except socket.error: if s: s.close() return None
def get_cert(commonname, sans=None, full_name=False): commonname = utils.to_bytes(commonname) isip = check_ip_valid(commonname) with ca_lock: if not isip and not full_name and commonname.count(b'.') >= 2 and [ len(x) for x in reversed(commonname.split(b'.')) ] > [2, 4]: commonname = commonname.partition(b'.')[-1] certfile = os.path.join(ca_certdir, utils.to_str(commonname) + '.crt') if os.path.exists(certfile): return certfile else: commonname = utils.to_str(commonname) cfile = os.path.join(ca_certdir, commonname + '.config') rfile = os.path.join(ca_certdir, commonname + '.request') config = open(cfile, 'w') config.write(OPENSSL_CONFIG_TEMPLATE % {'commonname': commonname}) config.close() openssl_c('req', '-new', '-key', cert_keyfile, '-out', rfile, '-config', cfile) openssl_c( 'x509', '-req', '-days', str(days), '-in', rfile, '-CA', ca_cert, '-CAkey', ca_key, '-set_serial', '0x%s' % hashlib.md5( (str(commonname) + str( datetime.datetime.now())).encode("utf-8")).hexdigest(), '-out', certfile, '-extensions', 'v3_req', '-extfile', cfile, *X509_EXTRA_ARGS) return certfile
def query(self, domain, type=None): if utils.check_ip_valid(domain): return [domain] ips = g.domain_cache.get_ordered_ips(domain, type) if ips: return ips if g.user_rules.check_host(domain, 0) == "direct" or \ (g.config.auto_direct and not g.gfwlist.check(domain)): ips = g.dns_client.query(domain, timeout=1) if not ips or not self.in_country(ips): if "." in domain: ips = remote_query_dns(domain, type) g.domain_cache.set_ips(domain, ips, type) if not ips or self.in_country(ips): if g.config.auto_direct or g.user_rules.check_host(domain, 0) == "direct": ips = g.dns_client.query(domain) else: ips = ["127.0.0.1|XX"] g.domain_cache.set_ips(domain, ips, type) return ips
def req_importip_handler(self): req = urllib.parse.urlparse(self.path).query reqs = urllib.parse.parse_qs(req, keep_blank_values=True) data = '' if reqs['cmd'] == ['importip']: count = 0 ip_list = self.postvars['ipList'][0] lines = ip_list.split("\n") for line in lines: addresses = line.split('|') for ip in addresses: ip = ip.strip() if not utils.check_ip_valid(ip): continue if front.ip_manager.add_ip(ip, 100, "google.com", "gws"): count += 1 data = '{"res":"%s"}' % count front.ip_manager.save(force=True) elif reqs['cmd'] == ['exportip']: data = '{"res":"' for ip in front.ip_manager.ip_list: if front.ip_manager.ip_dict[ip]['fail_times'] > 0: continue data += "%s|" % ip data = data[0:len(data) - 1] data += '"}' self.send_response_nc('text/html', data)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): global fake_host if not fake_host and g.gae_proxy: fake_host = g.gae_proxy.web_control.get_fake_host() if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == fake_host: rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) record = g.domain_cache.get(host) ips = g.dns_srv.query(host) if check_local_network.IPv6.is_ok() and have_ipv6(ips): rule_list = ["direct", "gae", "socks", "redirect_https"] elif record: rule = record["r"] if rule == "gae" or not g.ip_region.check_ips(record["ip"]): rule_list = ["gae", "socks", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.gfwlist.check(host): rule_list = ["gae", "socks", "redirect_https", "direct"] else: ips = g.dns_srv.query(host) if g.ip_region.check_ips(ips): rule_list = ["direct", "gae", "socks", "redirect_https"] else: rule_list = ["gae", "socks", "redirect_https", "direct"] if not g.config.auto_direct and "direct" in rule_list: try: rule_list.remove("direct") rule_list.remove("redirect_https") except: pass if not g.config.auto_gae and "gae" in rule_list: try: rule_list.remove("gae") except: pass try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def is_valid_ip(args): hosts = [] vips = [] if args.host != None: hosts = args.host.split(",") if args.vip != None: vips = args.vip.split(",") if len(hosts): for h in hosts: check_ip_valid(h) if len(vips): for v in vips: check_ip_valid(v) return True
def ucarpconf(self, groups): local_clusterid = "" dst_clusterid = "" try: cmd = "%s | grep 'cluster id' | awk -F':' '{print $2}'| awk '{print $1}'" % ( self.config.uss_mdstat) out, err = exec_shell(cmd, need_return=True, p=False, timeout=30) local_clusterid = out.split("\n")[0] except Exception as e: raise Exp(1, "get local clusterid fail!") for group in groups: hosts = group.split(":")[0].split(",") vip_info = group.split(":")[1].strip() vip = vip_info.split("/")[0] master_vip = vip_info.split("/")[1] master_vip_found = 0 check_ip_valid(vip) for host in hosts: try: cmd = "%s | grep 'cluster id' | awk -F':' '{print $2}'| awk '{print $1}'" % ( self.config.uss_mdstat) x, y = exec_remote(host, cmd) dst_clusterid = x.split('\n')[0] except Exception as e: raise Exp(1, "get host %s cluster id fail!" % (host)) if dst_clusterid != local_clusterid: raise Exp(errno.EINVAL, "host %s not in cluster !" % (host)) if master_vip == host: master_vip_found = 1 if master_vip_found == 0: raise Exp( errno.EINVAL, "master vip %s not avaliable, please check it !" % (master_vip)) self._ucarp_conf_group(hosts, vip, master_vip)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): global fake_host if not fake_host and g.gae_proxy: fake_host = g.gae_proxy.web_control.get_fake_host() if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == fake_host: rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) record = g.domain_cache.get(host) ips = g.dns_srv.query(host) if check_local_network.IPv6.is_ok() and have_ipv6(ips): rule_list = ["direct", "gae", "socks", "redirect_https"] elif record: rule = record["r"] if rule == "gae" or not g.ip_region.check_ips(record["ip"]): rule_list = ["gae", "socks", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.gfwlist.check(host): rule_list = ["gae", "socks", "redirect_https", "direct"] else: ips = g.dns_srv.query(host) if g.ip_region.check_ips(ips): rule_list = ["direct", "gae", "socks", "redirect_https"] else: rule_list = ["gae", "socks", "redirect_https", "direct"] if not g.config.auto_direct and "direct" in rule_list: try: rule_list.remove("direct") rule_list.remove("redirect_https") except: pass if not g.config.auto_gae and "gae" in rule_list: try: rule_list.remove("gae") except: pass try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): global fake_host if not fake_host and g.gae_proxy: fake_host = g.gae_proxy.web_control.get_fake_host() if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == fake_host: rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) if g.config.block_advertisement and g.gfwlist.is_advertisement(host): xlog.info("block advertisement %s:%d", host, port) sock.close() return #ips = g.dns_srv.query(host) #if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443: # rule_list = ["direct", "gae", "socks", "redirect_https"] # gae is more faster then direct. record = g.domain_cache.get(host) if record and record["r"] != "unknown": rule = record["r"] if rule == "gae": rule_list = ["gae", "socks", "redirect_https", "direct"] elif rule == "socks": rule_list = ["socks", "gae", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.gfwlist.is_white(host): rule_list = ["direct", "gae", "socks", "redirect_https"] elif g.gfwlist.check(host): rule_list = ["gae", "socks", "redirect_https", "direct"] else: ips = g.dns_srv.query(host) if g.ip_region.check_ips(ips): rule_list = ["direct", "socks", "redirect_https"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
def query(self, domain, dns_type=1): t0 = time.time() try: sock = self.get_connection() if not sock: xlog.warn("query_over_tcp %s type:%s connect fail.", domain, dns_type) return [] d = DNSRecord(DNSHeader()) d.add_question(DNSQuestion(domain, dns_type)) data = d.pack() data = struct.pack("!H", len(data)) + data sock.sendall(data) response = sock.recv(8192) if not response: return [] length = struct.unpack("!H", bytes(response[:2]))[0] while len(response) - 2 < length: response += sock.recv(8192) t2 = time.time() p = DNSRecord.parse(response[2:]) if len(p.rr) == 0: xlog.warn("query_over_tcp for %s type:%d return none, cost:%f", domain, dns_type, t2 - t0) ips = [] for r in p.rr: ip = utils.to_bytes(str(r.rdata)) if not utils.check_ip_valid(ip): if ip == domain: continue ip_ips = self.query(ip, dns_type) ips += ip_ips else: ips.append(ip) xlog.debug("DNS %s %s return %s t:%f", self.protocol, domain, ips, t2 - t0) self.connections.append([sock, time.time()]) return ips except socket.timeout: xlog.warn("query_over_tcp %s type:%s timeout", domain, dns_type) return [] except Exception as e: xlog.exception("query_over_tcp %s type:%s except:%r", domain, dns_type, e) return []
def query(self, domain, type=None): if utils.check_ip_valid(domain): return [domain] ips = g.domain_cache.get_ordered_ips(domain, type) if not ips: if "." in domain: ips = remote_query_dns(domain, type) if not ips or self.in_country(ips): ips = self.query_local_dns(domain) g.domain_cache.set_ips(domain, ips, type) return ips
def query(self, domain, dns_type=1): t0 = time.time() try: client = self.get_connection() url = self.server d = DNSRecord(DNSHeader()) d.add_question(DNSQuestion(domain, dns_type)) data = d.pack() r = client.request("POST", url, headers={ "accept": "application/dns-message", "content-type": "application/dns-message" }, body=data) t2 = time.time() ips = [] if not r: xlog.warn("DNS s:%s query:%s fail t:%f", self.server, domain, t2 - t0) return ips p = DNSRecord.parse(r.text) self.connections.append([client, time.time()]) for r in p.rr: ip = utils.to_bytes(str(r.rdata)) if not utils.check_ip_valid(ip): if ip == domain: continue ip_ips = self.query(ip, dns_type) ips += ip_ips else: ips.append(ip) xlog.debug("DNS %s %s return %s t:%f", self.protocol, domain, ips, t2 - t0) return ips except Exception as e: t1 = time.time() t = t1 - t0 xlog.exception("DnsOverHttpsQuery query %s cost:%f fail:%r", domain, t, e) return []
def get_cert(commonname, sans=None, full_name=False): isip = check_ip_valid(commonname) with CertUtil.ca_lock: certfile = CertUtil._get_old_cert(commonname) if certfile: return certfile # some site need full name cert # like https://about.twitter.com in Google Chrome if not isip and not full_name and commonname.count('.') >= 2 and [len(x) for x in reversed(commonname.split('.'))] > [2, 4]: commonname = commonname.partition('.')[-1] certfile = CertUtil._get_old_cert(commonname) if certfile: return certfile return CertUtil._get_cert(commonname, isip, sans)
def query_recursively(self, domain, dns_type=None): if not dns_type: dns_types = [1, 28] else: dns_types = [dns_type] ips_out = [] for dns_type in dns_types: ips = self.query(domain, dns_type) for ip in ips: if utils.check_ip_valid(ip): ips_out.append(ip) else: ips_s = self.query_recursively(ip, dns_type) ips_out += ips_s return ips_out
def smart_route_dns_query(self): xlog.info("Start testing SmartRouter DNS Query") domain = "appsec.hicloud.com" d = DNSRecord(DNSHeader(123)) d.add_question(DNSQuestion(domain, 1)) req4_pack = d.pack() sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(req4_pack, ("127.0.0.1", 53)) sock.sendto(req4_pack, ("127.0.0.1", 8053)) response, server = sock.recvfrom(8192) p = DNSRecord.parse(response) for r in p.rr: ip = utils.to_bytes(str(r.rdata)) xlog.info("IP:%s" % ip) self.assertEqual(utils.check_ip_valid(ip), True) xlog.info("Finished testing SmartRouter DNS Query")
def do_connect(self, host, port): self.xlog.info("session_id:%s create_conn %d %s:%d", self.session.session_id, self.conn_id, host, port) connect_timeout = 30 sock = None # start_time = time.time() ip = "" try: if ':' in host: # IPV6 ip = host elif utils.check_ip_valid(host): # IPV4 ip = host else: # xlog.debug("getting ip of %s", host) ip = socket.gethostbyname(host) # xlog.debug("resolve %s to %s", host, ip) sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(connect_timeout) sock.connect((ip, port)) # record TCP connection time # conn_time = time.time() - start_time # xlog.debug("tcp conn %s %s time:%d", host, ip, conn_time * 1000) return sock, True except Exception as e: # conn_time = int((time.time() - start_time) * 1000) # xlog.debug("tcp conn host:%s %s:%d fail t:%d %r", host, ip, port, conn_time, e) if sock: sock.close() return e, False
def query(self, domain, type=None): if utils.check_ip_valid(domain): return [domain] if not is_valid_hostname(domain): xlog.warn("DNS query:%s not valid, type:%d", domain, type) return [] ips = g.domain_cache.get_ordered_ips(domain, type) if ips: return ips rule = g.user_rules.check_host(domain, 0) if rule == "black": ips = ["127.0.0.1|XX"] xlog.debug("DNS query:%s in black", domain) return ips if g.gfwlist.check(domain) or rule in ["gae", "socks"]: ips = remote_query_dns(domain, type) if not ips and g.config.auto_direct: ips = g.dns_client.query_over_tcp(domain, type) return ips if "." not in domain: ips = g.dns_client.query(domain, timeout=1, use_local=True) if not ips: ips = ["127.0.0.1|XX"] g.domain_cache.set_ips(domain, ips, type) return ips # case: normal domain, not in black list if g.config.auto_direct: ips = g.dns_client.query(domain, timeout=1) if not ips: ips = g.dns_client.query(domain, timeout=1, use_local=True) if not ips: ips = remote_query_dns(domain, type) return ips
def query(self, domain, type=None): if utils.check_ip_valid(domain): return [domain] if not is_valid_hostname(domain): xlog.warn("DNS query:%s not valid, type:%d", domain, type) return [] ips = g.domain_cache.get_ordered_ips(domain, type) if ips: return ips rule = g.user_rules.check_host(domain, 0) if rule == "black": ips = ["127.0.0.1|XX"] xlog.debug("DNS query:%s in black", domain) return ips if g.gfwlist.check(domain) or rule in ["gae", "socks"]: ips = remote_query_dns(domain, type) if not ips and g.config.auto_direct: ips = g.dns_client.query_over_tcp(domain, type) return ips if "." not in domain: ips = g.dns_client.query(domain, timeout=1, use_local=True) if not ips: ips = ["127.0.0.1|XX"] g.domain_cache.set_ips(domain, ips, type) return ips # case: normal domain, not in black list if g.config.auto_direct: ips = g.dns_client.query(domain, timeout=1) if not ips: ips = g.dns_client.query(domain, timeout=1, use_local=True) if not ips: ips = remote_query_dns(domain, type) return ips
def do_connect(self, host, port): self.xlog.info("session_id:%s create_conn %d %s:%d", self.session.session_id, self.conn_id, host, port) connect_timeout = 30 sock = None # start_time = time.time() ip = "" try: if ":" in host: # IPV6 ip = host elif utils.check_ip_valid(host): # IPV4 ip = host else: # xlog.debug("getting ip of %s", host) ip = socket.gethostbyname(host) # xlog.debug("resolve %s to %s", host, ip) sock = socket.socket(socket.AF_INET if ":" not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(connect_timeout) sock.connect((ip, port)) # record TCP connection time # conn_time = time.time() - start_time # xlog.debug("tcp conn %s %s time:%d", host, ip, conn_time * 1000) return sock, True except Exception as e: # conn_time = int((time.time() - start_time) * 1000) # xlog.debug("tcp conn host:%s %s:%d fail t:%d %r", host, ip, port, conn_time, e) if sock: sock.close() return e, False
def handle_ip_proxy(sock, ip, port, client_address): xlog.debug("handle_ip_proxy to %s:%d from:%s:%d", ip, port, client_address[0], client_address[1]) if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) if g.config.pac_policy == "all_X-Tunnel": rule = "socks" else: rule = g.user_rules.check_host(ip, port) if not rule: if utils.is_private_ip(ip): rule = "direct" if rule: return try_loop("ip user", [rule], sock, ip, port, client_address) if port == 443: try: host = get_sni(sock) if host and not utils.check_ip_valid(host): xlog.debug("ip connect to %s:%d translate to %s", ip, port, host) return handle_domain_proxy(sock, host, port, client_address) except SniNotExist as e: xlog.debug("ip:%s:%d get sni fail", ip, port) record = g.ip_cache.get(ip) if record and record["r"] != "unknown": rule = record["r"] if rule == "gae": rule_list = ["gae", "socks", "direct"] elif rule == "socks": rule_list = ["socks", "gae", "direct"] else: rule_list = ["direct", "gae", "socks"] elif g.ip_region.check_ip(ip): rule_list = ["direct", "socks"] else: rule_list = ["direct", "gae", "socks"] if not g.config.auto_direct: for rule in ["direct", "redirect_https"]: try: rule_list.remove(rule) except: pass elif g.config.auto_direct6 and "direct" in rule_list: rule_list.insert(rule_list.index("direct"), "direct6") if not g.config.enable_fake_ca and port == 443 or not g.config.auto_gae: try: rule_list.remove("gae") except: pass if g.config.pac_policy == "all_X-Tunnel": rule_list = [ "socks", ] try_loop("ip", rule_list, sock, ip, port, client_address)
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) start_time = time.time() rule = g.user_rules.check_host(host, port) if not rule: if host == "www.twitter.com": rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: if rule == "direct": ips = g.dns_srv.query(host) try: do_direct(sock, host, ips, port, client_address, left_buf) xlog.info("host:%s:%d user direct", host, port) except ConnectFail: xlog.warn("host:%s:%d user rule:%s connect fail", host, port, rule) sock.close() return elif rule == "redirect_https": ips = g.dns_srv.query(host) try: do_redirect_https(sock, host, ips, port, client_address, left_buf) xlog.info("host:%s:%d user redirect_https", host, port) except RedirectHttpsFail: xlog.warn("host:%s:%d user rule:%s connect fail", host, port, rule) sock.close() return elif rule == "gae": if not is_gae_workable(): xlog.debug("host:%s:%d user rule:%s, but gae not work", host, port, rule) sock.close() return try: host = get_sni(sock) do_gae(sock, host, port, client_address, left_buf) xlog.info("host:%s:%d user gae", host, port) except ssl.SSLError as e: xlog.warn("host:%s:%d user rule gae, GetReqTimeout:%d e:%r", host, port, (time.time() - start_time) * 1000, e) sock.close() except simple_http_server.GetReqTimeout as e: # xlog.warn("host:%s:%d user rule gae, GetReqTimeout:%d e:%r", host, port, (time.time()-start_time)*1000, e) sock.close() except Exception as e: xlog.warn("host:%s:%d user rule:%s except:%r", host, port, rule, e) sock.close() return elif rule == "socks": do_socks(sock, host, port, client_address, left_buf) xlog.info("host:%s:%d user rule:socks", host, port) return elif rule == "black": xlog.info("host:%s:%d user rule black", host, port) sock.close() return else: xlog.error("get rule:%s unknown", rule) sock.close() return record = g.domain_cache.get(host) if not record: rule = "direct" else: rule = record["r"] if not rule or rule == "direct": if g.config.auto_direct: ips = g.dns_srv.query(host) try: if port == 80 and g.gfwlist.check(host): do_redirect_https(sock, host, ips, port, client_address, left_buf) xlog.info("%s:%d redirect_https", host, port) return else: do_direct(sock, host, ips, port, client_address, left_buf) xlog.info("%s:%d direct", host, port) return except (ConnectFail, RedirectHttpsFail) as e: xlog.debug("%s:%d try direct/redirect fail:%r", host, port, e) rule = "gae" else: rule = "gae" if rule == "gae": if g.config.auto_gae and is_gae_workable( ) and g.domain_cache.accept_gae(host, port): try: sni_host = get_sni(sock) do_gae(sock, host, port, client_address, left_buf) xlog.info("%s:%d gae", host, port) return except SniNotExist: xlog.debug("domain:%s get sni fail", host) rule = "socks" except (SslWrapFail, simple_http_server.ParseReqFail) as e: xlog.warn("domain:%s sni:%s fail:%r", host, sni_host, e) g.domain_cache.report_gae_deny(host, port) sock.close() return except simple_http_server.GetReqTimeout: # Happen sometimes, don't known why. # xlog.warn("host:%s:%d try gae, GetReqTimeout:%d", host, port, # (time.time() - start_time) * 1000) sock.close() return except Exception as e: xlog.warn("host:%s:%d cache rule:%s except:%r", host, port, rule, e) g.domain_cache.report_gae_deny(host, port) sock.close() return else: rule = "socks" if rule == "socks": do_socks(sock, host, port, client_address, left_buf) xlog.info("%s:%d socks", host, port) return else: xlog.error("domain:%s get rule:%s unknown", host, rule) sock.close() return
def check_all_ip(check_ip): check = CheckAllIp(check_ip, "scan1.movistar.gq") check.run() if __name__ == "__main__": # case 1: only ip # case 2: ip + domain # connect use domain default_ip = "141.101.120.131" host = "cloudflare.com" if len(sys.argv) > 1: ip = sys.argv[1] if not utils.check_ip_valid(ip): ip = default_ip host = sys.argv[1] else: ip = default_ip print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]") xlog.info("test ip:%s", ip) if len(sys.argv) > 2: host = sys.argv[2] xlog.info("host:%s", host) if len(sys.argv) > 3: wait_time = int(sys.argv[3]) else: wait_time = 0
def query_over_tcp(self, domain, type=None, loop_count=0): if loop_count > 10: return [] if type is None: types = [1, 28] else: types = [type] ips = [] for t in types: query_time = 0 for i in range(0, 3): server_ip = self.dns_server.get_fastest_public() if not server_ip: return [] query_time += 1 if query_time > 3: break t0 = time.time() try: d = DNSRecord(DNSHeader()) d.add_question(DNSQuestion(domain, t)) a_pkt = d.send(server_ip, 53, tcp=True, timeout=2) t1 = time.time() self.dns_server.update_public_server(server_ip, t1-t0) p = DNSRecord.parse(a_pkt) if len(p.rr) == 0: xlog.warn("query_over_tcp for %s type:%d server:%s return none, cost:%f", domain, t, server_ip, t1-t0) continue for r in p.rr: ip = bytes(r.rdata) if utils.check_ip_valid(ip): if b"." in ip and g.ip_region.check_ip(ip): cn = g.ip_region.cn else: cn = b"XX" if type == 1 and b"." not in ip: continue ips.append(ip + b"|" + cn) else: # It is domain, loop search it. ipss = self.query_over_tcp(ip, type, loop_count+1) if not ipss: continue ips += ipss break except Exception as e: t1 = time.time() self.dns_server.update_public_server(server_ip, t1 - t0) xlog.warn("query_over_tcp %s type:%s server:%s except:%r", domain, type, server_ip,e) if ips: g.domain_cache.set_ips(domain, ips, type) return ips
# format: [ip] [domain [sni] ] # case 1: only ip # case 2: ip domain # case 3: ip domain sni # case 4: domain # case 5: domain sni ip = "141.101.120.131" host = "xx-net.net" sni = host args = list(sys.argv[1:]) if len(args): if utils.check_ip_valid(args[0]): ip = args.pop(0) if len(args): host = args.pop(0) sni = host if len(args): sni = args.pop(0) # print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]") xlog.info("test ip:%s", ip) xlog.info("host:%s", host) xlog.info("sni:%s", sni) wait_time = 0
def check_all_ip(check_ip): check = CheckAllIp(check_ip, "scan1.movistar.gq") check.run() if __name__ == "__main__": # case 1: only ip # case 2: ip + domain # connect use domain default_ip = "141.101.120.131" host = "xx-net.net" if len(sys.argv) > 1: ip = sys.argv[1] if not utils.check_ip_valid(ip): ip = default_ip host = sys.argv[1] else: ip = default_ip print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]") xlog.info("test ip:%s", ip) if len(sys.argv) > 2: host = sys.argv[2] xlog.info("host:%s", host) if len(sys.argv) > 3: wait_time = int(sys.argv[3]) else: wait_time = 0
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): global fake_host if not fake_host and g.gae_proxy: fake_host = g.gae_proxy.web_control.get_fake_host() if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == fake_host: rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) if g.config.block_advertisement and g.gfwlist.is_advertisement(host): xlog.info("block advertisement %s:%d", host, port) sock.close() return #ips = g.dns_srv.query(host) #if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443: # rule_list = ["direct", "gae", "socks", "redirect_https"] # gae is more faster then direct. record = g.domain_cache.get(host) if record and record["r"] != "unknown": rule = record["r"] if rule == "gae": rule_list = ["gae", "socks", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.gfwlist.is_white(host): rule_list = ["direct", "gae", "socks", "redirect_https"] elif g.gfwlist.check(host): rule_list = ["gae", "socks", "redirect_https", "direct"] else: ips = g.dns_srv.query(host) if g.ip_region.check_ips(ips): rule_list = ["direct", "gae", "socks", "redirect_https"] else: rule_list = ["gae", "socks", "redirect_https", "direct"] if not g.config.auto_direct and "direct" in rule_list: try: rule_list.remove("direct") rule_list.remove("redirect_https") except: pass if not g.config.enable_fake_ca and port == 443 and "gae" in rule_list: try: rule_list.remove("gae") except: pass if not g.config.auto_gae and "gae" in rule_list: try: rule_list.remove("gae") except: pass try_loop("domain", rule_list, sock, host, port, client_address, left_buf)
check.run() if __name__ == "__main__": # case 1: only ip # case 2: ip + domain # connect use domain ip = "141.101.120.131" host = "xx-net.net" sni = host args = list(sys.argv[1:]) if len(args): if utils.check_ip_valid(args[0]): ip = args.pop(0) if len(args): host = args.pop(0) sni = host if len(args): sni = args.pop(0) # print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]") xlog.info("test ip:%s", ip) xlog.info("host:%s", host) xlog.info("sni:%s", sni) wait_time = 0
def get_cert(commonname, sans=None, full_name=False): commonname = utils.to_bytes(commonname) isip = check_ip_valid(commonname) sr = '0x%s' % hashlib.md5( (str(commonname) + str(datetime.datetime.now())).encode("utf-8")).hexdigest() with ca_lock: if not isip and not full_name and commonname.count(b'.') >= 2 and [ len(x) for x in reversed(commonname.split(b'.')) ] > [2, 4]: commonname = commonname.partition(b'.')[-1] certfile = os.path.join(ca_certdir, utils.to_str(commonname) + '.crt') if os.path.exists(certfile): return certfile else: commonname = utils.to_str(commonname) if (platform.system() == 'Windows'): X509_EXTRA_ARGS = () cfile = os.path.join(ca_certdir, commonname + '.config') rfile = os.path.join(ca_certdir, commonname + '.request') config = open(cfile, 'w') config.write(OPENSSL_CONFIG_TEMPLATE % {'commonname': commonname}) config.close() openssl_c('req', '-new', '-key', cert_keyfile, '-out', rfile, '-config', cfile) openssl_c('x509', '-req', '-days', str(days), '-in', rfile, '-CA', ca_cert, '-CAkey', ca_key, '-set_serial', sr, '-out', certfile, '-extensions', 'v3_req', '-extfile', cfile, *X509_EXTRA_ARGS) return certfile elif (platform.system() == 'Linux'): script1 = ''' CONFIG=" prompt = no distinguished_name = req_distinguished_name req_extensions = v3_req [ req_distinguished_name ] C = CN ST = Internet L = Cernet O = GoAgent-XX-Net-mini-4.5.2 OU = GoAgent-XX-Net-mini-4.5.2 Root CN = GoAgent-XX-Net-mini-4.5.2 CA emailAddress = [email protected] [ v3_req ] basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment subjectAltName = @alt_names [ alt_names ] DNS.1 = %(commonname)s DNS.2 = *.%(commonname)s " ''' script1 = script1 % {'commonname': commonname} script2 = "openssl req -new -key " + cert_keyfile + ''' -config <(echo "$CONFIG")''' script = script1 + script2 p = subprocess.Popen(['bash', '-c', script], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() os.environ['request'] = stdout.decode() script3 = '''openssl x509 -req -days 3650 -in <(echo "$request") -CA ''' + ca_cert + ''' -CAkey ''' + ca_key + ''' -set_serial ''' + sr + ''' -extensions v3_req -extfile <(echo "$CONFIG")''' script4 = script1 + script3 p = subprocess.Popen(['bash', '-c', script4], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() with open(certfile, 'wb') as fp: fp.write(stdout) return certfile else: pass
def handle_domain_proxy(sock, host, port, client_address, left_buf=""): global fake_host if not fake_host and g.gae_proxy: fake_host = g.gae_proxy.web_control.get_fake_host() if not isinstance(sock, SocketWrap): sock = SocketWrap(sock, client_address[0], client_address[1]) # Check user rules sock.target = "%s:%d" % (host, port) rule = g.user_rules.check_host(host, port) if not rule: if host == fake_host: rule = "gae" elif utils.check_ip_valid(host) and utils.is_private_ip(host): rule = "direct" if rule: return try_loop("domain user", [rule], sock, host, port, client_address, left_buf) if g.config.block_advertisement and g.gfwlist.is_advertisement(host): xlog.info("block advertisement %s:%d", host, port) sock.close() return #ips = g.dns_query.query(host) #if check_local_network.IPv6.is_ok() and have_ipv6(ips) and port == 443: # rule_list = ["direct", "gae", "socks", "redirect_https"] # gae is more faster then direct. rule = g.domain_cache.get_rule(host) if rule != "unknown": if rule == "gae": rule_list = ["gae", "socks", "redirect_https", "direct"] elif rule == "socks": rule_list = ["socks", "gae", "redirect_https", "direct"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] if not g.domain_cache.accept_gae(host): rule_list.remove("gae") elif g.config.country_code == "CN": if g.gfwlist.in_white_list(host): rule_list = ["direct", "gae", "socks", "redirect_https"] elif g.gfwlist.in_block_list(host): if g.config.pac_policy == "black_X-Tunnel": rule_list = ["socks", "redirect_https", "direct", "gae"] else: rule_list = ["gae", "socks", "redirect_https", "direct"] else: ips = g.dns_query.query_recursively(host, 1) if g.ip_region.check_ips(ips): rule_list = ["direct", "socks", "redirect_https"] else: rule_list = ["direct", "gae", "socks", "redirect_https"] else: rule_list = ["direct", "socks", "gae", "redirect_https"] # check config. if not g.config.auto_direct: for rule in ["direct", "redirect_https"]: try: rule_list.remove(rule) except: pass elif g.config.auto_direct6 and "direct" in rule_list: rule_list.insert(rule_list.index("direct"), "direct6") if not g.config.enable_fake_ca and port == 443 or not g.config.auto_gae: try: rule_list.remove("gae") except: pass xlog.debug("connect to %s:%d from:%s:%d, rule:%s", host, port, client_address[0], client_address[1], utils.to_str(rule_list)) try_loop("domain", rule_list, sock, host, port, client_address, left_buf)