def __init__(self):
self.running = True
self.last_host = "center.xx-net.net"
self.logger = logger
config_path = os.path.join(module_data_path, "cloudfront_front.json")
self.config = Config(config_path)
sni_fn = os.path.join(current_path, "sni_list.txt")
self.host_manager = host_manager.HostManager(fn=sni_fn, max_size=120)
ca_certs = os.path.join(current_path, "cacert.pem")
openssl_context = SSLContext(logger, ca_certs=ca_certs, support_http2=True)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = Ipv4RangeSource(
logger, self.config,
os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "cloudfront_ip_range.txt")
)
self.ip_manager = IpManager(
logger, self.config, ip_source, check_local_network,
self.check_ip,
os.path.join(current_path, "good_ip.txt"),
os.path.join(module_data_path, "cloudfront_ip_list.txt"),
scan_ip_log=None)
self.connect_manager = ConnectManager(
logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.dispatchs = {}
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.logger,
self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def __init__(self):
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
self.account = ""
self.password = ""
threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager,
debug=True)
self.check_ip = CheckIp(logger, config, connect_creator)
self.lock = threading.Lock()
self.in_fd = open("good_ip.txt", "r")
self.out_fd = open(
os.path.join(module_data_path, "heroku_checked_ip.txt"),
"w"
)
def check_one(ip, top_domain, wait_time):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager,
debug=True)
check_ip = CheckIp(logger, config, connect_creator)
res = check_ip.check_ip(ip, host=top_domain, wait_time=wait_time)
if not res:
print("connect fail")
elif res.ok:
print(("success, domain:%s handshake:%d" % (res.domain, res.handshake_time)))
else:
print("not support")
def check_one(ip, top_domain, wait_time):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager,
debug=True)
check_ip = CheckIp(logger, config, connect_creator)
res = check_ip.check_ip(ip, host=top_domain, wait_time=wait_time)
if not res:
print("connect fail")
elif res.ok:
print("success, domain:%s handshake:%d" % (res.domain, res.handshake_time))
else:
print("not support")
def __init__(self):
self.running = True
self.last_host = "center.xx-net.net"
self.logger = logger
config_path = os.path.join(module_data_path, "cloudflare_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
default_domain_fn = os.path.join(current_path, "front_domains.json")
domain_fn = os.path.join(module_data_path, "cloudflare_domains.json")
self.host_manager = host_manager.HostManager(self.config, logger,
default_domain_fn,
domain_fn, self)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = Ipv4RangeSource(
logger, self.config, os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "cloudflare_ip_range.txt"))
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
os.path.join(current_path, "good_ip.txt"),
os.path.join(module_data_path,
"cloudflare_ip_list.txt"),
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.dispatchs = {}
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self):
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
if not os.path.isdir(module_data_path):
os.mkdir(module_data_path)
if not os.path.isdir(tls_certs_path):
os.mkdir(tls_certs_path)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = connect_creator.ConnectCreator(
logger, self.config, self.openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.account = ""
self.password = ""
def __init__(self):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager,
debug=True)
self.check_ip = CheckIp(logger, config, connect_creator)
self.lock = threading.Lock()
self.in_fd = open("good_ip.txt", "r")
self.out_fd = open(
os.path.join(module_data_path, "heroku_checked_ip.txt"),
"w"
)
def __init__(self):
self.running = True
self.last_host = "center.xx-net.org"
self.logger = logger
config_path = os.path.join(module_data_path, "cloudflare_front.json")
self.config = Config(config_path)
self.light_config = Config(config_path)
self.light_config.dispather_min_workers = 1
self.light_config.max_good_ip_num = 10
ca_certs = os.path.join(current_path, "cacert.pem")
default_domain_fn = os.path.join(current_path, "front_domains.json")
domain_fn = os.path.join(module_data_path, "cloudflare_domains.json")
self.host_manager = host_manager.HostManager(self.config, logger,
default_domain_fn,
domain_fn, self)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = Ipv4RangeSource(
logger, self.config, os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "cloudflare_ip_range.txt"))
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
os.path.join(current_path, "good_ip.txt"),
os.path.join(module_data_path,
"cloudflare_ip_list.txt"),
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.dispatchs = {}
class CheckAllIp(object):
def __init__(self):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger,
config,
openssl_context,
host_manager,
debug=True)
self.check_ip = CheckIp(logger, config, connect_creator)
self.lock = threading.Lock()
self.in_fd = open("good_ip.txt", "r")
self.out_fd = open(
os.path.join(module_data_path, "heroku_checked_ip.txt"), "w")
def get_ip(self):
with self.lock:
while True:
line = self.in_fd.readline()
if not line:
raise Exception()
try:
ip = line.split()[0]
return ip
except:
continue
def write_ip(self, ip, host, handshake):
with self.lock:
self.out_fd.write("%s %s gws %d 0 0\n" % (ip, host, handshake))
self.out_fd.flush()
def checker(self):
while True:
try:
ip = self.get_ip()
except Exception as e:
xlog.info("no ip left")
return
try:
res = self.check_ip.check_ip(ip)
except Exception as e:
xlog.warn("check except:%r", e)
continue
if not res or not res.ok:
xlog.debug("ip:%s fail", ip)
continue
self.write_ip(ip, res.domain, res.handshake_time)
def run(self):
for i in range(0, 10):
threading.Thread(target=self.checker).run()
class Front(object):
name = "cloudfront_front"
def __init__(self):
self.running = True
self.last_host = "center.xx-net.net"
self.logger = logger
config_path = os.path.join(module_data_path, "cloudfront_front.json")
self.config = Config(config_path)
sni_fn = os.path.join(current_path, "sni_list.txt")
self.host_manager = host_manager.HostManager(fn=sni_fn, max_size=120)
ca_certs = os.path.join(current_path, "cacert.pem")
openssl_context = SSLContext(logger, ca_certs=ca_certs, support_http2=True)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = Ipv4RangeSource(
logger, self.config,
os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "cloudfront_ip_range.txt")
)
self.ip_manager = IpManager(
logger, self.config, ip_source, check_local_network,
self.check_ip,
os.path.join(current_path, "good_ip.txt"),
os.path.join(module_data_path, "cloudfront_ip_list.txt"),
scan_ip_log=None)
self.connect_manager = ConnectManager(
logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.dispatchs = {}
def check_ip(self, ip):
sni, host = self.host_manager.get_sni_host(ip)
host = "scan1.xx-net.net"
return self.ip_checker.check_ip(ip, sni=sni, host=host)
def get_dispatcher(self, host=None):
if host is None:
host = self.last_host
else:
self.last_host = host
if host not in self.dispatchs:
http_dispatcher = HttpsDispatcher(
logger, self.config, self.ip_manager, self.connect_manager
)
self.dispatchs[host] = http_dispatcher
dispatcher = self.dispatchs[host]
return dispatcher
def request(self, method, host, path="/", headers={}, data="", timeout=120):
dispatcher = self.get_dispatcher(host)
headers = dict(headers)
response = dispatcher.request(method, host, path, headers, data, timeout=timeout)
if not response:
self.logger.warn("req %s get response timeout", path)
return "", 602, {}
status = response.status
content = response.task.read_all()
if status == 200:
self.logger.debug("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status,
response.task.get_trace())
else:
self.logger.warn("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status,
response.task.get_trace())
return content, status, response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
for host in self.dispatchs:
dispatcher = self.dispatchs[host]
dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
top_domain = sys.argv[2]
else:
top_domain = None
if len(sys.argv) > 3:
wait_time = int(sys.argv[3])
else:
wait_time = 0
config_path = os.path.join(module_data_path, "tls_relay.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_fn = os.path.join(module_data_path, "tls_host.json")
host_manager = HostManager(host_fn)
connect_creator = ConnectCreator(logger, config, openssl_context,
host_manager)
check_ip = CheckIp(logger, config, connect_creator)
res = check_ip.check_ip(ip,
sni=top_domain,
host=top_domain,
wait_time=wait_time)
if not res:
print("connect fail")
elif res.ok:
print(("success, domain:%s handshake:%d" %
(res.top_domain, res.handshake_time)))
else:
print("not support")
xlog.info("host:%s", host)
if len(sys.argv) > 3:
wait_time = int(sys.argv[3])
else:
wait_time = 0
config_path = os.path.join(module_data_path, "cloudflare_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger,
config,
openssl_context,
host_manager,
debug=True)
check_ip = CheckIp(logger, config, connect_creator)
#check_all_domain(check_ip)
check_all_ip(check_ip)
exit(0)
res = check_ip.check_ip(ip, host=host, wait_time=wait_time)
if not res:
xlog.warn("connect fail")
elif res.ok:
xlog.info("success, domain:%s handshake:%d", res.host,
res.handshake_time)
else:
xlog.warn("not support")
ip = sys.argv[1]
else:
ip = "104.28.77.8"
print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
print("test ip:%s" % ip)
if len(sys.argv) > 2:
top_domain = sys.argv[2]
else:
top_domain = None
if len(sys.argv) > 3:
wait_time = int(sys.argv[3])
else:
wait_time = 0
config_path = os.path.join(module_data_path, "cloudflare_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
connect_creator = ConnectCreator(logger, config, openssl_context)
check_ip = CheckIp(logger, config, connect_creator)
res = check_ip.check_ip(ip, top_domain=top_domain, wait_time=wait_time)
if not res:
print("connect fail")
elif res.ok:
print("success, domain:%s handshake:%d" % (res.top_domain, res.handshake_time))
else:
print("not support")
sni = host
if len(args):
sni = args.pop(0)
# print("Usage: check_ip.py [ip] [top_domain] [wait_time=0]")
xlog.info("test ip:%s", ip)
xlog.info("host:%s", host)
xlog.info("sni:%s", sni)
wait_time = 0
config_path = os.path.join(module_data_path, "cloudflare_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager, debug=True)
check_ip = CheckIp(logger, config, connect_creator)
#check_all_domain(check_ip)
#check_all_ip(check_ip)
#exit(0)
res = check_ip.check_ip(ip, sni=sni, host=host, wait_time=wait_time)
if not res:
xlog.warn("connect fail")
elif res.ok:
xlog.info("success, domain:%s handshake:%d", res.host, res.handshake_time)
else:
xlog.warn("not support")
class CheckAllIp(object):
def __init__(self):
config_path = os.path.join(module_data_path, "heroku_front.json")
config = Config(config_path)
openssl_context = SSLContext(logger)
host_manager = HostManagerBase()
connect_creator = ConnectCreator(logger, config, openssl_context, host_manager,
debug=True)
self.check_ip = CheckIp(logger, config, connect_creator)
self.lock = threading.Lock()
self.in_fd = open("good_ip.txt", "r")
self.out_fd = open(
os.path.join(module_data_path, "heroku_checked_ip.txt"),
"w"
)
def get_ip(self):
with self.lock:
while True:
line = self.in_fd.readline()
if not line:
raise Exception()
try:
ip = line.split()[0]
return ip
except:
continue
def write_ip(self, ip, host, handshake):
with self.lock:
self.out_fd.write("%s %s gws %d 0 0\n" % (ip, host, handshake))
self.out_fd.flush()
def checker(self):
while True:
try:
ip = self.get_ip()
except Exception as e:
xlog.info("no ip left")
return
try:
res = self.check_ip.check_ip(ip)
except Exception as e:
xlog.warn("check except:%r", e)
continue
if not res or not res.ok:
xlog.debug("ip:%s fail", ip)
continue
self.write_ip(ip, res.domain, res.handshake_time)
def run(self):
for i in range(0, 10):
threading.Thread(target=self.checker).run()