def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.logger,
self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def __init__(self):
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
self.account = ""
self.password = ""
threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self):
self.running = True
self.logger = logger
self.config = config
self.host_manager = host_manager.HostManager(self.config, logger)
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
cipher_suites=[
'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA",
"!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256",
"!ECDHE-RSA-AES128-SHA", "!AES128-SHA"
])
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.appid_manager = AppidManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
self.ipv4_source = Ipv4RangeSource(
logger, self.config, os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "ip_range.txt"))
self.ipv6_source = Ipv6PoolSource(
logger, self.config, os.path.join(current_path, "ipv6_list.txt"))
self.ip_source = IpCombineSource(logger, self.config, self.ipv4_source,
self.ipv6_source)
self.ip_manager = IpManager(logger,
self.config,
self.ip_source,
check_local_network,
self.check_ip.check_ip,
None,
os.path.join(module_data_path,
"good_ip.txt"),
scan_ip_log=None)
self.appid_manager.check_api = self.check_ip.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def __init__(self):
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
if not os.path.isdir(module_data_path):
os.mkdir(module_data_path)
if not os.path.isdir(tls_certs_path):
os.mkdir(tls_certs_path)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = connect_creator.ConnectCreator(
logger, self.config, self.openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.account = ""
self.password = ""
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
threading.Thread(target=self.debug_data_clearup_thread).start()
def get_dispatcher(self, host):
if host not in self.dispatchs:
http_dispatcher = HttpsDispatcher(logger, config, front.ip_manager,
self.connect_manager)
self.dispatchs[host] = http_dispatcher
return self.dispatchs[host]
def start(self):
self.running = True
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
cipher_suites=[
'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA",
"!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256",
"!ECDHE-RSA-AES128-SHA", "!AES128-SHA"
])
self.appid_manager = AppidManager(self.config, logger)
self.host_manager = host_manager.HostManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
#self.ip_checker = CheckIp(logger, self.config, self.connect_creator)
self.ipv6_source = Ipv6PoolSource(logger, self.config, "ipv6_list.txt")
self.ip_source = self.ipv6_source
self.ip_manager = IpManager(logger,
self.config,
self.ip_source,
check_local_network,
None,
os.path.join(data_path, "good_ip.txt"),
scan_ip_log=None)
#self.appid_manager.check_api = self.ip_checker.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def init_host_dispatcher(self, host):
if host not in self.dispatchs:
http_dispatcher = HttpsDispatcher(
logger,
self.config,
self.ip_manager,
self.connect_manager,
http2worker=CloudflareHttp2Worker)
self.dispatchs[host] = http_dispatcher
def start(self):
self.running = True
ca_certs = 'cacert.pem'
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
protocol="TLSv1_2"
#cipher_suites=[b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA", b"!ECDHE-RSA-AES128-GCM-SHA256",
# b"!AES128-GCM-SHA256", b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA"]
)
self.appid_manager = AppidManager(self.config, logger)
self.host_manager = host_manager.HostManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
# self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator)
self.ip_manager = IpManager(logger,
self.config,
check_local_network,
None,
'good_ip.txt',
scan_ip_log=None)
# self.appid_manager.check_api = self.ip_checker.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def __init__(self):
self.running = True
self.logger = logger
self.config = config
self.host_manager = host_manager.HostManager(self.config, logger)
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger, ca_certs=ca_certs, support_http2=config.support_http2,
cipher_suites=['ALL', "!RC4-SHA","!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256",
"!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA"]
)
self.connect_creator = ConnectCreator(
logger, self.config, self.openssl_context, self.host_manager)
self.appid_manager = AppidManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
self.ipv4_source = Ipv4RangeSource(
logger, self.config,
os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "ip_range.txt")
)
self.ipv6_source = Ipv6PoolSource(
logger, self.config,
os.path.join(current_path, "ipv6_list.txt")
)
self.ip_source = IpCombineSource(
logger, self.config,
self.ipv4_source, self.ipv6_source
)
self.ip_manager = IpManager(
logger, self.config, self.ip_source, check_local_network,
self.check_ip.check_ip,
None,
os.path.join(module_data_path, "good_ip.txt"),
scan_ip_log=None)
self.appid_manager.check_api = self.check_ip.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(
logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(
logger, self.config, self.ip_manager, self.connect_manager
)
def get_dispatcher(self, host=None):
if host is None:
host = self.last_host
else:
self.last_host = host
if host not in self.dispatchs:
http_dispatcher = HttpsDispatcher(
logger, self.config, self.ip_manager, self.connect_manager
)
self.dispatchs[host] = http_dispatcher
dispatcher = self.dispatchs[host]
return dispatcher
def __init__(self):
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
self.account = ""
self.password = ""
threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.logger, self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network,
self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def get_dispatcher(self, host=None):
if not host:
host = self.last_host
else:
self.last_host = host
if host not in self.dispatchs:
if host in ["center.xx-net.org", "dns.xx-net.org"]:
config = self.light_config
else:
config = self.config
http_dispatcher = HttpsDispatcher(
logger,
config,
self.ip_manager,
self.connect_manager,
http2worker=CloudflareHttp2Worker)
self.dispatchs[host] = http_dispatcher
dispatcher = self.dispatchs[host]
return dispatcher
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network,
self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
threading.Thread(target=self.debug_data_clearup_thread).start()
class Front(object):
name = "tls_relay_front"
def __init__(self):
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
self.account = ""
self.password = ""
threading.Thread(target=self.debug_data_clearup_thread).start()
def set_x_tunnel_account(self, account, password):
self.account = account
self.password = password
def log_debug_data(self, rtt, sent, received):
now = time.time()
self.rtts.append((rtt, now))
with self.traffics_lock:
self.traffics.append((sent, received, now))
self.recent_sent += sent
self.recent_received += received
self.total_sent += sent
self.total_received += received
def get_rtt(self):
now = time.time()
while len(self.rtts) > 1:
with self.rtts_lock:
rtt, log_time = rtt_log = max(self.rtts)
if now - log_time > 5:
self.rtts.remove(rtt_log)
continue
return rtt
return self.rtts[0][0]
def debug_data_clearup_thread(self):
while self.running:
now = time.time()
with self.rtts_lock:
if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5:
self.rtts.popleft()
with self.traffics_lock:
if self.traffics and now - self.traffics[0][-1] > 60:
sent, received, _ = self.traffics.popleft()
self.recent_sent -= sent
self.recent_received -= received
time.sleep(1)
def worker_num(self):
return len(self.http_dispatcher.workers)
def set_ips(self, ips):
if not ips:
return
host_info = {}
ca_certs = []
ipss = []
for ip in ips:
dat = ips[ip]
ca_cert = dat["ca_crt"]
sni = dat["sni"]
host_info[ip] = {"sni":sni, "ca_crt":ca_cert}
if ca_cert not in ca_certs:
ca_certs.append(ca_cert)
ipss.append(ip)
self.ip_manager.update_ips(ipss)
self.ip_manager.save(True)
self.host_manager.set_host(host_info)
ca_content = "\n\n".join(ca_certs)
with open(self.ca_cert_fn, "w") as fd:
fd.write(ca_content)
self.openssl_context.set_ca(self.ca_cert_fn)
self.logger.info("set_ips:%s", ",".join(ipss))
def get_score(self, host=None):
now = time.time()
if now - self.last_fail_time < self.config.front_continue_fail_block and \
self.continue_fail_num > self.config.front_continue_fail_num:
return None
worker = self.http_dispatcher.get_worker(nowait=True)
if not worker:
return None
return worker.get_score()
def request(self, method, host, path="/", headers={}, data="", timeout=120):
headers = dict(headers)
headers["XX-Account"] = self.account
response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
if not response:
logger.warn("req %s get response timeout", path)
return "", 602, {}
status = response.status
if status not in [200, 405]:
# logger.warn("front request %s %s%s fail, status:%d", method, host, path, status)
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
else:
self.success_num += 1
self.continue_fail_num = 0
content = response.task.read_all()
if status == 200:
logger.debug("%s %s%s status:%d trace:%s", method, host, path, status,
response.task.get_trace())
else:
logger.warn("%s %s%s status:%d trace:%s", method, host, path, status,
response.task.get_trace())
return content, status, response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "gae_front"
def __init__(self):
self.logger = logger
self.config = config
def start(self):
self.running = True
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
cipher_suites=[
b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA",
b"!ECDHE-RSA-AES128-GCM-SHA256", b"!AES128-GCM-SHA256",
b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA"
])
self.appid_manager = AppidManager(self.config, logger)
self.host_manager = host_manager.HostManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator)
self.ipv4_source = Ipv4RangeSource(
logger, self.config, os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "ip_range.txt"))
self.ipv6_source = Ipv6PoolSource(
logger, self.config, os.path.join(current_path, "ipv6_list.txt"))
self.ip_source = IpCombineSource(logger, self.config, self.ipv4_source,
self.ipv6_source)
self.ip_manager = IpManager(logger,
self.config,
self.ip_source,
check_local_network,
self.check_ip,
None,
os.path.join(module_data_path,
"good_ip.txt"),
scan_ip_log=None)
self.appid_manager.check_api = self.ip_checker.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def check_ip(self, ip):
sni = self.host_manager.sni_manager.get()
host = self.config.check_ip_host
return self.ip_checker.check_ip(ip, sni=sni, host=host)
def get_dispatcher(self):
return self.http_dispatcher
def request(self,
method,
host,
path=b"/",
headers={},
data="",
timeout=120):
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
return response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "heroku_front"
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.logger, self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network,
self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def get_dispatcher(self, host=None):
return self.http_dispatcher
def set_ips(self, ips):
self.ip_manager.set_ips(ips)
def _request(self, method, host, path="/", headers={}, data="", timeout=40):
try:
response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
if not response:
return "", 500, {}
status = response.status
if status != 200:
self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status)
content = response.task.read_all()
# self.logger.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
return content, status, response
except Exception as e:
self.logger.exception("front request %s %s%s fail:%s", method, host, path, e)
return "", 500, {}
def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}
heroku_host = ""
content, status, response = self._request(
"POST", heroku_host, "/2/",
request_headers, request_body, timeout)
# self.logger.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# self.logger.debug("status:%d", status)
if hasattr(response, "task"):
self.logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace())
else:
self.logger.debug("%s %s%s status:%d", method, host, path, status)
if status == 404:
heroku_host = response.ssl_sock.host
self.logger.warn("heroku appid:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
return "", 501, {}
if not content:
return "", 501, {}
try:
res = simple_http_client.TxtResponse(content)
except Exception as e:
self.logger.warn("decode %s response except:%r", content, e)
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "heroku_front"
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network,
self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
threading.Thread(target=self.debug_data_clearup_thread).start()
def log_debug_data(self, rtt, sent, received):
now = time.time()
self.rtts.append((rtt, now))
with self.traffics_lock:
self.traffics.append((sent, received, now))
self.recent_sent += sent
self.recent_received += received
self.total_sent += sent
self.total_received += received
def get_rtt(self):
now = time.time()
while len(self.rtts) > 1:
with self.rtts_lock:
rtt, log_time = rtt_log = max(self.rtts)
if now - log_time > 5:
self.rtts.remove(rtt_log)
continue
return rtt
return self.rtts[0][0]
def debug_data_clearup_thread(self):
while self.running:
now = time.time()
with self.rtts_lock:
if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5:
self.rtts.popleft()
with self.traffics_lock:
if self.traffics and now - self.traffics[0][-1] > 60:
sent, received, _ = self.traffics.popleft()
self.recent_sent -= sent
self.recent_received -= received
time.sleep(1)
def worker_num(self):
return len(self.http_dispatcher.workers)
def set_ips(self, ips):
self.ip_manager.set_ips(ips)
def get_score(self, host=None):
if not self.host_manager.appids:
return None
now = time.time()
if now - self.last_fail_time < self.config.front_continue_fail_block and \
self.continue_fail_num > self.config.front_continue_fail_num:
return None
worker = self.http_dispatcher.get_worker(nowait=True)
if not worker:
return None
return worker.get_score()
def _request(self, method, host, path="/", headers={}, data="", timeout=40):
try:
response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
if not response:
return "", 500, {}
status = response.status
if status != 200:
xlog.warn("front request %s %s%s fail, status:%d", method, host, path, status)
content = response.task.read_all()
# xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
return content, status, response
except Exception as e:
xlog.exception("front request %s %s%s fail:%r", method, host, path, e)
return "", 500, {}
def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
# change top domain to xx-net.net
# this domain bypass the cloudflare front for ipv4
#p = host.find(".")
#host_sub = host[:p]
#host = host_sub + ".xx-net.net"
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}
heroku_host = ""
content, status, response = self._request(
"POST", heroku_host, "/2/",
request_headers, request_body, timeout)
# xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# xlog.debug("status:%d", status)
if status == 200:
xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace())
self.last_success_time = time.time()
self.continue_fail_num = 0
self.success_num += 1
else:
if status == 404:
heroku_host = response.ssl_sock.host
xlog.warn("heroku:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
self.last_fail_time = time.time()
self.continue_fail_num += 1
self.fail_num += 1
try:
res = simple_http_client.TxtResponse(content)
except:
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "gae_front"
def __init__(self):
self.running = True
self.logger = logger
self.config = config
self.host_manager = host_manager.HostManager(self.config, logger)
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger, ca_certs=ca_certs, support_http2=config.support_http2,
cipher_suites=['ALL', "!RC4-SHA","!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256",
"!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA"]
)
self.connect_creator = ConnectCreator(
logger, self.config, self.openssl_context, self.host_manager)
self.appid_manager = AppidManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
self.ipv4_source = Ipv4RangeSource(
logger, self.config,
os.path.join(current_path, "ip_range.txt"),
os.path.join(module_data_path, "ip_range.txt")
)
self.ipv6_source = Ipv6PoolSource(
logger, self.config,
os.path.join(current_path, "ipv6_list.txt")
)
self.ip_source = IpCombineSource(
logger, self.config,
self.ipv4_source, self.ipv6_source
)
self.ip_manager = IpManager(
logger, self.config, self.ip_source, check_local_network,
self.check_ip.check_ip,
None,
os.path.join(module_data_path, "good_ip.txt"),
scan_ip_log=None)
self.appid_manager.check_api = self.check_ip.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(
logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(
logger, self.config, self.ip_manager, self.connect_manager
)
def get_dispatcher(self):
return self.http_dispatcher
def request(self, method, host, path="/", headers={}, data="", timeout=120):
response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
return response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "heroku_front"
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.logger, self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network,
self.check_ip.check_ip,
default_ip_list_fn, ip_list_fn, scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def get_dispatcher(self, host=None):
if len(self.host_manager.appids) == 0:
return None
return self.http_dispatcher
def set_ips(self, ips):
self.ip_manager.set_ips(ips)
def _request(self, method, host, path="/", headers={}, data="", timeout=40):
try:
response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
if not response:
return "", 500, {}
status = response.status
if status != 200:
self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status)
content = response.task.read_all()
# self.logger.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
return content, status, response
except Exception as e:
self.logger.exception("front request %s %s%s fail:%s", method, host, path, e)
return "", 500, {}
def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}
heroku_host = ""
content, status, response = self._request(
"POST", heroku_host, "/2/",
request_headers, request_body, timeout)
# self.logger.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# self.logger.debug("status:%d", status)
if hasattr(response, "task"):
self.logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace())
else:
self.logger.debug("%s %s%s status:%d", method, host, path, status)
if status == 404:
heroku_host = response.ssl_sock.host
self.logger.warn("heroku appid:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
return "", 501, {}
if not content:
return "", 501, {}
try:
res = simple_http_client.TxtResponse(content)
except Exception as e:
self.logger.warn("decode %s response except:%r", content, e)
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "tls_relay_front"
def __init__(self):
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.account = ""
self.password = ""
def get_dispatcher(self, host=None):
return self.http_dispatcher
def set_x_tunnel_account(self, account, password):
self.account = account
self.password = password
def set_ips(self, ips):
if not ips:
return
host_info = {}
ca_certs = []
ipss = []
for ip in ips:
dat = ips[ip]
ca_cert = dat["ca_crt"]
sni = dat["sni"]
host_info[ip] = {"sni": sni, "ca_crt": ca_cert}
if ca_cert not in ca_certs:
ca_certs.append(ca_cert)
ipss.append(ip)
self.ip_manager.update_ips(ipss)
self.ip_manager.save(True)
self.host_manager.set_host(host_info)
ca_content = "\n\n".join(ca_certs)
with open(self.ca_cert_fn, "w") as fd:
fd.write(ca_content)
self.openssl_context.set_ca(self.ca_cert_fn)
self.logger.info("set_ips:%s", ",".join(ipss))
def request(self,
method,
host,
path="/",
headers={},
data="",
timeout=120):
headers = dict(headers)
headers["XX-Account"] = self.account
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
if not response:
logger.warn("req %s get response timeout", path)
return "", 602, {}
status = response.status
content = response.task.read_all()
if status == 200:
logger.debug("%s %s%s status:%d trace:%s", method, host, path,
status, response.task.get_trace())
else:
logger.warn("%s %s%s status:%d trace:%s", method, host, path,
status, response.task.get_trace())
return content, status, response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "gae_front"
def __init__(self):
self.logger = logger
self.config = config
def start(self):
self.running = True
ca_certs = os.path.join(current_path, "cacert.pem")
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
cipher_suites=[
'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA",
"!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256",
"!ECDHE-RSA-AES128-SHA", "!AES128-SHA"
])
self.appid_manager = AppidManager(self.config, logger)
self.host_manager = host_manager.HostManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
#self.ip_checker = CheckIp(logger, self.config, self.connect_creator)
self.ipv6_source = Ipv6PoolSource(logger, self.config, "ipv6_list.txt")
self.ip_source = self.ipv6_source
self.ip_manager = IpManager(logger,
self.config,
self.ip_source,
check_local_network,
None,
os.path.join(data_path, "good_ip.txt"),
scan_ip_log=None)
#self.appid_manager.check_api = self.ip_checker.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def request(self,
method,
host,
path="/",
headers={},
data="",
timeout=120):
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
return response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
class Front(object):
name = "gae_front"
def __init__(self):
self.logger = logger
self.config = config
def start(self):
self.running = True
ca_certs = 'cacert.pem'
self.openssl_context = SSLContext(
logger,
ca_certs=ca_certs,
support_http2=config.support_http2,
protocol="TLSv1_2"
#cipher_suites=[b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA", b"!ECDHE-RSA-AES128-GCM-SHA256",
# b"!AES128-GCM-SHA256", b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA"]
)
self.appid_manager = AppidManager(self.config, logger)
self.host_manager = host_manager.HostManager(self.config, logger)
self.host_manager.appid_manager = self.appid_manager
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
# self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator)
self.ip_manager = IpManager(logger,
self.config,
check_local_network,
None,
'good_ip.txt',
scan_ip_log=None)
# self.appid_manager.check_api = self.ip_checker.check_ip
self.appid_manager.ip_manager = self.ip_manager
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
def get_dispatcher(self):
return self.http_dispatcher
def request(self,
method,
host,
path=b"/",
headers={},
data="",
timeout=120):
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
return response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "tls_relay_front"
def __init__(self):
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.logger = logger
config_path = os.path.join(module_data_path, "tls_relay.json")
self.config = Config(config_path)
self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt")
self.openssl_context = SSLContext(logger)
if os.path.isfile(self.ca_cert_fn):
self.openssl_context.set_ca(self.ca_cert_fn)
host_fn = os.path.join(module_data_path, "tls_host.json")
self.host_manager = host_manager.HostManager(host_fn)
self.connect_creator = ConnectCreator(logger, self.config,
self.openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = IpSimpleSource(self.config.ip_source_ips)
default_ip_list_fn = ""
ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
for ip in self.config.ip_source_ips:
self.ip_manager.add_ip(ip, 100)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
self.account = ""
self.password = ""
threading.Thread(target=self.debug_data_clearup_thread).start()
def set_x_tunnel_account(self, account, password):
self.account = account
self.password = password
def log_debug_data(self, rtt, sent, received):
now = time.time()
self.rtts.append((rtt, now))
with self.traffics_lock:
self.traffics.append((sent, received, now))
self.recent_sent += sent
self.recent_received += received
self.total_sent += sent
self.total_received += received
def get_rtt(self):
now = time.time()
while len(self.rtts) > 1:
with self.rtts_lock:
rtt, log_time = rtt_log = max(self.rtts)
if now - log_time > 5:
self.rtts.remove(rtt_log)
continue
return rtt
return self.rtts[0][0]
def debug_data_clearup_thread(self):
while self.running:
now = time.time()
with self.rtts_lock:
if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5:
self.rtts.popleft()
with self.traffics_lock:
if self.traffics and now - self.traffics[0][-1] > 60:
sent, received, _ = self.traffics.popleft()
self.recent_sent -= sent
self.recent_received -= received
time.sleep(1)
def worker_num(self):
return len(self.http_dispatcher.workers)
def set_ips(self, ips):
if not ips:
return
host_info = {}
ca_certs = []
ipss = []
for ip in ips:
dat = ips[ip]
ca_cert = dat["ca_crt"]
sni = dat["sni"]
host_info[ip] = {"sni": sni, "ca_crt": ca_cert}
if ca_cert not in ca_certs:
ca_certs.append(ca_cert)
ipss.append(ip)
self.ip_manager.update_ips(ipss)
self.ip_manager.save(True)
self.host_manager.set_host(host_info)
ca_content = "\n\n".join(ca_certs)
with open(self.ca_cert_fn, "w") as fd:
fd.write(ca_content)
self.openssl_context.set_ca(self.ca_cert_fn)
self.logger.info("set_ips:%s", ",".join(ipss))
def get_score(self, host=None):
now = time.time()
if now - self.last_fail_time < self.config.front_continue_fail_block and \
self.continue_fail_num > self.config.front_continue_fail_num:
return None
worker = self.http_dispatcher.get_worker(nowait=True)
if not worker:
return None
return worker.get_score()
def request(self,
method,
host,
path="/",
headers={},
data="",
timeout=120):
headers = dict(headers)
headers["XX-Account"] = self.account
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
if not response:
logger.warn("req %s get response timeout", path)
return "", 602, {}
status = response.status
if status not in [200, 405]:
# logger.warn("front request %s %s%s fail, status:%d", method, host, path, status)
self.fail_num += 1
self.continue_fail_num += 1
self.last_fail_time = time.time()
else:
self.success_num += 1
self.continue_fail_num = 0
content = response.task.read_all()
if status == 200:
logger.debug("%s %s%s status:%d trace:%s", method, host, path,
status, response.task.get_trace())
else:
logger.warn("%s %s%s status:%d trace:%s", method, host, path,
status, response.task.get_trace())
return content, status, response
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()
class Front(object):
name = "heroku_front"
def __init__(self):
self.logger = logger
config_path = os.path.join(module_data_path, "heroku_front.json")
self.config = Config(config_path)
ca_certs = os.path.join(current_path, "cacert.pem")
self.host_manager = host_manager.HostManager(self.config.appids)
openssl_context = SSLContext(logger, ca_certs=ca_certs)
self.connect_creator = ConnectCreator(logger, self.config,
openssl_context,
self.host_manager)
self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator)
ip_source = None
default_ip_list_fn = os.path.join(current_path, "good_ip.txt")
ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt")
self.ip_manager = IpManager(logger,
self.config,
ip_source,
check_local_network,
self.check_ip.check_ip,
default_ip_list_fn,
ip_list_fn,
scan_ip_log=None)
self.connect_manager = ConnectManager(logger, self.config,
self.connect_creator,
self.ip_manager,
check_local_network)
self.http_dispatcher = HttpsDispatcher(logger, self.config,
self.ip_manager,
self.connect_manager)
self.success_num = 0
self.fail_num = 0
self.continue_fail_num = 0
self.last_fail_time = 0
self.running = True
self.rtts = collections.deque([(0, time.time())])
self.rtts_lock = threading.Lock()
self.traffics = collections.deque()
self.traffics_lock = threading.Lock()
self.recent_sent = 0
self.recent_received = 0
self.total_sent = 0
self.total_received = 0
threading.Thread(target=self.debug_data_clearup_thread).start()
def log_debug_data(self, rtt, sent, received):
now = time.time()
self.rtts.append((rtt, now))
with self.traffics_lock:
self.traffics.append((sent, received, now))
self.recent_sent += sent
self.recent_received += received
self.total_sent += sent
self.total_received += received
def get_rtt(self):
now = time.time()
while len(self.rtts) > 1:
with self.rtts_lock:
rtt, log_time = rtt_log = max(self.rtts)
if now - log_time > 5:
self.rtts.remove(rtt_log)
continue
return rtt
return self.rtts[0][0]
def debug_data_clearup_thread(self):
while self.running:
now = time.time()
with self.rtts_lock:
if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5:
self.rtts.popleft()
with self.traffics_lock:
if self.traffics and now - self.traffics[0][-1] > 60:
sent, received, _ = self.traffics.popleft()
self.recent_sent -= sent
self.recent_received -= received
time.sleep(1)
def worker_num(self):
return len(self.http_dispatcher.workers)
def set_ips(self, ips):
self.ip_manager.set_ips(ips)
def get_score(self, host=None):
if not self.host_manager.appids:
return None
now = time.time()
if now - self.last_fail_time < self.config.front_continue_fail_block and \
self.continue_fail_num > self.config.front_continue_fail_num:
return None
worker = self.http_dispatcher.get_worker(nowait=True)
if not worker:
return None
return worker.get_score()
def _request(self,
method,
host,
path="/",
headers={},
data="",
timeout=40):
try:
response = self.http_dispatcher.request(method,
host,
path,
dict(headers),
data,
timeout=timeout)
status = response.status
if status != 200:
xlog.warn("front request %s %s%s fail, status:%d", method,
host, path, status)
content = response.task.read_all()
# xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
return content, status, response
except Exception as e:
xlog.exception("front request %s %s%s fail:%r", method, host, path,
e)
return "", 500, {}
def request(self,
method,
host,
schema="http",
path="/",
headers={},
data="",
timeout=40):
# change top domain to xx-net.net
# this domain bypass the cloudflare front for ipv4
p = host.find(".")
host_sub = host[:p]
host = host_sub + ".xx-net.net"
schema = "http"
# force schema to http, avoid cert fail on heroku curl.
# and all x-server provide ipv4 access
url = schema + "://" + host + path
payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
for k in headers:
v = headers[k]
payloads.append('%s: %s\r\n' % (k, v))
head_payload = "".join(payloads)
request_body = '%s%s%s%s' % \
((struct.pack('!H', len(head_payload)), head_payload,
struct.pack('!I', len(data)), data))
request_headers = {
'Content-Length': len(data),
'Content-Type': 'application/octet-stream'
}
heroku_host = ""
content, status, response = self._request("POST", heroku_host, "/2/",
request_headers,
request_body, timeout)
# xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
# xlog.debug("status:%d", status)
if status == 200:
xlog.debug("%s %s%s trace:%s", method, host, path,
response.task.get_trace())
self.last_success_time = time.time()
self.continue_fail_num = 0
self.success_num += 1
else:
if status == 404:
heroku_host = response.ssl_sock.host
xlog.warn("heroku:%s fail", heroku_host)
try:
self.host_manager.remove(heroku_host)
except:
pass
self.last_fail_time = time.time()
self.continue_fail_num += 1
self.fail_num += 1
try:
res = simple_http_client.TxtResponse(content)
except:
return "", 501, {}
res.worker = response.worker
res.task = response.task
return res.body, res.status, res
def stop(self):
logger.info("terminate")
self.connect_manager.set_ssl_created_cb(None)
self.http_dispatcher.stop()
self.connect_manager.stop()
self.ip_manager.stop()
self.running = False
def set_proxy(self, args):
logger.info("set_proxy:%s", args)
self.config.PROXY_ENABLE = args["enable"]
self.config.PROXY_TYPE = args["type"]
self.config.PROXY_HOST = args["host"]
self.config.PROXY_PORT = args["port"]
self.config.PROXY_USER = args["user"]
self.config.PROXY_PASSWD = args["passwd"]
self.config.save()
self.connect_creator.update_config()