def __init__(self): self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 self.account = "" self.password = "" threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.logger, self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudfront_front.json") self.config = Config(config_path) sni_fn = os.path.join(current_path, "sni_list.txt") self.host_manager = host_manager.HostManager(fn=sni_fn, max_size=120) ca_certs = os.path.join(current_path, "cacert.pem") openssl_context = SSLContext(logger, ca_certs=ca_certs, support_http2=True) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudfront_ip_range.txt") ) self.ip_manager = IpManager( logger, self.config, ip_source, check_local_network, self.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudfront_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {}
def __init__(self): self.running = True self.logger = logger self.config = config self.host_manager = host_manager.HostManager(self.config, logger) ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=[ 'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA" ]) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.appid_manager = AppidManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) self.ipv4_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "ip_range.txt")) self.ipv6_source = Ipv6PoolSource( logger, self.config, os.path.join(current_path, "ipv6_list.txt")) self.ip_source = IpCombineSource(logger, self.config, self.ipv4_source, self.ipv6_source) self.ip_manager = IpManager(logger, self.config, self.ip_source, check_local_network, self.check_ip.check_ip, None, os.path.join(module_data_path, "good_ip.txt"), scan_ip_log=None) self.appid_manager.check_api = self.check_ip.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt")) self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {} self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self): self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) if not os.path.isdir(module_data_path): os.mkdir(module_data_path) if not os.path.isdir(tls_certs_path): os.mkdir(tls_certs_path) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = connect_creator.ConnectCreator( logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.account = "" self.password = ""
def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt") ) self.ip_manager = IpManager( logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {}
def start(self): self.running = True ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=[ 'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA" ]) self.appid_manager = AppidManager(self.config, logger) self.host_manager = host_manager.HostManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) #self.ip_checker = CheckIp(logger, self.config, self.connect_creator) self.ipv6_source = Ipv6PoolSource(logger, self.config, "ipv6_list.txt") self.ip_source = self.ipv6_source self.ip_manager = IpManager(logger, self.config, self.ip_source, check_local_network, None, os.path.join(data_path, "good_ip.txt"), scan_ip_log=None) #self.appid_manager.check_api = self.ip_checker.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def __init__(self): self.running = True self.last_host = "center.xx-net.org" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) self.light_config = Config(config_path) self.light_config.dispather_min_workers = 1 self.light_config.max_good_ip_num = 10 ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt")) self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {}
def start(self): self.running = True ca_certs = 'cacert.pem' self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, protocol="TLSv1_2" #cipher_suites=[b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA", b"!ECDHE-RSA-AES128-GCM-SHA256", # b"!AES128-GCM-SHA256", b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA"] ) self.appid_manager = AppidManager(self.config, logger) self.host_manager = host_manager.HostManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) # self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator) self.ip_manager = IpManager(logger, self.config, check_local_network, None, 'good_ip.txt', scan_ip_log=None) # self.appid_manager.check_api = self.ip_checker.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
def __init__(self): self.running = True self.logger = logger self.config = config self.host_manager = host_manager.HostManager(self.config, logger) ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=['ALL', "!RC4-SHA","!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA"] ) self.connect_creator = ConnectCreator( logger, self.config, self.openssl_context, self.host_manager) self.appid_manager = AppidManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) self.ipv4_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "ip_range.txt") ) self.ipv6_source = Ipv6PoolSource( logger, self.config, os.path.join(current_path, "ipv6_list.txt") ) self.ip_source = IpCombineSource( logger, self.config, self.ipv4_source, self.ipv6_source ) self.ip_manager = IpManager( logger, self.config, self.ip_source, check_local_network, self.check_ip.check_ip, None, os.path.join(module_data_path, "good_ip.txt"), scan_ip_log=None) self.appid_manager.check_api = self.check_ip.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher( logger, self.config, self.ip_manager, self.connect_manager )
def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt") ) self.ip_manager = IpManager( logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {} self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self): self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 self.account = "" self.password = "" threading.Thread(target=self.debug_data_clearup_thread).start()
def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.logger, self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager)
class Front(object): name = "gae_front" def __init__(self): self.running = True self.logger = logger self.config = config self.host_manager = host_manager.HostManager(self.config, logger) ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=['ALL', "!RC4-SHA","!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA"] ) self.connect_creator = ConnectCreator( logger, self.config, self.openssl_context, self.host_manager) self.appid_manager = AppidManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) self.ipv4_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "ip_range.txt") ) self.ipv6_source = Ipv6PoolSource( logger, self.config, os.path.join(current_path, "ipv6_list.txt") ) self.ip_source = IpCombineSource( logger, self.config, self.ipv4_source, self.ipv6_source ) self.ip_manager = IpManager( logger, self.config, self.ip_source, check_local_network, self.check_ip.check_ip, None, os.path.join(module_data_path, "good_ip.txt"), scan_ip_log=None) self.appid_manager.check_api = self.check_ip.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher( logger, self.config, self.ip_manager, self.connect_manager ) def get_dispatcher(self): return self.http_dispatcher def request(self, method, host, path="/", headers={}, data="", timeout=120): response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) return response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "heroku_front" def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.logger, self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) def get_dispatcher(self, host=None): if len(self.host_manager.appids) == 0: return None return self.http_dispatcher def set_ips(self, ips): self.ip_manager.set_ips(ips) def _request(self, method, host, path="/", headers={}, data="", timeout=40): try: response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: return "", 500, {} status = response.status if status != 200: self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) content = response.task.read_all() # self.logger.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace()) return content, status, response except Exception as e: self.logger.exception("front request %s %s%s fail:%s", method, host, path, e) return "", 500, {} def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40): schema = "http" # force schema to http, avoid cert fail on heroku curl. # and all x-server provide ipv4 access url = schema + "://" + host + path payloads = ['%s %s HTTP/1.1\r\n' % (method, url)] for k in headers: v = headers[k] payloads.append('%s: %s\r\n' % (k, v)) head_payload = "".join(payloads) request_body = '%s%s%s%s' % \ ((struct.pack('!H', len(head_payload)), head_payload, struct.pack('!I', len(data)), data)) request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'} heroku_host = "" content, status, response = self._request( "POST", heroku_host, "/2/", request_headers, request_body, timeout) # self.logger.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-')) # self.logger.debug("status:%d", status) if hasattr(response, "task"): self.logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) else: self.logger.debug("%s %s%s status:%d", method, host, path, status) if status == 404: heroku_host = response.ssl_sock.host self.logger.warn("heroku appid:%s fail", heroku_host) try: self.host_manager.remove(heroku_host) except: pass return "", 501, {} if not content: return "", 501, {} try: res = simple_http_client.TxtResponse(content) except Exception as e: self.logger.warn("decode %s response except:%r", content, e) return "", 501, {} res.worker = response.worker res.task = response.task return res.body, res.status, res def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "tls_relay_front" def __init__(self): self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.account = "" self.password = "" def get_dispatcher(self, host=None): return self.http_dispatcher def set_x_tunnel_account(self, account, password): self.account = account self.password = password def set_ips(self, ips): if not ips: return host_info = {} ca_certs = [] ipss = [] for ip in ips: dat = ips[ip] ca_cert = dat["ca_crt"] sni = dat["sni"] host_info[ip] = {"sni": sni, "ca_crt": ca_cert} if ca_cert not in ca_certs: ca_certs.append(ca_cert) ipss.append(ip) self.ip_manager.update_ips(ipss) self.ip_manager.save(True) self.host_manager.set_host(host_info) ca_content = "\n\n".join(ca_certs) with open(self.ca_cert_fn, "w") as fd: fd.write(ca_content) self.openssl_context.set_ca(self.ca_cert_fn) self.logger.info("set_ips:%s", ",".join(ipss)) def request(self, method, host, path="/", headers={}, data="", timeout=120): headers = dict(headers) headers["XX-Account"] = self.account response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status content = response.task.read_all() if status == 200: logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) else: logger.warn("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "heroku_front" def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start() def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def worker_num(self): return len(self.http_dispatcher.workers) def set_ips(self, ips): self.ip_manager.set_ips(ips) def get_score(self, host=None): if not self.host_manager.appids: return None now = time.time() if now - self.last_fail_time < self.config.front_continue_fail_block and \ self.continue_fail_num > self.config.front_continue_fail_num: return None worker = self.http_dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() def _request(self, method, host, path="/", headers={}, data="", timeout=40): try: response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: return "", 500, {} status = response.status if status != 200: xlog.warn("front request %s %s%s fail, status:%d", method, host, path, status) content = response.task.read_all() # xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace()) return content, status, response except Exception as e: xlog.exception("front request %s %s%s fail:%r", method, host, path, e) return "", 500, {} def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40): # change top domain to xx-net.net # this domain bypass the cloudflare front for ipv4 #p = host.find(".") #host_sub = host[:p] #host = host_sub + ".xx-net.net" schema = "http" # force schema to http, avoid cert fail on heroku curl. # and all x-server provide ipv4 access url = schema + "://" + host + path payloads = ['%s %s HTTP/1.1\r\n' % (method, url)] for k in headers: v = headers[k] payloads.append('%s: %s\r\n' % (k, v)) head_payload = "".join(payloads) request_body = '%s%s%s%s' % \ ((struct.pack('!H', len(head_payload)), head_payload, struct.pack('!I', len(data)), data)) request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'} heroku_host = "" content, status, response = self._request( "POST", heroku_host, "/2/", request_headers, request_body, timeout) # xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-')) # xlog.debug("status:%d", status) if status == 200: xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace()) self.last_success_time = time.time() self.continue_fail_num = 0 self.success_num += 1 else: if status == 404: heroku_host = response.ssl_sock.host xlog.warn("heroku:%s fail", heroku_host) try: self.host_manager.remove(heroku_host) except: pass self.last_fail_time = time.time() self.continue_fail_num += 1 self.fail_num += 1 try: res = simple_http_client.TxtResponse(content) except: return "", 501, {} res.worker = response.worker res.task = response.task return res.body, res.status, res def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "cloudfront_front" def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudfront_front.json") self.config = Config(config_path) sni_fn = os.path.join(current_path, "sni_list.txt") self.host_manager = host_manager.HostManager(fn=sni_fn, max_size=120) ca_certs = os.path.join(current_path, "cacert.pem") openssl_context = SSLContext(logger, ca_certs=ca_certs, support_http2=True) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudfront_ip_range.txt") ) self.ip_manager = IpManager( logger, self.config, ip_source, check_local_network, self.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudfront_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {} def check_ip(self, ip): sni, host = self.host_manager.get_sni_host(ip) host = "scan1.xx-net.net" return self.ip_checker.check_ip(ip, sni=sni, host=host) def get_dispatcher(self, host=None): if host is None: host = self.last_host else: self.last_host = host if host not in self.dispatchs: http_dispatcher = HttpsDispatcher( logger, self.config, self.ip_manager, self.connect_manager ) self.dispatchs[host] = http_dispatcher dispatcher = self.dispatchs[host] return dispatcher def request(self, method, host, path="/", headers={}, data="", timeout=120): dispatcher = self.get_dispatcher(host) headers = dict(headers) response = dispatcher.request(method, host, path, headers, data, timeout=timeout) if not response: self.logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status content = response.task.read_all() if status == 200: self.logger.debug("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status, response.task.get_trace()) else: self.logger.warn("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) for host in self.dispatchs: dispatcher = self.dispatchs[host] dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "gae_front" def __init__(self): self.logger = logger self.config = config def start(self): self.running = True ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=[ 'ALL', "!RC4-SHA", "!ECDHE-RSA-RC4-SHA", "!ECDHE-RSA-AES128-GCM-SHA256", "!AES128-GCM-SHA256", "!ECDHE-RSA-AES128-SHA", "!AES128-SHA" ]) self.appid_manager = AppidManager(self.config, logger) self.host_manager = host_manager.HostManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) #self.ip_checker = CheckIp(logger, self.config, self.connect_creator) self.ipv6_source = Ipv6PoolSource(logger, self.config, "ipv6_list.txt") self.ip_source = self.ipv6_source self.ip_manager = IpManager(logger, self.config, self.ip_source, check_local_network, None, os.path.join(data_path, "good_ip.txt"), scan_ip_log=None) #self.appid_manager.check_api = self.ip_checker.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) def request(self, method, host, path="/", headers={}, data="", timeout=120): response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) return response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False
class Front(object): name = "tls_relay_front" def __init__(self): self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 self.account = "" self.password = "" threading.Thread(target=self.debug_data_clearup_thread).start() def set_x_tunnel_account(self, account, password): self.account = account self.password = password def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def worker_num(self): return len(self.http_dispatcher.workers) def set_ips(self, ips): if not ips: return host_info = {} ca_certs = [] ipss = [] for ip in ips: dat = ips[ip] ca_cert = dat["ca_crt"] sni = dat["sni"] host_info[ip] = {"sni": sni, "ca_crt": ca_cert} if ca_cert not in ca_certs: ca_certs.append(ca_cert) ipss.append(ip) self.ip_manager.update_ips(ipss) self.ip_manager.save(True) self.host_manager.set_host(host_info) ca_content = "\n\n".join(ca_certs) with open(self.ca_cert_fn, "w") as fd: fd.write(ca_content) self.openssl_context.set_ca(self.ca_cert_fn) self.logger.info("set_ips:%s", ",".join(ipss)) def get_score(self, host=None): now = time.time() if now - self.last_fail_time < self.config.front_continue_fail_block and \ self.continue_fail_num > self.config.front_continue_fail_num: return None worker = self.http_dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() def request(self, method, host, path="/", headers={}, data="", timeout=120): headers = dict(headers) headers["XX-Account"] = self.account response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status if status not in [200, 405]: # logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() else: self.success_num += 1 self.continue_fail_num = 0 content = response.task.read_all() if status == 200: logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) else: logger.warn("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "heroku_front" def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start() def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def worker_num(self): return len(self.http_dispatcher.workers) def set_ips(self, ips): self.ip_manager.set_ips(ips) def get_score(self, host=None): if not self.host_manager.appids: return None now = time.time() if now - self.last_fail_time < self.config.front_continue_fail_block and \ self.continue_fail_num > self.config.front_continue_fail_num: return None worker = self.http_dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() def _request(self, method, host, path="/", headers={}, data="", timeout=40): try: response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) status = response.status if status != 200: xlog.warn("front request %s %s%s fail, status:%d", method, host, path, status) content = response.task.read_all() # xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace()) return content, status, response except Exception as e: xlog.exception("front request %s %s%s fail:%r", method, host, path, e) return "", 500, {} def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40): # change top domain to xx-net.net # this domain bypass the cloudflare front for ipv4 p = host.find(".") host_sub = host[:p] host = host_sub + ".xx-net.net" schema = "http" # force schema to http, avoid cert fail on heroku curl. # and all x-server provide ipv4 access url = schema + "://" + host + path payloads = ['%s %s HTTP/1.1\r\n' % (method, url)] for k in headers: v = headers[k] payloads.append('%s: %s\r\n' % (k, v)) head_payload = "".join(payloads) request_body = '%s%s%s%s' % \ ((struct.pack('!H', len(head_payload)), head_payload, struct.pack('!I', len(data)), data)) request_headers = { 'Content-Length': len(data), 'Content-Type': 'application/octet-stream' } heroku_host = "" content, status, response = self._request("POST", heroku_host, "/2/", request_headers, request_body, timeout) # xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-')) # xlog.debug("status:%d", status) if status == 200: xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace()) self.last_success_time = time.time() self.continue_fail_num = 0 self.success_num += 1 else: if status == 404: heroku_host = response.ssl_sock.host xlog.warn("heroku:%s fail", heroku_host) try: self.host_manager.remove(heroku_host) except: pass self.last_fail_time = time.time() self.continue_fail_num += 1 self.fail_num += 1 try: res = simple_http_client.TxtResponse(content) except: return "", 501, {} res.worker = response.worker res.task = response.task return res.body, res.status, res def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "heroku_front" def __init__(self): self.logger = logger config_path = os.path.join(module_data_path, "heroku_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") self.host_manager = host_manager.HostManager(self.logger, self.config.appids) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = None default_ip_list_fn = os.path.join(current_path, "good_ip.txt") ip_list_fn = os.path.join(module_data_path, "heroku_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) def get_dispatcher(self, host=None): return self.http_dispatcher def set_ips(self, ips): self.ip_manager.set_ips(ips) def _request(self, method, host, path="/", headers={}, data="", timeout=40): try: response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: return "", 500, {} status = response.status if status != 200: self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) content = response.task.read_all() # self.logger.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace()) return content, status, response except Exception as e: self.logger.exception("front request %s %s%s fail:%s", method, host, path, e) return "", 500, {} def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40): schema = "http" # force schema to http, avoid cert fail on heroku curl. # and all x-server provide ipv4 access url = schema + "://" + host + path payloads = ['%s %s HTTP/1.1\r\n' % (method, url)] for k in headers: v = headers[k] payloads.append('%s: %s\r\n' % (k, v)) head_payload = "".join(payloads) request_body = '%s%s%s%s' % \ ((struct.pack('!H', len(head_payload)), head_payload, struct.pack('!I', len(data)), data)) request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'} heroku_host = "" content, status, response = self._request( "POST", heroku_host, "/2/", request_headers, request_body, timeout) # self.logger.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-')) # self.logger.debug("status:%d", status) if hasattr(response, "task"): self.logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) else: self.logger.debug("%s %s%s status:%d", method, host, path, status) if status == 404: heroku_host = response.ssl_sock.host self.logger.warn("heroku appid:%s fail", heroku_host) try: self.host_manager.remove(heroku_host) except: pass return "", 501, {} if not content: return "", 501, {} try: res = simple_http_client.TxtResponse(content) except Exception as e: self.logger.warn("decode %s response except:%r", content, e) return "", 501, {} res.worker = response.worker res.task = response.task return res.body, res.status, res def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "cloudflare_front" def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt")) self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {} self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start() def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def init_host_dispatcher(self, host): if host not in self.dispatchs: http_dispatcher = HttpsDispatcher( logger, self.config, self.ip_manager, self.connect_manager, http2worker=CloudflareHttp2Worker) self.dispatchs[host] = http_dispatcher def worker_num(self): host = self.last_host self.init_host_dispatcher(host) dispatcher = self.dispatchs[host] return len(dispatcher.workers) def get_score(self, host=None): now = time.time() if now - self.last_fail_time < 60 and \ self.continue_fail_num > 10: return None if host is None: host = self.last_host self.init_host_dispatcher(host) dispatcher = self.dispatchs[host] worker = dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() * 10 def request(self, method, host, path="/", headers={}, data="", timeout=120): self.init_host_dispatcher(host) self.last_host = host dispatcher = self.dispatchs[host] response = dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: self.logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status if status not in [200, 405]: # self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() else: self.success_num += 1 self.continue_fail_num = 0 content = response.task.read_all() if status == 200: self.logger.debug("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status, response.task.get_trace()) else: self.logger.warn("%s %s%s status:%d trace:%s", method, response.worker.ssl_sock.host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) for host in self.dispatchs: dispatcher = self.dispatchs[host] dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "gae_front" def __init__(self): self.logger = logger self.config = config def start(self): self.running = True ca_certs = os.path.join(current_path, "cacert.pem") self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, cipher_suites=[ b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA", b"!ECDHE-RSA-AES128-GCM-SHA256", b"!AES128-GCM-SHA256", b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA" ]) self.appid_manager = AppidManager(self.config, logger) self.host_manager = host_manager.HostManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator) self.ipv4_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "ip_range.txt")) self.ipv6_source = Ipv6PoolSource( logger, self.config, os.path.join(current_path, "ipv6_list.txt")) self.ip_source = IpCombineSource(logger, self.config, self.ipv4_source, self.ipv6_source) self.ip_manager = IpManager(logger, self.config, self.ip_source, check_local_network, self.check_ip, None, os.path.join(module_data_path, "good_ip.txt"), scan_ip_log=None) self.appid_manager.check_api = self.ip_checker.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) def check_ip(self, ip): sni = self.host_manager.sni_manager.get() host = self.config.check_ip_host return self.ip_checker.check_ip(ip, sni=sni, host=host) def get_dispatcher(self): return self.http_dispatcher def request(self, method, host, path=b"/", headers={}, data="", timeout=120): response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) return response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "cloudflare_front" def __init__(self): self.running = True self.last_host = "center.xx-net.net" self.logger = logger config_path = os.path.join(module_data_path, "cloudflare_front.json") self.config = Config(config_path) ca_certs = os.path.join(current_path, "cacert.pem") default_domain_fn = os.path.join(current_path, "front_domains.json") domain_fn = os.path.join(module_data_path, "cloudflare_domains.json") self.host_manager = host_manager.HostManager(self.config, logger, default_domain_fn, domain_fn, self) openssl_context = SSLContext(logger, ca_certs=ca_certs) self.connect_creator = ConnectCreator(logger, self.config, openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = Ipv4RangeSource( logger, self.config, os.path.join(current_path, "ip_range.txt"), os.path.join(module_data_path, "cloudflare_ip_range.txt") ) self.ip_manager = IpManager( logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, os.path.join(current_path, "good_ip.txt"), os.path.join(module_data_path, "cloudflare_ip_list.txt"), scan_ip_log=None) self.connect_manager = ConnectManager( logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.dispatchs = {} self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 threading.Thread(target=self.debug_data_clearup_thread).start() def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def init_host_dispatcher(self, host): if host not in self.dispatchs: http_dispatcher = HttpsDispatcher( logger, self.config, self.ip_manager, self.connect_manager, http2worker=CloudflareHttp2Worker) self.dispatchs[host] = http_dispatcher def worker_num(self): host = self.last_host self.init_host_dispatcher(host) dispatcher = self.dispatchs[host] return len(dispatcher.workers) def get_score(self, host=None): now = time.time() if now - self.last_fail_time < 60 and \ self.continue_fail_num > 10: return None if host is None: host = self.last_host self.init_host_dispatcher(host) dispatcher = self.dispatchs[host] worker = dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() * 10 def request(self, method, host, path="/", headers={}, data="", timeout=120): self.init_host_dispatcher(host) self.last_host = host dispatcher = self.dispatchs[host] response = dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: self.logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status if status not in [200, 405]: # self.logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() else: self.success_num += 1 self.continue_fail_num = 0 content = response.task.read_all() if status == 200: self.logger.debug("%s %s%s status:%d trace:%s", method, response.worker.host, path, status, response.task.get_trace()) else: self.logger.warn("%s %s%s status:%d trace:%s", method, response.worker.host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) for host in self.dispatchs: dispatcher = self.dispatchs[host] dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "tls_relay_front" def __init__(self): self.success_num = 0 self.fail_num = 0 self.continue_fail_num = 0 self.last_fail_time = 0 self.running = True self.logger = logger config_path = os.path.join(module_data_path, "tls_relay.json") self.config = Config(config_path) self.ca_cert_fn = os.path.join(module_data_path, "tls_relay_CA.crt") self.openssl_context = SSLContext(logger) if os.path.isfile(self.ca_cert_fn): self.openssl_context.set_ca(self.ca_cert_fn) host_fn = os.path.join(module_data_path, "tls_host.json") self.host_manager = host_manager.HostManager(host_fn) self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) self.check_ip = CheckIp(xlog.null, self.config, self.connect_creator) ip_source = IpSimpleSource(self.config.ip_source_ips) default_ip_list_fn = "" ip_list_fn = os.path.join(module_data_path, "tls_relay_ip_list.txt") self.ip_manager = IpManager(logger, self.config, ip_source, check_local_network, self.check_ip.check_ip, default_ip_list_fn, ip_list_fn, scan_ip_log=None) for ip in self.config.ip_source_ips: self.ip_manager.add_ip(ip, 100) self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) self.rtts = collections.deque([(0, time.time())]) self.rtts_lock = threading.Lock() self.traffics = collections.deque() self.traffics_lock = threading.Lock() self.recent_sent = 0 self.recent_received = 0 self.total_sent = 0 self.total_received = 0 self.account = "" self.password = "" threading.Thread(target=self.debug_data_clearup_thread).start() def set_x_tunnel_account(self, account, password): self.account = account self.password = password def log_debug_data(self, rtt, sent, received): now = time.time() self.rtts.append((rtt, now)) with self.traffics_lock: self.traffics.append((sent, received, now)) self.recent_sent += sent self.recent_received += received self.total_sent += sent self.total_received += received def get_rtt(self): now = time.time() while len(self.rtts) > 1: with self.rtts_lock: rtt, log_time = rtt_log = max(self.rtts) if now - log_time > 5: self.rtts.remove(rtt_log) continue return rtt return self.rtts[0][0] def debug_data_clearup_thread(self): while self.running: now = time.time() with self.rtts_lock: if len(self.rtts) > 1 and now - self.rtts[0][-1] > 5: self.rtts.popleft() with self.traffics_lock: if self.traffics and now - self.traffics[0][-1] > 60: sent, received, _ = self.traffics.popleft() self.recent_sent -= sent self.recent_received -= received time.sleep(1) def worker_num(self): return len(self.http_dispatcher.workers) def set_ips(self, ips): if not ips: return host_info = {} ca_certs = [] ipss = [] for ip in ips: dat = ips[ip] ca_cert = dat["ca_crt"] sni = dat["sni"] host_info[ip] = {"sni":sni, "ca_crt":ca_cert} if ca_cert not in ca_certs: ca_certs.append(ca_cert) ipss.append(ip) self.ip_manager.update_ips(ipss) self.ip_manager.save(True) self.host_manager.set_host(host_info) ca_content = "\n\n".join(ca_certs) with open(self.ca_cert_fn, "w") as fd: fd.write(ca_content) self.openssl_context.set_ca(self.ca_cert_fn) self.logger.info("set_ips:%s", ",".join(ipss)) def get_score(self, host=None): now = time.time() if now - self.last_fail_time < self.config.front_continue_fail_block and \ self.continue_fail_num > self.config.front_continue_fail_num: return None worker = self.http_dispatcher.get_worker(nowait=True) if not worker: return None return worker.get_score() def request(self, method, host, path="/", headers={}, data="", timeout=120): headers = dict(headers) headers["XX-Account"] = self.account response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) if not response: logger.warn("req %s get response timeout", path) return "", 602, {} status = response.status if status not in [200, 405]: # logger.warn("front request %s %s%s fail, status:%d", method, host, path, status) self.fail_num += 1 self.continue_fail_num += 1 self.last_fail_time = time.time() else: self.success_num += 1 self.continue_fail_num = 0 content = response.task.read_all() if status == 200: logger.debug("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) else: logger.warn("%s %s%s status:%d trace:%s", method, host, path, status, response.task.get_trace()) return content, status, response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()
class Front(object): name = "gae_front" def __init__(self): self.logger = logger self.config = config def start(self): self.running = True ca_certs = 'cacert.pem' self.openssl_context = SSLContext( logger, ca_certs=ca_certs, support_http2=config.support_http2, protocol="TLSv1_2" #cipher_suites=[b'ALL', b"!RC4-SHA", b"!ECDHE-RSA-RC4-SHA", b"!ECDHE-RSA-AES128-GCM-SHA256", # b"!AES128-GCM-SHA256", b"!ECDHE-RSA-AES128-SHA", b"!AES128-SHA"] ) self.appid_manager = AppidManager(self.config, logger) self.host_manager = host_manager.HostManager(self.config, logger) self.host_manager.appid_manager = self.appid_manager self.connect_creator = ConnectCreator(logger, self.config, self.openssl_context, self.host_manager) # self.ip_checker = CheckIp(xlog.null, self.config, self.connect_creator) self.ip_manager = IpManager(logger, self.config, check_local_network, None, 'good_ip.txt', scan_ip_log=None) # self.appid_manager.check_api = self.ip_checker.check_ip self.appid_manager.ip_manager = self.ip_manager self.connect_manager = ConnectManager(logger, self.config, self.connect_creator, self.ip_manager, check_local_network) self.http_dispatcher = HttpsDispatcher(logger, self.config, self.ip_manager, self.connect_manager) def get_dispatcher(self): return self.http_dispatcher def request(self, method, host, path=b"/", headers={}, data="", timeout=120): response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout) return response def stop(self): logger.info("terminate") self.connect_manager.set_ssl_created_cb(None) self.http_dispatcher.stop() self.connect_manager.stop() self.ip_manager.stop() self.running = False def set_proxy(self, args): logger.info("set_proxy:%s", args) self.config.PROXY_ENABLE = args["enable"] self.config.PROXY_TYPE = args["type"] self.config.PROXY_HOST = args["host"] self.config.PROXY_PORT = args["port"] self.config.PROXY_USER = args["user"] self.config.PROXY_PASSWD = args["passwd"] self.config.save() self.connect_creator.update_config()