Example #1
0
def request(url, retry=0, timeout=30):
    if retry == 0:
        if int(config.get(["proxy", "enable"], 0)):
            client = simple_http_client.Client(proxy={
                "type":
                config.get(["proxy", "type"], ""),
                "host":
                config.get(["proxy", "host"], ""),
                "port":
                int(config.get(["proxy", "port"], 0)),
                "user":
                config.get(["proxy", "user"], ""),
                "pass":
                config.get(["proxy", "passwd"], ""),
            },
                                               timeout=timeout)
        else:
            client = simple_http_client.Client(timeout=timeout)
    else:
        cert = os.path.join(data_root, "gae_proxy", "CA.crt")
        client = simple_http_client.Client(proxy={
            "type": "http",
            "host": "127.0.0.1",
            "port": 8087,
            "user": None,
            "pass": None
        },
                                           timeout=timeout,
                                           cert=cert)

    res = client.request("GET", url, read_payload=False)
    return res
Example #2
0
def request(url, retry=0, timeout=30):
    if retry == 0:
        if config.global_proxy_enable == 1:
            client = simple_http_client.Client(proxy={
                "type":
                config.global_proxy_type,
                "host":
                config.global_proxy_host,
                "port":
                config.global_proxy_port,
                "user":
                config.global_proxy_username,
                "pass":
                config.global_proxy_password,
            },
                                               timeout=timeout)
        else:
            client = simple_http_client.Client(timeout=timeout)
    else:
        cert = os.path.join(data_root, "gae_proxy", "CA.crt")
        client = simple_http_client.Client(proxy={
            "type": "http",
            "host": "127.0.0.1",
            "port": 8087,
            "user": None,
            "pass": None
        },
                                           timeout=timeout,
                                           cert=cert)

    res = client.request("GET", url, read_payload=False)
    return res
Example #3
0
    def update_front_domains(self):
        next_update_time = time.time()
        while self.front.running:
            if time.time() < next_update_time:
                time.sleep(4)
                continue

            try:
                timeout = 30
                if self.config.PROXY_ENABLE:
                    client = simple_http_client.Client(proxy={
                        "type":
                        self.config.PROXY_TYPE,
                        "host":
                        self.config.PROXY_HOST,
                        "port":
                        self.config.PROXY_PORT,
                        "user":
                        self.config.PROXY_USER,
                        "pass":
                        self.config.PROXY_PASSWD,
                    },
                                                       timeout=timeout)
                else:
                    client = simple_http_client.Client(timeout=timeout)

                url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
                response = client.request("GET", url)
                if not response or response.status != 200:
                    if response:
                        self.logger.warn("update front domains fail:%d",
                                         response.status)
                    next_update_time = time.time() + (1800)
                    continue

                content = response.text
                if isinstance(content, memoryview):
                    content = content.tobytes()

                need_update = True
                front_domains_fn = self.fn
                if os.path.exists(front_domains_fn):
                    with open(front_domains_fn, "r") as fd:
                        old_content = fd.read()
                        if content == old_content:
                            need_update = False

                if need_update:
                    with open(front_domains_fn, "w") as fd:
                        fd.write(content.decode("utf-8"))
                    self.load()

                    self.logger.info(
                        "updated cloudflare front domains from github.")

                next_update_time = time.time() + (4 * 3600)
            except Exception as e:
                next_update_time = time.time() + (1800)
                self.logger.exception(
                    "updated cloudflare front domains from github fail:%r", e)
Example #4
0
    def update_front_domains():
        next_update_time = time.time()
        while connect_control.keep_running:
            if time.time() < next_update_time:
                time.sleep(4)
                continue

            try:
                timeout = 30
                if config.getint("proxy", "enable", 0):
                    client = simple_http_client.Client(proxy={
                        "type":
                        config.get("proxy", "type", ""),
                        "host":
                        config.get("proxy", "host", ""),
                        "port":
                        int(config.get("proxy", "port", "0")),
                        "user":
                        config.get("proxy", "user", ""),
                        "pass":
                        config.get("proxy", "passwd", ""),
                    },
                                                       timeout=timeout)
                else:
                    client = simple_http_client.Client(timeout=timeout)

                url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
                response = client.request("GET", url)
                if response.status != 200:
                    xlog.warn("update front domains fail:%d", response.status)
                    raise Exception("status:%r", response.status)

                need_update = True
                front_domains_fn = os.path.join(config.DATA_PATH,
                                                "front_domains.json")
                if os.path.exists(front_domains_fn):
                    with open(front_domains_fn, "r") as fd:
                        old_content = fd.read()
                        if response.text == old_content:
                            need_update = False

                if need_update:
                    with open(front_domains_fn, "w") as fd:
                        fd.write(response.text)
                    check_ip.update_front_domains()

                next_update_time = time.time() + (4 * 3600)
                xlog.info("updated cloudflare front domains from github.")
            except Exception as e:
                next_update_time = time.time() + (1800)
                xlog.debug(
                    "updated cloudflare front domains from github fail:%r", e)
Example #5
0
 def test_get(self):
     client = simple_http_client.Client(timeout=10)
     url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/x_tunnel/local/cloudflare_front/front_domains.json"
     res = client.request("GET", url)
     self.assertEquals(res.status, 200)
     content = utils.to_str(res.text)
     data = json.loads(content)
     print(data)
Example #6
0
    def test_get2(self):
        url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/update_v4.txt"

        client = simple_http_client.Client(timeout=10)
        res = client.request("GET", url)
        self.assertEqual(res.status, 200)
        content = utils.to_str(res.text)
        print(content)
Example #7
0
    def get_connection(self):
        while len(self.connections):
            try:
                [client, last_query_time] = self.connections.pop()
                if time.time() - last_query_time < self.connection_timeout:
                    return client
            except:
                pass

        if g.config.PROXY_ENABLE == 1:
            return simple_http_client.Client(proxy={
                "type": g.config.PROXY_TYPE,
                "host": g.config.PROXY_HOST,
                "port": g.config.PROXY_PORT,
                "user": g.config.PROXY_USER,
                "pass": g.config.PROXY_PASSWD,
            }, timeout=self.timeout)
        else:
            return simple_http_client.Client(timeout=self.timeout)
Example #8
0
def request(url, retry=0, timeout=30):
    cert = os.path.join(data_path, "CA.crt")
    client = simple_http_client.Client(proxy={
        "type": "http",
        "host": "127.0.0.1",
        "port": 8087,
        "user": None,
        "pass": None
    }, timeout=timeout, cert=cert)

    res = client.request("GET", url, read_payload=False)
    return res
Example #9
0
    def test_get(self):
        server = simple_http_server.HTTPServer(
            ('', 8880), simple_http_server.TestHttpServer, ".")
        server.start()

        client = simple_http_client.Client(timeout=5)
        url = "http://localhost:8880/test"
        res = client.request("GET", url)
        self.assertEqual(res.status, 200)
        content = utils.to_str(res.text)
        print(content)

        server.shutdown()
Example #10
0
    def __init__(self, type="IPv4"):
        self.type = type
        self.urls = []
        self._checking_lock = threading.Lock()
        self._checking_num = 0
        self.network_stat = "unknown"
        self.last_check_time = 0
        self.continue_fail_count = 0

        if config.PROXY_USER:
            self.proxy = "%s://%s:%s@%s:%d" % \
                (config.PROXY_TYPE, config.PROXY_USER, config.PROXY_PASSWD, config.PROXY_HOST, config.PROXY_PORT)
        else:
            self.proxy = "%s://%s:%d" % \
                (config.PROXY_TYPE, config.PROXY_HOST, config.PROXY_PORT)

        self.http_client = simple_http_client.Client(self.proxy, timeout=30)
Example #11
0
    def test_get_bulk(self):
        timeout = 5
        client = simple_http_client.Client(timeout=timeout)
        url = "https://raw.githubusercontent.com/XX-net/XX-Net/master/code/default/update_v4.txt"

        start_time = time.time()
        req = client.request("GET", url, read_payload=False)
        self.assertIsNotNone(req)

        tp = tempfile.gettempdir()
        filename = os.path.join(tp, "v4.txt")
        if req.chunked:
            downloaded = 0
            with open(filename, 'wb') as fp:
                while True:
                    time_left = timeout - (time.time() - start_time)
                    if time_left < 0:
                        raise Exception("time out")

                    dat = req.read(timeout=time_left)
                    if not dat:
                        break

                    fp.write(dat)
                    downloaded += len(dat)

            return True
        else:
            file_size = int(req.getheader(b'Content-Length', 0))

            left = file_size
            downloaded = 0
            with open(filename, 'wb') as fp:
                while True:
                    chunk_len = min(65536, left)
                    if not chunk_len:
                        break

                    chunk = req.read(chunk_len)
                    if not chunk:
                        break
                    fp.write(chunk)
                    downloaded += len(chunk)
                    left -= len(chunk)
Example #12
0
def test_proxy(type, host, port, user, passwd):
    if not host:
        return False

    if host == "127.0.0.1":
        if port in [8087, 1080]:
            xlog.warn("set LAN Proxy to %s:%d fail.", host, port)
            return False

    client = simple_http_client.Client(proxy={
        "type": type,
        "host": host,
        "port": int(port),
        "user": user if len(user) else None,
        "pass": passwd if len(passwd) else None
    },
                                       timeout=5)

    urls = [
        "https://www.microsoft.com", "https://www.apple.com",
        "https://code.jquery.com", "https://cdn.bootcss.com",
        "https://cdnjs.cloudflare.com"
    ]

    for url in urls:

        header = {
            "user-agent":
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
            "accept": "application/json, text/javascript, */*; q=0.01",
            "accept-encoding": "gzip, deflate, sdch",
            "accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
            "connection": "keep-alive"
        }
        try:
            response = client.request("HEAD", url, header, "")
            if response:
                return True
        except Exception as e:
            xlog.exception("test_proxy %s fail:%r", url, e)
            pass

    return False