Ejemplo n.º 1
0
    def keep_alive_thread(self):
        while self.keep_alive:
            time.sleep(1)
            try:
                sock_list = self.new_conn_pool.get_need_keep_alive(
                    maxtime=self.keep_alive - 3)
                for ssl_sock in sock_list:
                    ssl_sock.close()

                sock_list = self.gae_conn_pool.get_need_keep_alive(
                    maxtime=self.keep_alive - 3)
                for ssl_sock in sock_list:
                    # only keep little alive link.
                    # if you have 25 appid, you can keep 5 alive link.
                    if self.gae_conn_pool.qsize() > max(
                            1,
                            len(appid_manager.working_appid_list) / 2):
                        ssl_sock.close()
                        continue

                    #inactive_time = time.time() -ssl_sock.last_use_time
                    #logging.debug("inactive_time:%d", inactive_time)
                    if self.head_request(ssl_sock):
                        self.save_ssl_connection_for_reuse(ssl_sock)
                    else:
                        ssl_sock.close()

                self.create_more_connection()
            except Exception as e:
                xlog.warn("keep alive except:%r", e)
Ejemplo n.º 2
0
    def save(self):
        CONFIG_USER_FILENAME = os.path.abspath( os.path.join(root_path, 'data', 'gae_proxy', 'config.ini'))
        try:
            f = open(CONFIG_USER_FILENAME, 'w')
            if self.user_special.appid != "":
                f.write("[gae]\n")
                f.write("appid = %s\n" % self.user_special.appid)
                f.write("password = %s\n\n" % self.user_special.password)

            f.write("[proxy]\n")
            f.write("enable = %s\n" % self.user_special.proxy_enable)
            f.write("type = %s\n" % self.user_special.proxy_type)
            f.write("host = %s\n" % self.user_special.proxy_host)
            f.write("port = %s\n" % self.user_special.proxy_port)
            f.write("user = %s\n" % self.user_special.proxy_user)
            f.write("passwd = %s\n\n" % self.user_special.proxy_passwd)

            if self.user_special.host_appengine_mode != "gae":
                f.write("[hosts]\n")
                f.write("appengine.google.com = %s\n" % self.user_special.host_appengine_mode)
                f.write("www.google.com = %s\n\n" % self.user_special.host_appengine_mode)

            f.write("[google_ip]\n")

            if int(self.user_special.auto_adjust_scan_ip_thread_num) != self.DEFAULT_CONFIG.getint('google_ip', 'auto_adjust_scan_ip_thread_num'):
                f.write("auto_adjust_scan_ip_thread_num = %d\n\n" % int(self.user_special.auto_adjust_scan_ip_thread_num))
            if int(self.user_special.scan_ip_thread_num) != self.DEFAULT_CONFIG.getint('google_ip', 'max_scan_ip_thread_num'):
                f.write("max_scan_ip_thread_num = %d\n\n" % int(self.user_special.scan_ip_thread_num))

            if int(self.user_special.use_ipv6) != self.DEFAULT_CONFIG.getint('google_ip', 'use_ipv6'):
                f.write("use_ipv6 = %d\n\n" % int(self.user_special.use_ipv6))

            f.close()
        except:
            xlog.warn("launcher.config save user config fail:%s", CONFIG_USER_FILENAME)
Ejemplo n.º 3
0
def load_tasks():
    tasks = []
    intervals = []

    import imp
    tasks_path = config.tasks_path
    for fileName in os.listdir(tasks_path):
        if fileName.lower().endswith('.py'):
            xlog.info('start load %s' % fileName)
            try:
                with open(os.path.join(tasks_path, fileName), 'rb') as fpy:
                    task = imp.load_source('Auto-tasks_%d_%s' % (len(tasks), fileName), tasks_path, fpy)

                    name = task.name
                    interval = task.run_interval

                    task.init()

                    tasks.append(task)
                    intervals.append(interval)

                    xlog.info('load task %s success.' % name)
            except:
                xlog.warn('load %s fail.' % fileName)
    del imp

    return (tasks, intervals)
Ejemplo n.º 4
0
    def test2(self):
        work_ciphers = ["AES128-SHA"]
        for cipher in self.cipher_list:
            if cipher in work_ciphers:
                continue
            else:
                work_ciphers.append(cipher)

            xlog.debug("%s", cipher)
            cipher_suites = (work_ciphers)

            openssl_context = SSLConnection.context_builder(
                ca_certs=g_cacertfile, cipher_suites=cipher_suites)
            try:
                ssl, _, _ = connect_ssl(self.ip,
                                        openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
                if "gws" not in server_type:
                    work_ciphers.remove(cipher)
            except Exception as e:
                xlog.warn("err:%s", e)
                try:
                    work_ciphers.remove(cipher)
                except:
                    pass

        work_str = ""
        for cipher in work_ciphers:
            work_str += cipher + ":"
        xlog.info("work ciphers:%s", work_str)
Ejemplo n.º 5
0
    def remove_ip_process(self):
        try:
            while connect_control.keep_running:

                try:
                    ip_str = self.to_remove_ip_list.get_nowait()
                except:
                    break

                result = check_ip.test(ip_str)
                if result and result.appspot_ok:
                    self.add_ip(ip_str, result.handshake_time, result.domain,
                                result.server_type)
                    xlog.debug("remove ip process, restore ip:%s", ip_str)
                    continue

                if not check_ip.network_is_ok():
                    self.to_remove_ip_list.put(ip_str)
                    xlog.warn(
                        "network is unreachable. check your network connection."
                    )
                    return

                xlog.info("real remove ip:%s ", ip_str)
                self.iplist_need_save = 1
        finally:
            self.remove_ip_thread_num_lock.acquire()
            self.remove_ip_thread_num -= 1
            self.remove_ip_thread_num_lock.release()
Ejemplo n.º 6
0
    def remove_ip_process(self):
        try:
            while True:

                try:
                    ip_str = self.to_remove_ip_list.get_nowait()
                except:
                    break

                result = check_ip.test(ip_str)
                if result and result.appspot_ok:
                    self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type)
                    xlog.debug("remove ip process, restore ip:%s", ip_str)
                    continue

                if not check_ip.network_is_ok():
                    self.to_remove_ip_list.put(ip_str)
                    xlog.warn("network is unreachable. check your network connection.")
                    return

                xlog.info("real remove ip:%s ", ip_str)
                self.iplist_need_save = 1
        finally:
            self.remove_ip_thread_num_lock.acquire()
            self.remove_ip_thread_num -= 1
            self.remove_ip_thread_num_lock.release()
Ejemplo n.º 7
0
    def do_POST(self):
        try:
            refer = self.headers.getheader('Referer')
            netloc = urlparse.urlparse(refer).netloc
            if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"):
                xlog.warn("web control ref:%s refuse", netloc)
                return
        except:
            pass
        xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path)
        try:
            ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
            if ctype == 'multipart/form-data':
                self.postvars = cgi.parse_multipart(self.rfile, pdict)
            elif ctype == 'application/x-www-form-urlencoded':
                length = int(self.headers.getheader('content-length'))
                self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1)
            else:
                self.postvars = {}
        except:
            self.postvars = {}

        path = urlparse.urlparse(self.path).path
        if path == '/deploy':
            return self.req_deploy_handler()
        elif path == "/config":
            return self.req_config_handler()
        elif path == "/scan_ip":
            return self.req_scan_ip_handler()
        elif path.startswith("/importip"):
            return self.req_importip_handler()
        else:
            self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')
            xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)
Ejemplo n.º 8
0
def fetch(method, host, path, headers, payload, bufsize=8192):
    request_data = '%s %s HTTP/1.1\r\n' % (method, path)
    request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
    request_data += '\r\n'

    ssl_sock = https_manager.get_ssl_connection(host)
    if not ssl_sock:
        return

    ssl_sock.send(request_data.encode())
    payload_len = len(payload)
    start = 0
    while start < payload_len:
        send_size = min(payload_len - start, 65535)
        sended = ssl_sock.send(payload[start:start+send_size])
        start += sended

    response = httplib.HTTPResponse(ssl_sock, buffering=True)

    response.ssl_sock = ssl_sock
    try:
        orig_timeout = ssl_sock.gettimeout()
        ssl_sock.settimeout(90)
        response.begin()
        ssl_sock.settimeout(orig_timeout)
    except httplib.BadStatusLine as e:
        xlog.warn("direct_handler.fetch bad status line:%r", e)
        response = None
    except Exception as e:
        xlog.warn("direct_handler.fetch:%r", e)
    return response
Ejemplo n.º 9
0
def test(ip_str, loop=1):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str, check_cert=False)

    for i in range(loop):

        result = check.check(callback=test_app_head)
        if not result:
            if "gws" in check.result.server_type:
                xlog.warn("ip:%s server_type:%s but appengine check fail.",
                          ip_str, check.result.server_type)

            xlog.warn("check fail")
            #continue
        else:
            xlog.debug("=======app check ok: %s", ip_str)
            check.result.appspot_ok = result

        result = check.check(callback=test_server_type, check_ca=True)
        if not result:
            xlog.debug("test server type fail")
            continue

        check.result.server_type = result
        xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str,
                  check.result.server_type, check.result.domain,
                  check.result.handshake_time)

    return check.result
Ejemplo n.º 10
0
    def load_ip_range(self):
        self.ip_range_map = {}
        self.ip_range_list = []
        self.ip_range_index = []
        self.candidate_amount_ip = 0

        content = self.load_range_content()
        lines = content.splitlines()
        for line in lines:
            if len(line) == 0 or line[0] == '#':
                continue

            try:
                begin, end = ip_utils.split_ip(line)
                nbegin = ip_utils.ip_string_to_num(begin)
                nend = ip_utils.ip_string_to_num(end)
                if not nbegin or not nend or nend < nbegin:
                    xlog.warn("load ip range:%s fail", line)
                    continue
            except Exception as e:
                xlog.exception("load ip range:%s fail:%r", line, e)
                continue

            self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
            self.ip_range_list.append( [nbegin, nend] )
            self.ip_range_index.append(self.candidate_amount_ip)
            num = nend - nbegin
            self.candidate_amount_ip += num
            # print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num

        self.ip_range_index.sort()
Ejemplo n.º 11
0
    def load_ip_range(self):
        self.ip_range_map = {}
        self.ip_range_list = []
        self.ip_range_index = []
        self.candidate_amount_ip = 0

        content = self.load_range_content()
        lines = content.splitlines()
        for line in lines:
            if len(line) == 0 or line[0] == '#':
                continue

            try:
                begin, end = ip_utils.split_ip(line)
                nbegin = ip_utils.ip_string_to_num(begin)
                nend = ip_utils.ip_string_to_num(end)
                if not nbegin or not nend or nend < nbegin:
                    xlog.warn("load ip range:%s fail", line)
                    continue
            except Exception as e:
                xlog.exception("load ip range:%s fail:%r", line, e)
                continue

            self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
            self.ip_range_list.append([nbegin, nend])
            self.ip_range_index.append(self.candidate_amount_ip)
            num = nend - nbegin
            self.candidate_amount_ip += num
            # print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num

        self.ip_range_index.sort()
Ejemplo n.º 12
0
    def check_response(self, response):
        server_type = response.headers.get('Server', "")
        self.logger.debug("status:%d", response.status)
        self.logger.debug("Server type:%s", server_type)

        if response.status not in self.config.check_ip_accept_status:
            return False

        if response.status == 503:
            # out of quota
            if b"gws" not in server_type and b"Google Frontend" not in server_type and b"GFE" not in server_type:
                xlog.warn("503 but server type:%s", server_type)
                return False
            else:
                return True

        try:
            content = response.read()
        except Exception as e:
            if sys.version_info[0] == 3 and (
                    isinstance(e, ConnectionError) or
                    isinstance(e, ConnectionResetError) or
                    isinstance(e, BrokenPipeError)
            ):
                return False

            self.logger.warn("app check except:%r", e)
            return False

        if self.config.check_ip_content not in content:
            self.logger.warn("app check content:%s", content)
            return False

        return True
Ejemplo n.º 13
0
def test(ip_str, loop=1):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str, check_cert=False)

    for i in range(loop):

        result = check.check(callback=test_app_head)
        if not result:
            if "gws" in check.result.server_type:
                xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type)

            xlog.warn("check fail")
            #continue
        else:
            xlog.debug("=======app check ok: %s", ip_str)
            check.result.appspot_ok = result


        result = check.check(callback=test_server_type, check_ca=True)
        if not result:
            xlog.debug("test server type fail")
            continue

        check.result.server_type = result
        xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type,
                     check.result.domain, check.result.handshake_time)

    return check.result
Ejemplo n.º 14
0
    def load(self):
        ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')

        self.USER_CONFIG = ConfigParser.ConfigParser()
        CONFIG_USER_FILENAME = os.path.abspath( os.path.join(root_path, 'data', 'gae_proxy', 'config.ini'))

        self.DEFAULT_CONFIG = ConfigParser.ConfigParser()
        DEFAULT_CONFIG_FILENAME = os.path.abspath( os.path.join(current_path, 'proxy.ini'))

        try:
            if os.path.isfile(CONFIG_USER_FILENAME):
                self.USER_CONFIG.read(CONFIG_USER_FILENAME)
            else:
                return

            if os.path.isfile(DEFAULT_CONFIG_FILENAME):
                self.DEFAULT_CONFIG.read(DEFAULT_CONFIG_FILENAME)
            else:
                return

            try:
                self.user_special.appid = self.USER_CONFIG.get('gae', 'appid')
                self.user_special.password = self.USER_CONFIG.get('gae', 'password')
            except:
                pass

            try:
                self.user_special.host_appengine_mode = self.USER_CONFIG.get('hosts', 'appengine.google.com')
            except:
                pass

            try:
                self.user_special.ip_connect_interval = config.CONFIG.getint('google_ip', 'ip_connect_interval')
            except:
                pass

            try:
                self.user_special.scan_ip_thread_num = config.CONFIG.getint('google_ip', 'max_scan_ip_thread_num')
            except:
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint('google_ip', 'max_scan_ip_thread_num')

            try:
                self.user_special.auto_adjust_scan_ip_thread_num = config.CONFIG.getint('google_ip', 'auto_adjust_scan_ip_thread_num')
            except:
                pass

            try:
                self.user_special.use_ipv6 = config.CONFIG.getint('google_ip', 'use_ipv6')
            except:
                pass

            self.user_special.proxy_enable = self.USER_CONFIG.get('proxy', 'enable')
            self.user_special.proxy_type = self.USER_CONFIG.get('proxy', 'type')
            self.user_special.proxy_host = self.USER_CONFIG.get('proxy', 'host')
            self.user_special.proxy_port = self.USER_CONFIG.get('proxy', 'port')
            self.user_special.proxy_user = self.USER_CONFIG.get('proxy', 'user')
            self.user_special.proxy_passwd = self.USER_CONFIG.get('proxy', 'passwd')

        except Exception as e:
            xlog.warn("User_config.load except:%s", e)
Ejemplo n.º 15
0
    def test2(self):
        work_ciphers = ["AES128-SHA"]
        for cipher in self.cipher_list:
            if cipher in work_ciphers:
                continue
            else:
                work_ciphers.append(cipher)

            xlog.debug("%s", cipher)
            cipher_suites = (work_ciphers)

            openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
            try:
                ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
                if "gws" not in server_type:
                    work_ciphers.remove(cipher)
            except Exception as e:
                xlog.warn("err:%s", e)
                try:
                    work_ciphers.remove(cipher)
                except:
                    pass

        work_str = ""
        for cipher in work_ciphers:
            work_str += cipher + ":"
        xlog.info("work ciphers:%s", work_str)
Ejemplo n.º 16
0
    def _request(self,
                 method,
                 host,
                 path="/",
                 headers={},
                 data="",
                 timeout=40):
        try:
            response = self.http_dispatcher.request(method,
                                                    host,
                                                    path,
                                                    dict(headers),
                                                    data,
                                                    timeout=timeout)
            status = response.status
            if status != 200:
                xlog.warn("front request %s %s%s fail, status:%d", method,
                          host, path, status)

            content = response.task.read_all()
            # xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
            return content, status, response
        except Exception as e:
            xlog.exception("front request %s %s%s fail:%r", method, host, path,
                           e)

            return "", 500, {}
Ejemplo n.º 17
0
def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug('create_ssl_connection fail')
                continue

            if ssl_sock.host == '':
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    raise GAE_Exception(1, "no appid can use")
                headers['Host'] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers['Host']
            else:
                headers['Host'] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.warn('request failed:%s', e)
            if ssl_sock:
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")
Ejemplo n.º 18
0
def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug("create_ssl_connection fail")
                continue

            if ssl_sock.host == "":
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    raise GAE_Exception(1, "no appid can use")
                headers["Host"] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers["Host"]
            else:
                headers["Host"] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.warn("request failed:%s", e)
            if ssl_sock:
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")
Ejemplo n.º 19
0
    def keep_alive_thread(self):
        while self.keep_alive:
            time.sleep(1)
            try:
                sock_list = self.new_conn_pool.get_need_keep_alive(maxtime=self.keep_alive-3)
                for ssl_sock in sock_list:
                    ssl_sock.close()

                sock_list = self.gae_conn_pool.get_need_keep_alive(maxtime=self.keep_alive-3)
                for ssl_sock in sock_list:
                    # only keep little alive link.
                    # if you have 25 appid, you can keep 5 alive link.
                    if self.gae_conn_pool.qsize() > max(1, len(appid_manager.working_appid_list)/2):
                        ssl_sock.close()
                        continue

                    #inactive_time = time.time() -ssl_sock.last_use_time
                    #logging.debug("inactive_time:%d", inactive_time)
                    if self.head_request(ssl_sock):
                        self.save_ssl_connection_for_reuse(ssl_sock)
                    else:
                        ssl_sock.close()

                self.create_more_connection()
            except Exception as e:
                xlog.warn("keep alive except:%r", e)
Ejemplo n.º 20
0
def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes
    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong),
                    ('dwMajorVersion', ctypes.c_ulong),
                    ('dwMinorVersion', ctypes.c_ulong),
                    ('dwBuildNumber', ctypes.c_ulong),
                    ('dwPlatformId', ctypes.c_ulong),
                    ('szCSDVersion', ctypes.c_wchar*128),
                    ('wServicePackMajor', ctypes.c_ushort),
                    ('wServicePackMinor', ctypes.c_ushort),
                    ('wSuiteMask', ctypes.c_ushort),
                    ('wProductType', ctypes.c_byte),
                    ('wReserved', ctypes.c_byte)]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False
Ejemplo n.º 21
0
def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes

    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [
            ("dwOSVersionInfoSize", ctypes.c_ulong),
            ("dwMajorVersion", ctypes.c_ulong),
            ("dwMinorVersion", ctypes.c_ulong),
            ("dwBuildNumber", ctypes.c_ulong),
            ("dwPlatformId", ctypes.c_ulong),
            ("szCSDVersion", ctypes.c_wchar * 128),
            ("wServicePackMajor", ctypes.c_ushort),
            ("wServicePackMinor", ctypes.c_ushort),
            ("wSuiteMask", ctypes.c_ushort),
            ("wProductType", ctypes.c_byte),
            ("wReserved", ctypes.c_byte),
        ]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False
Ejemplo n.º 22
0
def fetch(method, host, path, headers, payload, bufsize=8192):
    request_data = "%s %s HTTP/1.1\r\n" % (method, path)
    request_data += "".join("%s: %s\r\n" % (k, v) for k, v in headers.items())
    request_data += "\r\n"

    ssl_sock = https_manager.get_ssl_connection(host)
    if not ssl_sock:
        return

    ssl_sock.send(request_data.encode())
    payload_len = len(payload)
    start = 0
    while start < payload_len:
        send_size = min(payload_len - start, 65535)
        sended = ssl_sock.send(payload[start : start + send_size])
        start += sended

    response = httplib.HTTPResponse(ssl_sock, buffering=True)

    response.ssl_sock = ssl_sock
    try:
        orig_timeout = ssl_sock.gettimeout()
        ssl_sock.settimeout(90)
        response.begin()
        ssl_sock.settimeout(orig_timeout)
    except httplib.BadStatusLine as e:
        xlog.warn("direct_handler.fetch bad status line:%r", e)
        response = None
    except Exception as e:
        xlog.warn("direct_handler.fetch:%r", e)
    return response
Ejemplo n.º 23
0
    def load(self):
        ConfigParser.RawConfigParser.OPTCRE = re.compile(r"(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$")

        self.DEFAULT_CONFIG = ConfigParser.ConfigParser()
        DEFAULT_CONFIG_FILENAME = os.path.abspath(os.path.join(current_path, "proxy.ini"))

        self.USER_CONFIG = ConfigParser.ConfigParser()
        CONFIG_USER_FILENAME = os.path.abspath(os.path.join(root_path, "data", "gae_proxy", "config.ini"))

        try:
            if os.path.isfile(DEFAULT_CONFIG_FILENAME):
                self.DEFAULT_CONFIG.read(DEFAULT_CONFIG_FILENAME)
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint("google_ip", "max_scan_ip_thread_num")
            else:
                return

            if os.path.isfile(CONFIG_USER_FILENAME):
                self.USER_CONFIG.read(CONFIG_USER_FILENAME)
            else:
                return

            try:
                self.user_special.appid = self.USER_CONFIG.get("gae", "appid")
                self.user_special.password = self.USER_CONFIG.get("gae", "password")
            except:
                pass

            try:
                self.user_special.host_appengine_mode = self.USER_CONFIG.get("hosts", "appengine.google.com")
            except:
                pass

            try:
                self.user_special.scan_ip_thread_num = config.CONFIG.getint("google_ip", "max_scan_ip_thread_num")
            except:
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint("google_ip", "max_scan_ip_thread_num")

            try:
                self.user_special.auto_adjust_scan_ip_thread_num = config.CONFIG.getint(
                    "google_ip", "auto_adjust_scan_ip_thread_num"
                )
            except:
                pass

            try:
                self.user_special.use_ipv6 = config.CONFIG.getint("google_ip", "use_ipv6")
            except:
                pass

            self.user_special.proxy_enable = self.USER_CONFIG.get("proxy", "enable")
            self.user_special.proxy_type = self.USER_CONFIG.get("proxy", "type")
            self.user_special.proxy_host = self.USER_CONFIG.get("proxy", "host")
            self.user_special.proxy_port = self.USER_CONFIG.get("proxy", "port")
            self.user_special.proxy_user = self.USER_CONFIG.get("proxy", "user")
            self.user_special.proxy_passwd = self.USER_CONFIG.get("proxy", "passwd")

        except Exception as e:
            xlog.warn("User_config.load except:%s", e)
Ejemplo n.º 24
0
    def save(self):
        CONFIG_USER_FILENAME = os.path.abspath(
            os.path.join(root_path, 'data', 'gae_proxy', 'config.ini'))
        try:
            f = open(CONFIG_USER_FILENAME, 'w')
            if self.user_special.appid != "":
                f.write("[gae]\n")
                f.write("appid = %s\n" % self.user_special.appid)
                f.write("password = %s\n\n" % self.user_special.password)

            f.write("[proxy]\n")
            f.write("enable = %s\n" % self.user_special.proxy_enable)
            f.write("type = %s\n" % self.user_special.proxy_type)
            f.write("host = %s\n" % self.user_special.proxy_host)
            f.write("port = %s\n" % self.user_special.proxy_port)
            f.write("user = %s\n" % self.user_special.proxy_user)
            f.write("passwd = %s\n\n" % self.user_special.proxy_passwd)

            if self.user_special.host_appengine_mode != "gae":
                f.write("[hosts]\n")
                f.write("appengine.google.com = %s\n" %
                        self.user_special.host_appengine_mode)
                f.write("www.google.com = %s\n\n" %
                        self.user_special.host_appengine_mode)

            f.write("[google_ip]\n")
            if self.user_special.ip_connect_interval != self.DEFAULT_CONFIG.getint(
                    'google_ip', 'ip_connect_interval'):
                f.write("ip_connect_interval = %d\n" %
                        int(self.user_special.ip_connect_interval))

            if int(self.user_special.auto_adjust_scan_ip_thread_num
                   ) != self.DEFAULT_CONFIG.getint(
                       'google_ip', 'auto_adjust_scan_ip_thread_num'):
                f.write("auto_adjust_scan_ip_thread_num = %d\n\n" %
                        int(self.user_special.auto_adjust_scan_ip_thread_num))
            if int(self.user_special.
                   scan_ip_thread_num) != self.DEFAULT_CONFIG.getint(
                       'google_ip', 'max_scan_ip_thread_num'):
                f.write("max_scan_ip_thread_num = %d\n\n" %
                        int(self.user_special.scan_ip_thread_num))

            if int(self.user_special.use_ipv6) != self.DEFAULT_CONFIG.getint(
                    'google_ip', 'use_ipv6'):
                f.write("use_ipv6 = %d\n\n" % int(self.user_special.use_ipv6))

            f.write("[connect_manager]\n")
            if int(self.user_special.
                   connect_interval) != self.DEFAULT_CONFIG.getint(
                       'connect_manager', 'connect_interval'):
                f.write("connect_interval = %d\n\n" %
                        int(self.user_special.connect_interval))

            f.close()
        except:
            xlog.warn("launcher.config save user config fail:%s",
                      CONFIG_USER_FILENAME)
Ejemplo n.º 25
0
    def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
        # change top domain to xx-net.net
        # this domain bypass the cloudflare front for ipv4
        #p = host.find(".")
        #host_sub = host[:p]
        #host = host_sub + ".xx-net.net"

        schema = "http"
        # force schema to http, avoid cert fail on heroku curl.
        # and all x-server provide ipv4 access

        url = schema + "://" + host + path
        payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
        for k in headers:
            v = headers[k]
            payloads.append('%s: %s\r\n' % (k, v))
        head_payload = "".join(payloads)

        request_body = '%s%s%s%s' % \
                       ((struct.pack('!H', len(head_payload)),  head_payload,
                         struct.pack('!I', len(data)), data))
        request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}

        heroku_host = ""
        content, status, response = self._request(
                                            "POST", heroku_host, "/2/",
                                            request_headers, request_body, timeout)

        # xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
        # xlog.debug("status:%d", status)
        if status == 200:
            xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace())
            self.last_success_time = time.time()
            self.continue_fail_num = 0
            self.success_num += 1
        else:
            if status == 404:
                heroku_host = response.ssl_sock.host
                xlog.warn("heroku:%s fail", heroku_host)
                try:
                    self.host_manager.remove(heroku_host)
                except:
                    pass

            self.last_fail_time = time.time()
            self.continue_fail_num += 1
            self.fail_num += 1

        try:
            res = simple_http_client.TxtResponse(content)
        except:
            return "", 501, {}

        res.worker = response.worker
        res.task = response.task
        return res.body, res.status, res
Ejemplo n.º 26
0
 def runJob(self):
     while self.check_num < 1000000:
         try:
             time.sleep(1)
             ip_int = ip_range.get_ip()
             #ip_int = ip_range.random_get_ip()
             ip_str = ip_utils.ip_num_to_string(ip_int)
             self.check_ip(ip_str)
         except Exception as e:
             xlog.warn("google_ip.runJob fail:%s", e)
Ejemplo n.º 27
0
 def runJob(self):
     while self.check_num < 1000000:
         try:
             time.sleep(1)
             ip_int = ip_range.get_ip()
             #ip_int = ip_range.random_get_ip()
             ip_str = ip_utils.ip_num_to_string(ip_int)
             self.check_ip(ip_str)
         except Exception as e:
             xlog.warn("google_ip.runJob fail:%s", e)
Ejemplo n.º 28
0
    def req_deploy_handler(self):
        global deploy_proc
        req = urlparse.urlparse(self.path).query
        reqs = urlparse.parse_qs(req, keep_blank_values=True)
        data = ''

        log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'upload.log'))
        time_now = datetime.datetime.today().strftime('%H:%M:%S-%a/%d/%b/%Y')

        if reqs['cmd'] == ['deploy']:
            appid = self.postvars['appid'][0]

            if deploy_proc and deploy_proc.poll() == None:
                xlog.warn("deploy is running, request denied.")
                data = '{"res":"deploy is running", "time":"%s"}' % (time_now)

            else:
                try:
                    if os.path.isfile(log_path):
                        os.remove(log_path)
                    script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'uploader.py'))

                    email = self.postvars['email'][0]
                    passwd = self.postvars['passwd'][0]
                    rc4_passwd = self.postvars['rc4_passwd'][0]
                    deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd])
                    xlog.info("deploy begin.")
                    data = '{"res":"success", "time":"%s"}' % time_now
                except Exception as e:
                    data = '{"res":"%s", "time":"%s"}' % (e, time_now)

        elif reqs['cmd'] == ['cancel']:
            if deploy_proc and deploy_proc.poll() == None:
                deploy_proc.kill()
                data = '{"res":"deploy is killed", "time":"%s"}' % (time_now)
            else:
                data = '{"res":"deploy is not running", "time":"%s"}' % (time_now)

        elif reqs['cmd'] == ['get_log']:
            if deploy_proc and os.path.isfile(log_path):
                with open(log_path, "r") as f:
                    content = f.read()
            else:
                content = ""

            status = 'init'
            if deploy_proc:
                if deploy_proc.poll() == None:
                    status = 'running'
                else:
                    status = 'finished'

            data = json.dumps({'status':status,'log':content, 'time':time_now})

        self.send_response('text/html', data)
Ejemplo n.º 29
0
    def req_deploy_handler(self):
        global deploy_proc
        req = urlparse.urlparse(self.path).query
        reqs = urlparse.parse_qs(req, keep_blank_values=True)
        data = ""

        log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "upload.log"))
        time_now = datetime.datetime.today().strftime("%H:%M:%S-%a/%d/%b/%Y")

        if reqs["cmd"] == ["deploy"]:
            appid = self.postvars["appid"][0]

            if deploy_proc and deploy_proc.poll() == None:
                xlog.warn("deploy is running, request denied.")
                data = '{"res":"deploy is running", "time":"%s"}' % (time_now)

            else:
                try:
                    if os.path.isfile(log_path):
                        os.remove(log_path)
                    script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "uploader.py"))

                    email = self.postvars["email"][0]
                    passwd = self.postvars["passwd"][0]
                    rc4_passwd = self.postvars["rc4_passwd"][0]
                    deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd])
                    xlog.info("deploy begin.")
                    data = '{"res":"success", "time":"%s"}' % time_now
                except Exception as e:
                    data = '{"res":"%s", "time":"%s"}' % (e, time_now)

        elif reqs["cmd"] == ["cancel"]:
            if deploy_proc and deploy_proc.poll() == None:
                deploy_proc.kill()
                data = '{"res":"deploy is killed", "time":"%s"}' % (time_now)
            else:
                data = '{"res":"deploy is not running", "time":"%s"}' % (time_now)

        elif reqs["cmd"] == ["get_log"]:
            if deploy_proc and os.path.isfile(log_path):
                with open(log_path, "r") as f:
                    content = f.read()
            else:
                content = ""

            status = "init"
            if deploy_proc:
                if deploy_proc.poll() == None:
                    status = "running"
                else:
                    status = "finished"

            data = json.dumps({"status": status, "log": content, "time": time_now})

        self.send_response("text/html", data)
Ejemplo n.º 30
0
    def check(self, callback=None, check_ca=True, close_ssl=True):

        ssl_sock = None
        try:
            ssl_sock, self.result.connct_time, self.result.handshake_time = connect_ssl(
                self.ip,
                timeout=self.timeout,
                openssl_context=self.openssl_context)

            # verify SSL certificate issuer.
            def check_ssl_cert(ssl_sock):
                cert = ssl_sock.get_peer_certificate()
                if not cert:
                    #raise HoneypotError(' certficate is none')
                    raise SSLError("no cert")

                issuer_commonname = next(
                    (v for k, v in cert.get_issuer().get_components()
                     if k == 'CN'), '')

                ssl_cert = cert_util.SSLCert(cert)
                xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
                self.result.domain = ssl_cert.cn

            if check_ca:
                check_ssl_cert(ssl_sock)

            if callback:
                return callback(ssl_sock, self.ip)

            return True
        except SSLError as e:
            xlog.debug("Check_appengine %s SSLError:%s", self.ip, e)
            pass
        except IOError as e:
            xlog.warn("Check %s IOError:%s", self.ip, e)
            pass
        except httplib.BadStatusLine:
            #logging.debug('Check_appengine http.bad status line ip:%s', ip)
            #import traceback
            #traceback.print_exc()
            pass
        except Exception as e:
            if len(e.args) > 0:
                errno_str = e.args[0]
            else:
                errno_str = e.message
            xlog.exception('check_appengine %s %s err:%s', self.ip, errno_str,
                           e)
        finally:
            if ssl_sock and close_ssl:
                ssl_sock.close()

        return False
Ejemplo n.º 31
0
    def test(self):
        for cipher in self.cipher_list:
            xlog.debug("%s", cipher)

            openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,))
            try:
                ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
            except Exception as e:
                xlog.warn("err:%s", e)
Ejemplo n.º 32
0
    def test(self):
        for cipher in self.cipher_list:
            xlog.debug("%s", cipher)

            openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=(cipher,))
            try:
                ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
            except Exception as e:
                xlog.warn("err:%s", e)
Ejemplo n.º 33
0
def check_all_domain(check_ip):
    with open(os.path.join(current_path, "front_domains.json"), "r") as fd:
        content = fd.read()
        cs = json.loads(content)
        for host in cs:
            host = "scan1." + host
            res = check_ip.check_ip(ip, host=host, wait_time=wait_time)
            if not res or not res.ok:
                xlog.warn("host:%s fail", host)
            else:
                xlog.info("host:%s ok", host)
Ejemplo n.º 34
0
def check_all_domain(check_ip):
    with open(os.path.join(current_path, "front_domains.json"), "r") as fd:
        content = fd.read()
        cs = json.loads(content)
        for host in cs:
            host = "scan1." + host
            res = check_ip.check_ip(ip, host=host, wait_time=wait_time)
            if not res or not res.ok:
                xlog.warn("host:%s fail", host)
            else:
                xlog.info("host:%s ok", host)
Ejemplo n.º 35
0
def test_gws(ip_str):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str)

    result = check.check(callback=test_server_type, check_ca=True)
    if not result or not "gws" in result:
        xlog.warn("Server:%s not gws", result)
        return False

    check.result.server_type = result

    return check.result
Ejemplo n.º 36
0
    def check(self, callback=None, check_ca=True, close_ssl=True):

        ssl_sock = None
        try:
            ssl_sock,self.result.connct_time,self.result.handshake_time = connect_ssl(self.ip, timeout=self.timeout, openssl_context=self.openssl_context)

            # verify SSL certificate issuer.
            def check_ssl_cert(ssl_sock):
                cert = ssl_sock.get_peer_certificate()
                if not cert:
                    #raise HoneypotError(' certficate is none')
                    raise SSLError("no cert")

                issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
                if self.check_cert and not issuer_commonname.startswith('Google'):
                    raise HoneypotError(' certficate is issued by %r, not Google' % ( issuer_commonname))


                ssl_cert = cert_util.SSLCert(cert)
                xlog.info("%s CN:%s", self.ip, ssl_cert.cn)
                self.result.domain = ssl_cert.cn
            if check_ca:
                check_ssl_cert(ssl_sock)

            if callback:
                return callback(ssl_sock, self.ip)

            return True
        except HoneypotError as e:
            xlog.warn("honeypot %s", self.ip)
            raise e
        except SSLError as e:
            xlog.debug("Check_appengine %s SSLError:%s", self.ip, e)
            pass
        except IOError as e:
            xlog.warn("Check %s IOError:%s", self.ip, e)
            pass
        except httplib.BadStatusLine:
            #logging.debug('Check_appengine http.bad status line ip:%s', ip)
            #import traceback
            #traceback.print_exc()
            pass
        except Exception as e:
            if len(e.args)>0:
                errno_str = e.args[0]
            else:
                errno_str = e.message
            xlog.exception('check_appengine %s %s err:%s', self.ip, errno_str, e)
        finally:
            if ssl_sock and close_ssl:
                ssl_sock.close()

        return False
Ejemplo n.º 37
0
def test_gws(ip_str):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str)

    result = check.check(callback=test_server_type, check_ca=True)
    if not result or not "gws" in result:
        xlog.warn("Server:%s not gws", result)
        return False

    check.result.server_type = result

    return check.result
Ejemplo n.º 38
0
    def handle(self):
        #xlog.info('Connected from %r', self.client_address)
        while True:
            try:
                self.close_connection = 1
                self.handle_one_request()
            except Exception as e:
                xlog.warn("handle err:%r close", e)
                self.close_connection = 1

            if self.close_connection:
                break
        self.connection.close()
Ejemplo n.º 39
0
    def handle(self):
        #xlog.info('Connected from %r', self.client_address)
        while True:
            try:
                self.close_connection = 1
                self.handle_one_request()
            except Exception as e:
                xlog.warn("handle err:%r close", e)
                self.close_connection = 1

            if self.close_connection:
                break
        self.connection.close()
Ejemplo n.º 40
0
    def head_request(self, ssl_sock):
        if ssl_sock.host == '':
            ssl_sock.appid = appid_manager.get_appid()
            if not ssl_sock.appid:
                xlog.error("no appid can use")
                return False
            host = ssl_sock.appid + ".appspot.com"
            ssl_sock.host = host
        else:
            host = ssl_sock.host

        # public appid don't keep alive, for quota limit.
        if ssl_sock.appid.startswith(
                "xxnet-") and ssl_sock.appid[7:].isdigit():
            #logging.info("public appid don't keep alive")
            #self.keep_alive = 0
            return False

        #logging.debug("head request %s", host)

        request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host

        response = None
        try:
            ssl_sock.settimeout(10)
            ssl_sock.sock.settimeout(10)

            data = request_data.encode()
            ret = ssl_sock.send(data)
            if ret != len(data):
                xlog.warn("head send len:%d %d", ret, len(data))
            response = httplib.HTTPResponse(ssl_sock, buffering=True)

            response.begin()

            status = response.status
            if status != 200:
                xlog.debug("app head fail status:%d", status)
                raise Exception("app check fail")
            return True
        except httplib.BadStatusLine as e:
            inactive_time = time.time() - ssl_sock.last_use_time
            xlog.debug("%s keep alive fail, time:%d", ssl_sock.ip,
                       inactive_time)
            return False
        except Exception as e:
            xlog.debug("head request fail:%r", e)
            return False
        finally:
            if response:
                response.close()
Ejemplo n.º 41
0
    def add_ip(self,
               ip_str,
               handshake_time,
               domain=None,
               server='',
               fail_times=0):
        if not isinstance(ip_str, basestring):
            xlog.error("add_ip input")
            return

        if config.USE_IPV6 and ":" not in ip_str:
            xlog.warn("add %s but ipv6", ip_str)
            return

        handshake_time = int(handshake_time)

        self.ip_lock.acquire()
        try:
            if ip_str in self.ip_dict:
                self.ip_dict[ip_str]['handshake_time'] = handshake_time
                self.ip_dict[ip_str]['fail_times'] = fail_times
                self.ip_dict[ip_str]['fail_time'] = 0
                self.append_ip_history(ip_str, handshake_time)
                return False

            self.iplist_need_save = 1
            self.good_ip_num += 1

            self.ip_dict[ip_str] = {
                'handshake_time': handshake_time,
                "fail_times": fail_times,
                "transfered_data": 0,
                'data_active': 0,
                'domain': domain,
                'server': server,
                "history": [[time.time(), handshake_time]],
                "fail_time": 0,
                "success_time": 0,
                "get_time": 0,
                "links": 0
            }

            if 'gws' in server:
                self.gws_ip_list.append(ip_str)
            return True
        except Exception as e:
            xlog.exception("add_ip err:%s", e)
        finally:
            self.ip_lock.release()
        return False
Ejemplo n.º 42
0
    def report_not_exist(self, appid):
        xlog.warn("APPID_manager, report_not_exist %s", appid)
        self.lock.acquire()
        try:
            config.GAE_APPIDS.remove(appid)
            self.not_exist_appids.append(appid)
            self.working_appid_list.remove(appid)
        except:
            pass
        finally:
            self.lock.release()

        if len(self.working_appid_list) == 0:
            self.reset_appid()
Ejemplo n.º 43
0
    def report_not_exist(self, appid):
        xlog.warn("APPID_manager, report_not_exist %s", appid)
        self.lock.acquire()
        try:
            config.GAE_APPIDS.remove(appid)
            self.not_exist_appids.append(appid)
            self.working_appid_list.remove(appid)
        except:
            pass
        finally:
            self.lock.release()

        if len(self.working_appid_list) == 0:
            self.reset_appid()
Ejemplo n.º 44
0
    def save(self):
        CONFIG_USER_FILENAME = os.path.abspath(os.path.join(root_path, "data", "gae_proxy", "config.ini"))
        try:
            f = open(CONFIG_USER_FILENAME, "w")
            if self.user_special.appid != "":
                f.write("[gae]\n")
                f.write("appid = %s\n" % self.user_special.appid)
                f.write("password = %s\n\n" % self.user_special.password)

            f.write("[proxy]\n")
            f.write("enable = %s\n" % self.user_special.proxy_enable)
            f.write("type = %s\n" % self.user_special.proxy_type)
            f.write("host = %s\n" % self.user_special.proxy_host)
            f.write("port = %s\n" % self.user_special.proxy_port)
            f.write("user = %s\n" % self.user_special.proxy_user)
            f.write("passwd = %s\n\n" % self.user_special.proxy_passwd)

            if self.user_special.host_appengine_mode != "gae":
                f.write("[hosts]\n")
                f.write("appengine.google.com = %s\n" % self.user_special.host_appengine_mode)
                f.write("www.google.com = %s\n\n" % self.user_special.host_appengine_mode)

            f.write("[google_ip]\n")
            if self.user_special.ip_connect_interval != self.DEFAULT_CONFIG.getint("google_ip", "ip_connect_interval"):
                f.write("ip_connect_interval = %d\n" % int(self.user_special.ip_connect_interval))

            if int(self.user_special.auto_adjust_scan_ip_thread_num) != self.DEFAULT_CONFIG.getint(
                "google_ip", "auto_adjust_scan_ip_thread_num"
            ):
                f.write(
                    "auto_adjust_scan_ip_thread_num = %d\n\n" % int(self.user_special.auto_adjust_scan_ip_thread_num)
                )
            if int(self.user_special.scan_ip_thread_num) != self.DEFAULT_CONFIG.getint(
                "google_ip", "max_scan_ip_thread_num"
            ):
                f.write("max_scan_ip_thread_num = %d\n\n" % int(self.user_special.scan_ip_thread_num))

            if int(self.user_special.use_ipv6) != self.DEFAULT_CONFIG.getint("google_ip", "use_ipv6"):
                f.write("use_ipv6 = %d\n\n" % int(self.user_special.use_ipv6))

            f.write("[connect_manager]\n")
            if int(self.user_special.connect_interval) != self.DEFAULT_CONFIG.getint(
                "connect_manager", "connect_interval"
            ):
                f.write("connect_interval = %d\n\n" % int(self.user_special.connect_interval))

            f.close()
        except:
            xlog.warn("launcher.config save user config fail:%s", CONFIG_USER_FILENAME)
Ejemplo n.º 45
0
    def head_request(self, ssl_sock):
        if ssl_sock.host == '':
            ssl_sock.appid = appid_manager.get_appid()
            if not ssl_sock.appid:
                xlog.error("no appid can use")
                return False
            host = ssl_sock.appid + ".appspot.com"
            ssl_sock.host = host
        else:
            host = ssl_sock.host

        # public appid don't keep alive, for quota limit.
        if ssl_sock.appid.startswith("xxnet-") and ssl_sock.appid[7:].isdigit():
            #logging.info("public appid don't keep alive")
            #self.keep_alive = 0
            return False

        #logging.debug("head request %s", host)

        request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host

        response = None
        try:
            ssl_sock.settimeout(10)
            ssl_sock.sock.settimeout(10)

            data = request_data.encode()
            ret = ssl_sock.send(data)
            if ret != len(data):
                xlog.warn("head send len:%d %d", ret, len(data))
            response = httplib.HTTPResponse(ssl_sock, buffering=True)

            response.begin()

            status = response.status
            if status != 200:
                xlog.debug("app head fail status:%d", status)
                raise Exception("app check fail")
            return True
        except httplib.BadStatusLine as e:
            inactive_time = time.time() - ssl_sock.last_use_time
            xlog.debug("%s keep alive fail, time:%d", ssl_sock.ip, inactive_time)
            return False
        except Exception as e:
            xlog.debug("head request fail:%r", e)
            return False
        finally:
            if response:
                response.close()
Ejemplo n.º 46
0
    def handle_one_request(self):
        try:
            try:
                self.raw_requestline = self.rfile.readline(65537)
            except Exception as e:
                #xlog.warn("simple server handle except %r", e)
                return

            if len(self.raw_requestline) > 65536:
                xlog.warn("recv command line too large")
                return
            if not self.raw_requestline:
                #xlog.warn("closed")
                return

            self.parse_request()

            if self.command == "GET":
                self.do_GET()
            elif self.command == "POST":
                self.do_POST()
            elif self.command == "CONNECT":
                self.do_CONNECT()
            elif self.command == "HEAD":
                self.do_HEAD()
            elif self.command == "DELETE":
                self.do_DELETE()
            elif self.command == "OPTIONS":
                self.do_OPTIONS()
            elif self.command == "PUT":
                self.do_PUT()
            else:
                xlog.warn("unhandler cmd:%s", self.command)
                return

            self.wfile.flush(
            )  #actually send the response if not already done.
            self.close_connection = 0
        except socket.error as e:
            xlog.warn("socket error:%r", e)
        except IOError as e:
            if e.errno == errno.EPIPE:
                xlog.warn("PIPE error:%r", e)
            else:
                xlog.warn("IOError:%r", e)
        #except OpenSSL.SSL.SysCallError as e:
        #    xlog.warn("socket error:%r", e)
        except Exception as e:
            xlog.exception("handler:%r", e)
Ejemplo n.º 47
0
    def do_GET(self):
        xlog.info('PAC from:%s %s %s ', self.address_string(), self.command,
                  self.path)

        path = urlparse.urlparse(self.path).path  # '/proxy.pac'
        filename = os.path.normpath('./' + path)  # proxy.pac

        if self.path.startswith(('http://', 'https://')):
            data = b'HTTP/1.1 200\r\nCache-Control: max-age=86400\r\nExpires:Oct, 01 Aug 2100 00:00:00 GMT\r\nConnection: close\r\n'
            if filename.endswith(('.jpg', '.gif', '.jpeg', '.bmp')):
                data += b'Content-Type: image/gif\r\n\r\n' + self.onepixel
            else:
                data += b'\r\n This is the Pac server, not proxy port, use 8087 as proxy port.'
            self.wfile.write(data)
            xlog.info('%s "%s %s HTTP/1.1" 200 -', self.address_string(),
                      self.command, self.path)
            return

        # check for '..', which will leak file
        if re.search(r'(\.{2})', self.path) is not None:
            self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
            xlog.warn('%s %s %s haking', self.address_string(), self.command,
                      self.path)
            return

        if filename != 'proxy.pac':
            xlog.warn("pac_server GET %s fail", filename)
            self.wfile.write(b'HTTP/1.1 404\r\n\r\n')
            return

        mimetype = 'text/plain'
        if self.path.endswith('.pac?flush') or time.time() - os.path.getmtime(
                get_serving_pacfile()) > config.PAC_EXPIRED:
            thread.start_new_thread(PacUtil.update_pacfile, (user_pacfile, ))

        pac_filename = get_serving_pacfile()
        with open(pac_filename, 'rb') as fp:
            data = fp.read()

        host = self.headers.getheader('Host')
        host, _, port = host.rpartition(":")
        gae_proxy_proxy = host + ":" + str(config.LISTEN_PORT)
        pac_proxy = host + ":" + str(config.PAC_PORT)
        data = data.replace(gae_proxy_listen, gae_proxy_proxy)
        data = data.replace(pac_listen, pac_proxy)
        self.wfile.write(
            ('HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' %
             (mimetype, len(data))).encode())
        self.wfile.write(data)
Ejemplo n.º 48
0
    def handle_one_request(self):
        try:
            try:
                self.raw_requestline = self.rfile.readline(65537)
            except Exception as e:
                #xlog.warn("simple server handle except %r", e)
                return

            if len(self.raw_requestline) > 65536:
                xlog.warn("recv command line too large")
                return
            if not self.raw_requestline:
                #xlog.warn("closed")
                return

            self.parse_request()

            if self.command == "GET":
                self.do_GET()
            elif self.command == "POST":
                self.do_POST()
            elif self.command == "CONNECT":
                self.do_CONNECT()
            elif self.command == "HEAD":
                self.do_HEAD()
            elif self.command == "DELETE":
                self.do_DELETE()
            elif self.command == "OPTIONS":
                self.do_OPTIONS()
            elif self.command == "PUT":
                self.do_PUT()
            else:
                xlog.warn("unhandler cmd:%s", self.command)
                return

            self.wfile.flush() #actually send the response if not already done.
            self.close_connection = 0
        except socket.error as e:
            xlog.warn("socket error:%r", e)
        except IOError as e:
            if e.errno == errno.EPIPE:
                xlog.warn("PIPE error:%r", e)
            else:
                xlog.warn("IOError:%r", e)
        #except OpenSSL.SSL.SysCallError as e:
        #    xlog.warn("socket error:%r", e)
        except Exception as e:
            xlog.exception("handler:%r", e)
Ejemplo n.º 49
0
    def serve_forever(self):
        fdset = [self.socket, ]
        while self.running:
            r, w, e = select.select(fdset, [], [], 1)
            if self.socket in r:
                try:
                    (sock, address) = self.socket.accept()
                except IOError as e:
                    xlog.warn("socket(%s:%s) accept fail(errno: %s).", self.server_address[0], self.server_address[1], e.args[0])
                    if e.args[0] == 10022:
                        xlog.info("server %s:%d restarted.", self.server_address[0], self.server_address[1])
                        self.init_socket()
                        fdset = [self.socket, ]
                    return

                self.process_connect(sock, address)
Ejemplo n.º 50
0
    def send_file(self, filename, mimetype):
        file_size = os.path.getsize(filename)
        try:
            tme = (datetime.datetime.today()+datetime.timedelta(minutes=330)).strftime('%a, %d %b %Y %H:%M:%S GMT')
            self.wfile.write(('HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nCache-Control:public, max-age=31536000\r\nExpires: %s\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (tme, mimetype, file_size)).encode())


            with open(filename, 'rb') as fp:
                while True:
                    data = fp.read(65535)
                    if not data:
                        break
                    self.wfile.write(data)
        except:
            #self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Open file fail')
            xlog.warn("download broken")
Ejemplo n.º 51
0
    def _request(self, method, host, path="/", headers={}, data="", timeout=40):
        try:
            response = self.http_dispatcher.request(method, host, path, dict(headers), data, timeout=timeout)
            if not response:
                return "", 500, {}

            status = response.status
            if status != 200:
                xlog.warn("front request %s %s%s fail, status:%d", method, host, path, status)

            content = response.task.read_all()
            # xlog.debug("%s %s%s trace:%s", method, response.ssl_sock.host, path, response.task.get_trace())
            return content, status, response
        except Exception as e:
            xlog.exception("front request %s %s%s fail:%r", method, host, path, e)
            return "", 500, {}
Ejemplo n.º 52
0
    def do_METHOD(self):
        touch_active()

        host = self.headers.get('Host', '')
        host_ip, _, port = host.rpartition(':')
        if host_ip == "127.0.0.1" and port == str(config.LISTEN_PORT):
            controler = web_control.ControlHandler(self.client_address,
                                                   self.headers, self.command,
                                                   self.path, self.rfile,
                                                   self.wfile)
            if self.command == "GET":
                return controler.do_GET()
            elif self.command == "POST":
                return controler.do_POST()
            else:
                xlog.warn("method not defined: %s", self.command)
                return

        if self.path[0] == '/' and host:
            self.path = 'http://%s%s' % (host, self.path)
        elif not host and '://' in self.path:
            host = urlparse.urlparse(self.path).netloc

        if host.startswith("127.0.0.1") or host.startswith("localhost"):
            xlog.warn("Your browser forward localhost to proxy.")
            return self.forward_local()

        self.parsed_url = urlparse.urlparse(self.path)

        if host in config.HOSTS_GAE:
            return self.do_AGENT()

        if host in config.HOSTS_FWD or host in config.HOSTS_DIRECT:
            return self.wfile.write(
                ('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
                 self.path.replace('http://', 'https://', 1)).encode())

        if host.endswith(config.HOSTS_GAE_ENDSWITH):
            return self.do_AGENT()

        if host.endswith(config.HOSTS_FWD_ENDSWITH) or host.endswith(
                config.HOSTS_DIRECT_ENDSWITH):
            return self.wfile.write(
                ('HTTP/1.1 301\r\nLocation: %s\r\n\r\n' %
                 self.path.replace('http://', 'https://', 1)).encode())

        return self.do_AGENT()
Ejemplo n.º 53
0
def network_is_ok():
    global checking_lock, checking_num, network_ok, last_check_time, check_network_interval
    if time.time() - last_check_time < check_network_interval:
        return network_ok

    if checking_num > 0:
        return network_ok

    if config.PROXY_ENABLE:
        socket.socket = socks.socksocket
        xlog.debug("patch socks")

    checking_lock.acquire()
    checking_num += 1
    checking_lock.release()
    try:
        conn = httplib.HTTPSConnection("code.jquery.com", 443, timeout=30)
        header = {
            "user-agent":
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
            "accept": "application/json, text/javascript, */*; q=0.01",
            "accept-encoding": "gzip, deflate, sdch",
            "accept-language": 'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2',
            "connection": "keep-alive"
        }
        conn.request("HEAD", "/", headers=header)
        response = conn.getresponse()
        if response.status:
            xlog.debug("network is ok")
            network_ok = True
            last_check_time = time.time()
            return True
    except:
        pass
    finally:
        checking_lock.acquire()
        checking_num -= 1
        checking_lock.release()

        if config.PROXY_ENABLE:
            socket.socket = default_socket
            xlog.debug("restore socket")

    xlog.warn("network fail.")
    network_ok = False
    last_check_time = time.time()
    return False
Ejemplo n.º 54
0
    def create_more_connection(self):
        need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize()

        target_thread_num = min(self.max_thread_num, need_conn_num)
        for i in range(0, target_thread_num):
            if not connect_control.allow_connect():
                xlog.warn("create more connect, control not allow")
                time.sleep(10)
                continue
            if self.thread_num > self.max_thread_num:
                break

            self.thread_num_lock.acquire()
            self.thread_num += 1
            self.thread_num_lock.release()
            p = threading.Thread(target=self.create_connection_worker)
            p.start()
Ejemplo n.º 55
0
    def do_CONNECT_FWD(self):
        """socket forward for http CONNECT command"""
        host, _, port = self.path.rpartition(':')
        port = int(port)
        xlog.info('FWD %s %s:%d ', self.command, host, port)
        if host == "appengine.google.com" or host == "www.google.com":
            connected_in_s = 5  # gae_proxy upload to appengine is slow, it need more 'fresh' connection.
        else:
            connected_in_s = 10  # gws connect can be used after tcp connection created 15 s

        try:
            self.wfile.write(b'HTTP/1.1 200 OK\r\n\r\n')
            data = self.connection.recv(1024)
        except Exception as e:
            xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port,
                           e)
            self.connection.close()
            return

        remote = forwork_manager.create_connection(host=host,
                                                   port=port,
                                                   sock_life=connected_in_s)
        if remote is None:
            self.connection.close()
            xlog.warn('FWD %s %s:%d create_connection fail', self.command,
                      host, port)
            return

        try:
            if data:
                remote.send(data)
        except Exception as e:
            xlog.exception('do_CONNECT_FWD (%r, %r) Exception:%s', host, port,
                           e)
            self.connection.close()
            remote.close()
            return

        # reset timeout default to avoid long http upload failure, but it will delay timeout retry :(
        remote.settimeout(None)

        forwork_manager.forward_socket(self.connection,
                                       remote,
                                       bufsize=self.bufsize)
        xlog.debug('FWD %s %s:%d with closed', self.command, host, port)
Ejemplo n.º 56
0
    def create_more_connection(self):
        need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize(
        )

        target_thread_num = min(self.max_thread_num, need_conn_num)
        for i in range(0, target_thread_num):
            if not connect_control.allow_connect():
                xlog.warn("create more connect, control not allow")
                time.sleep(10)
                continue
            if self.thread_num > self.max_thread_num:
                break

            self.thread_num_lock.acquire()
            self.thread_num += 1
            self.thread_num_lock.release()
            p = threading.Thread(target=self.create_connection_worker)
            p.start()
Ejemplo n.º 57
0
    def send_file(self, filename, mimetype):
        file_size = os.path.getsize(filename)
        try:
            tme = (datetime.datetime.today() + datetime.timedelta(minutes=330)
                   ).strftime('%a, %d %b %Y %H:%M:%S GMT')
            self.wfile.write((
                'HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nCache-Control:public, max-age=31536000\r\nExpires: %s\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n'
                % (tme, mimetype, file_size)).encode())

            with open(filename, 'rb') as fp:
                while True:
                    data = fp.read(65535)
                    if not data:
                        break
                    self.wfile.write(data)
        except:
            #self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Open file fail')
            xlog.warn("download broken")
Ejemplo n.º 58
0
    def checker(self):
        while True:
            try:
                ip = self.get_ip()
            except Exception as e:
                xlog.info("no ip left")
                return

            try:
                res = self.check_ip.check_ip(ip)
            except Exception as e:
                xlog.warn("check except:%r", e)
                continue

            if not res or not res.ok:
                xlog.debug("ip:%s fail", ip)
                continue

            self.write_ip(ip, res.domain, res.handshake_time)
Ejemplo n.º 59
0
    def update_ip(self, ip_str, handshake_time):
        if not isinstance(ip_str, basestring):
            xlog.error("set_ip input")
            return

        handshake_time = int(handshake_time)
        if handshake_time < 5:  # that's impossible
            xlog.warn("%s handshake:%d impossible", ip_str,
                      1000 * handshake_time)
            return

        self.ip_lock.acquire()
        try:
            if ip_str in self.ip_dict:
                time_now = time.time()

                # Case: some good ip, average handshake time is 300ms
                # some times ip package lost cause handshake time become 2000ms
                # this ip will not return back to good ip front until all become bad
                # There for, prevent handshake time increase too quickly.
                org_time = self.ip_dict[ip_str]['handshake_time']
                if handshake_time - org_time > 500:
                    self.ip_dict[ip_str]['handshake_time'] = org_time + 500
                else:
                    self.ip_dict[ip_str]['handshake_time'] = handshake_time

                self.ip_dict[ip_str]['success_time'] = time_now
                if self.ip_dict[ip_str]['fail_times'] > 0:
                    self.good_ip_num += 1
                self.ip_dict[ip_str]['fail_times'] = 0
                self.append_ip_history(ip_str, handshake_time)
                self.ip_dict[ip_str]["fail_time"] = 0

                self.iplist_need_save = 1

            #logging.debug("update ip:%s not exist", ip_str)
        except Exception as e:
            xlog.error("update_ip err:%s", e)
        finally:
            self.ip_lock.release()

        self.save_ip_list()