Пример #1
0
    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config=config)

        self.source_ip = config.get('source_ip', '0.0.0.0')
        self.source_port = config.get('source_port', 0)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        if self.source_ip == '0.0.0.0' and self.source_port == 0:
            self.source_address = None
        else:
            self.source_address = (self.source_ip, self.source_port)

        class socket(SocketBase):
            def __init__(self,
                         family=_socket.AF_INET,
                         type=_socket.SOCK_STREAM,
                         proto=0,
                         _sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family,
                                                   type=type,
                                                   proto=proto)
                    _sock.bind(self.source_address)
                SocketBase.__init__(self, _sock)

        socket.source_address = self.source_address
        socket.upstream = self.upstream
        socket.display_name = self.get_display_name()
        socket.name = self.get_name()

        self.socket = socket
Пример #2
0
    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        self._list = config.get('list', [{'type': 'direct'}])
        self.upstream_dict = {}
        for i in self._list:
            type = i.get('type')
            if not type:
                raise ConfigError(u'[upstream]代理类型不能为空! ')
            Upstream = upstream.get_upstream(type)
            u = Upstream(i)
            self.upstream_dict[u.get_name()] = u
Пример #3
0
    def __init__(self,config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self,config=config)

        self.source_ip = config.get('source_ip','0.0.0.0')
        self.source_port = config.get('source_port',0)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        if self.source_ip == '0.0.0.0' and self.source_port==0:
            self.source_address = None
        else:
            self.source_address=(self.source_ip,self.source_port)

        class socket(SocketBase):
            def __init__(self, family=_socket.AF_INET, type=_socket.SOCK_STREAM, proto=0,_sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family,type=type,proto=proto)
                    _sock.bind(self.source_address)
                SocketBase.__init__(self,_sock)

        socket.source_address = self.source_address
        socket.upstream = self.upstream

        self.socket = socket
Пример #4
0
    def __init__(self,
                 sock_mod=None,
                 max_site=20,
                 max_size_conn=10,
                 expire=60,
                 lock=None):
        if sock_mod is None:
            sock_mod = __import__("socket")
        self.sock_mod = sock_mod

        self.site_dict = LRUCacheDict(max_site, expire, _safe_del_connes)

        self.max_site_conn = max_size_conn

        if lock is None:
            self.lock = _lock()
        elif callable(lock):
            self.lock = lock()
        else:
            raise ValueError('callable(lock)==False')
Пример #5
0
    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        self._list = config.get('list', [{'type': 'direct'}])
        self.upstream_dict = {}
        for i in self._list:
            type = i.get('type')
            if not type:
                raise ConfigError(u'[upstream]代理类型不能为空! ')
            Upstream = upstream.get_upstream(type)
            u = Upstream(i)
            self.upstream_dict[u.get_name()] = u
Пример #6
0
    def __init__(self, sock_mod=None, max_site=20, max_size_conn=10, expire=60, lock=None):
        if sock_mod is None:
            sock_mod = __import__("socket")
        self.sock_mod = sock_mod

        self.site_dict = LRUCacheDict(max_site, expire, _safe_del_connes)

        self.max_site_conn = max_size_conn

        if lock is None:
            self.lock = _lock()
        elif callable(lock):
            self.lock = lock()
        else:
            raise ValueError('callable(lock)==False')
Пример #7
0
class MultipathUpstream(UpstreamBase):
    u""" 多路径 socket 模块

对于 tcp 连接,同时使用多个线路尝试连接,最终使用最快建立连接的线路。

使用方式为创建本类实例,然后把实例当作 socket 模块即可。所有的操作都会经过 config 配置的线路。

以后也许会弄个当连接多路复用。
"""

    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        self._list = config.get('list', [{'type': 'direct'}])
        self.upstream_dict = {}
        for i in self._list:
            type = i.get('type')
            if not type:
                raise ConfigError(u'[upstream]代理类型不能为空! ')
            Upstream = upstream.get_upstream(type)
            u = Upstream(i)
            self.upstream_dict[u.get_name()] = u

    u'''
        class socket(SocketBase):
            def __init__(self, family=_socket.AF_INET, type=_socket.SOCK_STREAM, proto=0, _sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family, type=type, proto=proto)
                    _sock.bind((socket.source_ip, socket.source_port))
                SocketBase.__init__(self, _sock)

        socket.upstream = self.upstream

        self.socket = socket'''

    def get_route_order_ping(self, hostname, port, default=None):
        route = self.get_route_cache(hostname, port, None)
        if route:
            return sorted(route.values(), key=lambda x: x['tcp_ping'])
        return None

    def get_route_cache(self, hostname, port, default=None):
        return self.route_cache.get('%s-%s' % (hostname, port), default)

    def __set_route_cache(self, hostname, port, value):
        self.route_cache['%s-%s' % (hostname, port)] = value

    def update_route_ping(self, proxyName, hostname, port, ping, ip=None):
        proxyDict = self.get_route_cache(hostname, port)
        if proxyDict == None:
            proxyDict = {}
            self.__set_route_cache(hostname, port, proxyDict)

        proxyDict['%s-%s' % (proxyName, ip)] = {
            'tcp_ping': ping,
            'proxy_name': proxyName,
            'hit_ip': ip
        }

    def _create_connection(self, _upstream, aync_task, address, timeout=10):
        # 实际连接部分
        start_time = int(time.time() * 1000)
        try:
            sock = _upstream.create_connection(address, timeout)
        except:
            t = int(time.time() * 1000) - start_time
            info = traceback.format_exc()
            logging.debug(
                u'[upstream]%s 连接 %s:%s 失败。time:%s' % (_upstream.get_display_name(), address[0], address[1], t))
            logging.debug('%s\r\n\r\n' % info)
            return
        t = int(time.time() * 1000) - start_time
        self.update_route_ping(_upstream.get_name(),address[0],address[1],t)
        if aync_task.sock:
            sock.close(safe=False)
            logging.debug(
                u'[upstream]%s 连接 %s:%s 未命中。time:%s' % (_upstream.get_display_name(), address[0], address[1], t))
        else:
            aync_task.sock = sock
            aync_task.evt.set()
            logging.debug(
                u'[upstream]%s 连接 %s:%s 命中。time:%s' % (_upstream.get_display_name(), address[0], address[1], t))

    def _create_connection_all_end(self, aync_task):
        u"""在所有链接全部出错时发出事件通知主协程。"""
        aync_task.group.join()
        if aync_task.sock is None:
            aync_task.evt.set()

    def create_connection(self, address, timeout=10):

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0],address[1],None)
        if route_list:
            try:
                route = route_list[0]

                cache_timeout = route['tcp_ping']

                if cache_timeout<1000:
                    cache_timeout = cache_timeout * 2
                else:
                    cache_timeout = cache_timeout+1000
                cache_timeout = int(math.ceil(cache_timeout/1000.0))

                _upstream = self.upstream_dict.get(route['proxy_name'])
                start_time = int(time.time() * 1000)
                sock = _upstream.create_connection(address, cache_timeout)
                t = int(time.time() * 1000) - start_time
                logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 命中。time:%s'%(_upstream.get_display_name(),address[0],address[1],t))
                self.update_route_ping(_upstream.get_name(),address[0],address[1],t)
                return sock
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 失败。time:%s' % (_upstream.get_display_name(), address[0], address[1],t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败,连接全部
        evt = Event()
        group = Group()
        aync_task = MultipathAsyncTask(evt, None, group)

        for _upstream in self.upstream_dict.values():
            group.add(gevent.spawn(self._create_connection, _upstream, aync_task, address, timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError()

    def get_display_name(self):
        return '[%s]' % (self.type)

    def get_name(self):
        return '%s' % (self.type)
Пример #8
0
class HttpPool():
    u"""http 池"""

    def __init__(self, sock_mod=None, max_site=20, max_size_conn=10, expire=60, lock=None):
        if sock_mod is None:
            sock_mod = __import__("socket")
        self.sock_mod = sock_mod

        self.site_dict = LRUCacheDict(max_site, expire, _safe_del_connes)

        self.max_site_conn = max_size_conn

        if lock is None:
            self.lock = _lock()
        elif callable(lock):
            self.lock = lock()
        else:
            raise ValueError('callable(lock)==False')

    def get_conn(self, address):
        u"""获得连接

没有连接时自动创建链接。
        """
        host, port = address
        with self.lock:
            site_connes = self.site_dict.get(u'%s:%s' % (host, poll), None)
            if site_connes:
                while True:
                    try:
                        conn = site_connes.get(False)
                        if is_connection_dropped(conn):
                            try:
                                conn.close()
                            except:
                                pass
                        else:
                            return conn
                    except Empty:
                        break
        return HttpConn(self.sock_mod.create_connection((host, port)), self, host, port)

    def _put_conn(self, host, port, sock):
        u""" 将连接添加回连接池

会检查连接状态,不正常的连接会被抛弃。

        """
        if hasattr(self.sock_mod, "get_display_name"):
            sock_name = self.sock_mod.get_display_name()
        else:
            sock_name = None
        sock_info = 'sock_mod:%s host:%s port:%s' % (sock_name, host, port)

        if sock:
            if is_connection_dropped(sock):
                logging.debug(u'已关闭连接无法添加回连接池。%s' % sock_info)
                try:
                    sock.close()
                except:
                    pass
            else:
                with self.lock:
                    site_connes = self.site_dict.get(u'%s:%s' % (host, port), None)
                    if site_connes is None:
                        site_connes = LifoQueue(self.max_site_conn)
                    try:
                        site_connes.put(sock)
                        logging.debug(u'添加连接回连接池。 %s' % sock_info)
                    except Full:
                        logging.debug(u'连接池满. %s' % sock_info)
                        try:
                            sock.close()
                        except:
                            pass
                        return
                    self.site_dict[u'%s:%s' % (host, port)] = site_connes

    def __del__(self):
        with self.lock:
            del self.site_dict
Пример #9
0
ATYP_IPV4 = 0x01
ATYP_DOMAINNAME = 0x03
ATYP_IPV6 = 0x04

RE_IPV4_ADDR = re.compile(
    r'^((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)$')
RE_IPV6_ADDR = re.compile(
    r'^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$'
)

dnsPool = ThreadPool(20)
configIpBlacklist = []
errIP = {}
nameservers = 'system'
nameservers_backup = ['8.8.8.8', '208.67.222.222']
dns_cache = LRUCacheDict(500, 10 * 60 * 1000)


def get_addr_type(addr):
    u""" 分析地址类型(IPv4、IPv6、域名)
>>> get_addr_type('0.0.0.0') == ATYP_IPV4
True
>>> get_addr_type('255.255.255.255') == ATYP_IPV4
True
>>> get_addr_type('1.1.1.256') == ATYP_IPV4
False
>>> get_addr_type('1.1.0') == ATYP_IPV4
False
>>> get_addr_type('192.168.1.1') == ATYP_IPV4
True
>>> get_addr_type('fe80:0000:0000:0000:0204:61ff:fe9d:f156') == ATYP_IPV6
Пример #10
0
class HttpPool():
    u"""http 池"""
    def __init__(self,
                 sock_mod=None,
                 max_site=20,
                 max_size_conn=10,
                 expire=60,
                 lock=None):
        if sock_mod is None:
            sock_mod = __import__("socket")
        self.sock_mod = sock_mod

        self.site_dict = LRUCacheDict(max_site, expire, _safe_del_connes)

        self.max_site_conn = max_size_conn

        if lock is None:
            self.lock = _lock()
        elif callable(lock):
            self.lock = lock()
        else:
            raise ValueError('callable(lock)==False')

    def get_conn(self, address):
        u"""获得连接

没有连接时自动创建链接。
        """
        host, port = address
        with self.lock:
            site_connes = self.site_dict.get(u'%s:%s' % (host, poll), None)
            if site_connes:
                while True:
                    try:
                        conn = site_connes.get(False)
                        if is_connection_dropped(conn):
                            try:
                                conn.close()
                            except:
                                pass
                        else:
                            return conn
                    except Empty:
                        break
        return HttpConn(self.sock_mod.create_connection((host, port)), self,
                        host, port)

    def _put_conn(self, host, port, sock):
        u""" 将连接添加回连接池

会检查连接状态,不正常的连接会被抛弃。

        """
        if hasattr(self.sock_mod, "get_display_name"):
            sock_name = self.sock_mod.get_display_name()
        else:
            sock_name = None
        sock_info = 'sock_mod:%s host:%s port:%s' % (sock_name, host, port)

        if sock:
            if is_connection_dropped(sock):
                logging.debug(u'已关闭连接无法添加回连接池。%s' % sock_info)
                try:
                    sock.close()
                except:
                    pass
            else:
                with self.lock:
                    site_connes = self.site_dict.get(u'%s:%s' % (host, port),
                                                     None)
                    if site_connes is None:
                        site_connes = LifoQueue(self.max_site_conn)
                    try:
                        site_connes.put(sock)
                        logging.debug(u'添加连接回连接池。 %s' % sock_info)
                    except Full:
                        logging.debug(u'连接池满. %s' % sock_info)
                        try:
                            sock.close()
                        except:
                            pass
                        return
                    self.site_dict[u'%s:%s' % (host, port)] = site_connes

    def __del__(self):
        with self.lock:
            del self.site_dict
Пример #11
0
class DirectUpstream(UpstreamBase):
    u""" 直接连接 socket 模块

    使用方式为创建本类实例,然后把实例当作 socket 模块即可。所有的操作都会经过 config 配置的线路。
    """
    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config=config)

        self.source_ip = config.get('source_ip', '0.0.0.0')
        self.source_port = config.get('source_port', 0)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        if self.source_ip == '0.0.0.0' and self.source_port == 0:
            self.source_address = None
        else:
            self.source_address = (self.source_ip, self.source_port)

        class socket(SocketBase):
            def __init__(self,
                         family=_socket.AF_INET,
                         type=_socket.SOCK_STREAM,
                         proto=0,
                         _sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family,
                                                   type=type,
                                                   proto=proto)
                    _sock.bind(self.source_address)
                SocketBase.__init__(self, _sock)

        socket.source_address = self.source_address
        socket.upstream = self.upstream
        socket.display_name = self.get_display_name()
        socket.name = self.get_name()

        self.socket = socket

    def get_display_name(self):
        return '[%s]source_ip=%s,source_port=%s' % (self.type, self.source_ip,
                                                    self.source_port)

    def get_name(self):
        return '%s?source=%s&source_port=%s' % (self.type, self.source_ip,
                                                self.source_port)

    def get_route_order_ping(self, hostname, port, default=None):
        route = self.get_route_cache(hostname, port, None)
        if route:
            return sorted(route.values(), key=lambda x: x['tcp_ping'])
        return None

    def get_route_cache(self, hostname, port, default=None):
        return self.route_cache.get('%s-%s' % (hostname, port), default)

    def __set_route_cache(self, hostname, port, value):
        self.route_cache['%s-%s' % (hostname, port)] = value

    def update_route_ping(self, hostname, port, ping, ip):
        proxyDict = self.get_route_cache(hostname, port)
        if proxyDict == None:
            proxyDict = {}
            self.__set_route_cache(hostname, port, proxyDict)

        proxyDict['%s' % (ip)] = {'tcp_ping': ping, 'hit_ip': ip}

    def _direct_create_connection(self, address, ip, timeout=10):
        # 实际连接
        _sock = self.upstream.create_connection(
            (ip, address[1]), timeout, source_address=self.source_address)
        _sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
        sock = self.socket(_sock=_sock)
        return sock

    def _create_connection(self, aync_task, address, ip, timeout=10):
        # 多线程连接执行部分
        start_time = int(time.time() * 1000)
        try:
            sock = self._direct_create_connection(address, ip, timeout)
        except:
            t = int(time.time() * 1000) - start_time
            info = traceback.format_exc()
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 失败。time:%s' %
                (self.get_display_name(), address[0], ip, address[1], t))
            logging.debug('%s\r\n\r\n' % info)
            return
        t = int(time.time() * 1000) - start_time
        self.update_route_ping(address[0], address[1], t, ip)
        if aync_task.sock:
            sock.close(safe=False)
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 未命中。time:%s' %
                (self.get_display_name(), address[0], ip, address[1], t))
        else:
            aync_task.sock = sock
            aync_task.evt.set()
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 命中。time:%s' %
                (self.get_display_name(), address[0], ip, address[1], t))

    def _create_connection_all_end(self, aync_task):
        u"""在所有链接全部出错时发出事件通知主协程。"""
        aync_task.group.join()
        if aync_task.sock is None:
            aync_task.evt.set()

    def create_connection(self, address, timeout=10):

        ip_list = dnslib.dnsQuery(address[0])

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0], address[1], None)
        if route_list:
            try:
                route = route_list[0]
                hit_ip = route['hit_ip']

                if hit_ip in ip_list:
                    cache_timeout = route['tcp_ping']

                    if cache_timeout < 1000:
                        cache_timeout = cache_timeout * 2
                    else:
                        cache_timeout = cache_timeout + 1000
                    cache_timeout = int(math.ceil(cache_timeout / 1000.0))

                    start_time = int(time.time() * 1000)
                    sock = self._direct_create_connection(
                        address, hit_ip, cache_timeout)
                    t = int(time.time() * 1000) - start_time
                    logging.debug(
                        u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 命中。time:%s'
                        % (self.get_display_name(), address[0], hit_ip,
                           address[1], t))
                    self.update_route_ping(address[0], address[1], t, hit_ip)
                    return sock
                else:
                    logging.debug(
                        u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s IP 不匹配,放弃缓存。'
                        % (self.get_display_name(), address[0], hit_ip,
                           address[1]))
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 失败。time:%s' %
                    (self.get_display_name(), address[0], hit_ip, address[1],
                     t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败,连接全部
        evt = Event()
        group = Group()
        aync_task = DirectAsyncTask(evt, None, group)

        for ip in ip_list:
            group.add(
                gevent.spawn(self._create_connection, aync_task, address, ip,
                             timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError()
Пример #12
0
class MultipathUpstream(UpstreamBase):
    u""" 多路径 socket 模块

对于 tcp 连接,同时使用多个线路尝试连接,最终使用最快建立连接的线路。

使用方式为创建本类实例,然后把实例当作 socket 模块即可。所有的操作都会经过 config 配置的线路。

以后也许会弄个当连接多路复用。
"""
    def __init__(self, config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self, config)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        self._list = config.get('list', [{'type': 'direct'}])
        self.upstream_dict = {}
        for i in self._list:
            type = i.get('type')
            if not type:
                raise ConfigError(u'[upstream]代理类型不能为空! ')
            Upstream = upstream.get_upstream(type)
            u = Upstream(i)
            self.upstream_dict[u.get_name()] = u

    u'''
        class socket(SocketBase):
            def __init__(self, family=_socket.AF_INET, type=_socket.SOCK_STREAM, proto=0, _sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family, type=type, proto=proto)
                    _sock.bind((socket.source_ip, socket.source_port))
                SocketBase.__init__(self, _sock)

        socket.upstream = self.upstream

        self.socket = socket'''

    def get_route_order_ping(self, hostname, port, default=None):
        route = self.get_route_cache(hostname, port, None)
        if route:
            return sorted(route.values(), key=lambda x: x['tcp_ping'])
        return None

    def get_route_cache(self, hostname, port, default=None):
        return self.route_cache.get('%s-%s' % (hostname, port), default)

    def __set_route_cache(self, hostname, port, value):
        self.route_cache['%s-%s' % (hostname, port)] = value

    def update_route_ping(self, proxyName, hostname, port, ping, ip=None):
        proxyDict = self.get_route_cache(hostname, port)
        if proxyDict == None:
            proxyDict = {}
            self.__set_route_cache(hostname, port, proxyDict)

        proxyDict['%s-%s' % (proxyName, ip)] = {
            'tcp_ping': ping,
            'proxy_name': proxyName,
            'hit_ip': ip
        }

    def _create_connection(self, _upstream, aync_task, address, timeout=10):
        # 实际连接部分
        start_time = int(time.time() * 1000)
        try:
            sock = _upstream.create_connection(address, timeout)
        except:
            t = int(time.time() * 1000) - start_time
            info = traceback.format_exc()
            logging.debug(
                u'[upstream]%s 连接 %s:%s 失败。time:%s' %
                (_upstream.get_display_name(), address[0], address[1], t))
            logging.debug('%s\r\n\r\n' % info)
            return
        t = int(time.time() * 1000) - start_time
        self.update_route_ping(_upstream.get_name(), address[0], address[1], t)
        if aync_task.sock:
            sock.close(safe=False)
            logging.debug(
                u'[upstream]%s 连接 %s:%s 未命中。time:%s' %
                (_upstream.get_display_name(), address[0], address[1], t))
        else:
            aync_task.sock = sock
            aync_task.evt.set()
            logging.debug(
                u'[upstream]%s 连接 %s:%s 命中。time:%s' %
                (_upstream.get_display_name(), address[0], address[1], t))

    def _create_connection_all_end(self, aync_task):
        u"""在所有链接全部出错时发出事件通知主协程。"""
        aync_task.group.join()
        if aync_task.sock is None:
            aync_task.evt.set()

    def create_connection(self, address, timeout=10):

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0], address[1], None)
        if route_list:
            try:
                route = route_list[0]

                cache_timeout = route['tcp_ping']

                if cache_timeout < 1000:
                    cache_timeout = cache_timeout * 2
                else:
                    cache_timeout = cache_timeout + 1000
                cache_timeout = int(math.ceil(cache_timeout / 1000.0))

                _upstream = self.upstream_dict.get(route['proxy_name'])
                start_time = int(time.time() * 1000)
                sock = _upstream.create_connection(address, cache_timeout)
                t = int(time.time() * 1000) - start_time
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 命中。time:%s' %
                    (_upstream.get_display_name(), address[0], address[1], t))
                self.update_route_ping(_upstream.get_name(), address[0],
                                       address[1], t)
                return sock
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s:%s 失败。time:%s' %
                    (_upstream.get_display_name(), address[0], address[1], t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败,连接全部
        evt = Event()
        group = Group()
        aync_task = MultipathAsyncTask(evt, None, group)

        for _upstream in self.upstream_dict.values():
            group.add(
                gevent.spawn(self._create_connection, _upstream, aync_task,
                             address, timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError()

    def get_display_name(self):
        return '[%s]' % (self.type)

    def get_name(self):
        return '%s' % (self.type)
Пример #13
0
class DirectUpstream(UpstreamBase):
    u""" 直接连接 socket 模块

    使用方式为创建本类实例,然后把实例当作 socket 模块即可。所有的操作都会经过 config 配置的线路。
    """

    def __init__(self,config):
        u""" 初始化直连 socket 环境 """
        UpstreamBase.__init__(self,config=config)

        self.source_ip = config.get('source_ip','0.0.0.0')
        self.source_port = config.get('source_port',0)

        self.route_cache = LRUCacheDict(500, 10 * 60 * 1000)

        if self.source_ip == '0.0.0.0' and self.source_port==0:
            self.source_address = None
        else:
            self.source_address=(self.source_ip,self.source_port)

        class socket(SocketBase):
            def __init__(self, family=_socket.AF_INET, type=_socket.SOCK_STREAM, proto=0,_sock=None):
                if _sock is None:
                    _sock = socket.upsocket.socket(family=family,type=type,proto=proto)
                    _sock.bind(self.source_address)
                SocketBase.__init__(self,_sock)

        socket.source_address = self.source_address
        socket.upstream = self.upstream

        self.socket = socket

    def get_display_name(self):
        return '[%s]source_ip=%s,source_port=%s' % (self.type,self.source_ip,self.source_port)

    def get_name(self):
        return '%s?source=%s&source_port=%s' % (self.type,self.source_ip,self.source_port)

    def get_route_order_ping(self, hostname, port, default=None):
        route = self.get_route_cache(hostname, port, None)
        if route:
            return sorted(route.values(), key=lambda x: x['tcp_ping'])
        return None

    def get_route_cache(self, hostname, port, default=None):
        return self.route_cache.get('%s-%s' % (hostname, port), default)

    def __set_route_cache(self, hostname, port, value):
        self.route_cache['%s-%s' % (hostname, port)] = value

    def update_route_ping(self,  hostname, port, ping, ip):
        proxyDict = self.get_route_cache(hostname, port)
        if proxyDict == None:
            proxyDict = {}
            self.__set_route_cache(hostname, port, proxyDict)

        proxyDict['%s' % ( ip)] = {
            'tcp_ping': ping,
            'hit_ip': ip
        }

    def _direct_create_connection(self,address,ip, timeout=10):
        # 实际连接
        _sock = self.upstream.create_connection((ip,address[1]),timeout,source_address=self.source_address)
        sock = self.socket(_sock=_sock)
        _sock.setsockopt(_socket.IPPROTO_TCP,_socket.TCP_NODELAY, 1)
        return sock

    def _create_connection(self, aync_task, address,ip, timeout=10):
        # 多线程连接执行部分
        start_time = int(time.time() * 1000)
        try:
            sock = self._direct_create_connection(address,ip, timeout)
        except:
            t = int(time.time() * 1000) - start_time
            info = traceback.format_exc()
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 失败。time:%s' % (self.get_display_name(), address[0],ip, address[1], t))
            logging.debug('%s\r\n\r\n' % info)
            return
        t = int(time.time() * 1000) - start_time
        self.update_route_ping(address[0],address[1],t,ip)
        if aync_task.sock:
            sock.close(safe=False)
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 未命中。time:%s' % (self.get_display_name(), address[0],ip, address[1], t))
        else:
            aync_task.sock = sock
            aync_task.evt.set()
            logging.debug(
                u'[upstream]%s 连接 %s(%s):%s 命中。time:%s' % (self.get_display_name(), address[0],ip, address[1], t))

    def _create_connection_all_end(self, aync_task):
        u"""在所有链接全部出错时发出事件通知主协程。"""
        aync_task.group.join()
        if aync_task.sock is None:
            aync_task.evt.set()

    def create_connection(self, address, timeout=10):

        ip_list = dnslib.dnsQuery(address[0])

        # 尝试连接缓存
        route_list = self.get_route_order_ping(address[0],address[1],None)
        if route_list:
            try:
                route = route_list[0]
                hit_ip = route['hit_ip']

                if hit_ip in ip_list:
                    cache_timeout = route['tcp_ping']

                    if cache_timeout<1000:
                        cache_timeout = cache_timeout * 2
                    else:
                        cache_timeout = cache_timeout+1000
                    cache_timeout = int(math.ceil(cache_timeout/1000.0))

                    start_time = int(time.time() * 1000)
                    sock = self._direct_create_connection(address,hit_ip, cache_timeout)
                    t = int(time.time() * 1000) - start_time
                    logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 命中。time:%s'%(self.get_display_name(),address[0],hit_ip,address[1],t))
                    self.update_route_ping(address[0],address[1],t,hit_ip)
                    return sock
                else:
                    logging.debug(u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s IP 不匹配,放弃缓存。'%(self.get_display_name(),address[0],hit_ip,address[1]))
            except:
                t = int(time.time() * 1000) - start_time
                info = traceback.format_exc()
                logging.debug(
                    u'[upstream][RouteCache]%s 缓存记录 连接 %s(%s):%s 失败。time:%s' % (self.get_display_name(), address[0],hit_ip, address[1],t))
                logging.debug('%s\r\n\r\n' % info)

        # 缓存失败,连接全部
        evt = Event()
        group = Group()
        aync_task = DirectAsyncTask(evt, None, group)

        for ip in ip_list:
            group.add(gevent.spawn(self._create_connection,  aync_task, address,ip, timeout))

        # 所有连接失败时发出通知
        gevent.spawn(self._create_connection_all_end, aync_task)

        evt.wait()
        if aync_task.sock:
            return aync_task.sock
        else:
            raise UpstreamConnectError()