Example #1
0
    def get_location(self):
        if self.country_code:
            return self.country_code
        if time.time() - self.last_ckeck < 300:
            return self.country_code
        try:
            self.last_ckeck = time.time()
            ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])

            if ip.is_loopback or ip.is_private:
                from connection import create_connection
                from httputil import read_response_line, read_headers
                try:
                    soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self)
                    soc.sendall(b'GET / HTTP/1.1\r\nConnection: keep_alive\r\nHost: bot.whatismyipaddress.com\r\nAccept-Encoding: identity\r\nUser-Agent: Python-urllib/2.7\r\n\r\n')
                    f = soc.makefile()
                    line, version, status, reason = read_response_line(f)
                    _, headers = read_headers(f)
                    assert status == 200
                    ip = soc.recv(int(headers['Content-Length']))
                    if not ip:
                        soc.close()
                        raise ValueError('%s: ip address is empty' % self.name)
                    self.country_code = ip_to_country_code(ip)
                    soc.close()
                except Exception:
                    sys.stderr.write(traceback.format_exc())
                    sys.stderr.flush()
                    self.country_code = None
            else:
                self.country_code = ip_to_country_code(ip)
        finally:
            return self.country_code
Example #2
0
    def get_location(self):
        if time.time() - self.last_ckeck < 60:
            return
        ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])

        if ip.is_loopback or ip.is_private:
            from connection import create_connection
            from httputil import read_reaponse_line, read_headers
            try:
                soc = create_connection(('bot.whatismyipaddress.com', 80),
                                        ctimeout=None,
                                        parentproxy=self)
                soc.sendall(
                    b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n'
                )
                f = soc.makefile()
                line, version, status, reason = read_reaponse_line(f)
                _, headers = read_headers(f)
                assert status == 200
                ip = soc.recv(int(headers['Content-Length']))
                if not ip:
                    soc.close()
                    raise ValueError('%s: ip address is empty' % self.name)
                self.country_code = ip_to_country_code(ip)
                soc.close()
            except Exception:
                sys.stderr.write(traceback.format_exc())
                sys.stderr.flush()
                self.country_code = None
        else:
            self.country_code = ip_to_country_code(ip)
        self.last_ckeck = time.time()
Example #3
0
    def get_location(self):
        if time.time() - self.last_ckeck < 60:
            return
        ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])

        if ip.is_loopback or ip.is_private:
            from connection import create_connection
            from httputil import read_reaponse_line, read_headers
            try:
                soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via)
                soc.sendall(b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n')
                f = soc.makefile()
                line, version, status, reason = read_reaponse_line(f)
                _, headers = read_headers(f)
                assert status == 200
                ip = soc.recv(int(headers['Content-Length']))
                if not ip:
                    soc.close()
                    raise ValueError('%s: ip address is empty' % self.name)
                self.country_code = ip_to_country_code(ip)
                soc.close()
            except Exception:
                sys.stderr.write(traceback.format_exc())
                sys.stderr.flush()
                self.country_code = None
        else:
            self.country_code = ip_to_country_code(ip)
        self.last_ckeck = time.time()
Example #4
0
 def ifhost_in_region(self, host, ip):
     try:
         code = ip_to_country_code(ip)
         if code in self.conf.region:
             self.logger.info('%s in %s' % (host, code))
             return True
         return False
     except:
         pass
Example #5
0
 def ifhost_in_region(self, host, ip):
     try:
         code = ip_to_country_code(ip)
         if code in self.conf.region:
             self.logger.info('%s in %s' % (host, code))
             return True
         return False
     except Exception:
         pass
Example #6
0
    def get_proxy(self, uri, host, command, ip, level=1):
        '''
            decide which parentproxy to use.
            url:  'www.google.com:443'
                  'http://www.inxian.com'
            host: ('www.google.com', 443) (without port number is allowed)
            level: 0 -- direct
                   1 -- auto:        proxy if local_rule, direct if ip in region or override, proxy if gfwlist
                   2 -- encrypt all: proxy if local_rule, direct if ip in region or override, proxy if gfwlist or not https
                   3 -- chnroute:    proxy if local_rule, direct if ip in region or override, proxy for all
                   4 -- global:      proxy if not local
                   5 -- global:      proxy if not localhost
        '''
        host, port = host

        ifgfwed = self.ifgfwed(uri, host, port, ip, level)

        if ifgfwed is False:
            if ip and ip.is_private:
                return [
                    self.conf.parentlist.local or self.conf.parentlist.direct
                ]
            return [self.conf.parentlist.direct]

        parentlist = list(self.conf.parentlist.httpsparents() if command ==
                          'CONNECT' else self.conf.parentlist.httpparents())
        if len(parentlist) < self.conf.maxretry:
            parentlist.extend(parentlist[1:] if not ifgfwed else parentlist)
            parentlist = parentlist[:self.conf.maxretry]

        location = ip_to_country_code(ip) or u'None'

        def priority(parent):
            return parent.priority(command, host, location)

        if len(parentlist) > 1:
            random.shuffle(parentlist)
            parentlist = sorted(parentlist, key=priority)

        if ifgfwed:
            if not parentlist:
                self.logger.warning(
                    'No parent proxy available, direct connection is used')
                return [self.conf.parentlist.get('direct')]
        else:
            parentlist.insert(0, self.conf.parentlist.direct)

        if len(parentlist
               ) == 1 and parentlist[0] is self.conf.parentlist.get('direct'):
            return parentlist

        if len(parentlist) > self.conf.maxretry:
            parentlist = parentlist[:self.conf.maxretry]
        return parentlist
Example #7
0
    def parentproxy(self, uri, host, command, ip, level=1):
        """
            decide which parentproxy to use.
            url:  'www.google.com:443'
                  'http://www.inxian.com'
            host: ('www.google.com', 443) (no port number is allowed)
            level: 0 -- direct
                   1 -- auto:        proxy if local_rule, direct if ip in region or override, proxy if gfwlist
                   2 -- encrypt all: proxy if local_rule or not https, direct if ip in region or override, proxy if gfwlist
                   3 -- chnroute:    proxy if local_rule, direct if ip in region or override, proxy if all
                   4 -- global:      proxy if not local
                   5 -- global:      proxy if not localhost
        """
        host, port = host

        ifgfwed = self.ifgfwed(uri, host, port, ip, level)

        if ifgfwed is False:
            if ip and ip.is_private:
                return [self.conf.parentlist.local or self.conf.parentlist.direct]
            return [self.conf.parentlist.direct]

        parentlist = list(
            self.conf.parentlist.httpsparents() if command == "CONNECT" else self.conf.parentlist.httpparents()
        )
        if len(parentlist) < self.conf.maxretry:
            parentlist.extend(parentlist[1:] if not ifgfwed else parentlist)
            parentlist = parentlist[: self.conf.maxretry]

        location = ip_to_country_code(ip) or u"None"

        def priority(parent):
            return parent.priority(command, host, location)

        if len(parentlist) > 1:
            random.shuffle(parentlist)
            parentlist = sorted(parentlist, key=priority)

        if ifgfwed:
            if not parentlist:
                self.logger.warning("No parent proxy available, direct connection is used")
                return [self.conf.parentlist.get("direct")]
        else:
            parentlist.insert(0, self.conf.parentlist.direct)

        if len(parentlist) == 1 and parentlist[0] is self.conf.parentlist.get("direct"):
            return parentlist

        if len(parentlist) > self.conf.maxretry:
            parentlist = parentlist[: self.conf.maxretry]
        return parentlist
Example #8
0
    def get_location(self):
        if self.country_code:
            return self.country_code
        if time.time() - self.last_ckeck < 300:
            return self.country_code
        try:
            self.last_ckeck = time.time()
            ip = ip_address(
                socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])

            if ip.is_loopback or ip.is_private:
                from connection import create_connection
                from httputil import read_response_line, read_headers
                try:
                    soc = create_connection(('bot.whatismyipaddress.com', 80),
                                            ctimeout=None,
                                            parentproxy=self)
                    soc.sendall(
                        b'GET / HTTP/1.1\r\nConnection: keep_alive\r\nHost: bot.whatismyipaddress.com\r\nAccept-Encoding: identity\r\nUser-Agent: Python-urllib/2.7\r\n\r\n'
                    )
                    f = soc.makefile()
                    line, version, status, reason = read_response_line(f)
                    _, headers = read_headers(f)
                    assert status == 200
                    ip = soc.recv(int(headers['Content-Length']))
                    if not ip:
                        soc.close()
                        raise ValueError('%s: ip address is empty' % self.name)
                    self.country_code = ip_to_country_code(ip)
                    soc.close()
                except Exception:
                    sys.stderr.write(traceback.format_exc())
                    sys.stderr.flush()
                    self.country_code = None
            else:
                self.country_code = ip_to_country_code(ip)
        finally:
            return self.country_code
Example #9
0
 def priority(parent):
     priority = parent.httpspriority if command == 'CONNECT' else parent.httppriority
     avg_time = self.STATS.avg_time(parent.name, host)
     if not ip:
         return priority + avg_time * 10
     result = priority
     if parent.country_code is None:
         parent.get_location()
     if parent.country_code is None:
         result = priority + 3
     parent_cc = parent.country_code
     dest = ''
     dest = ip_to_country_code(ip)
     if parent_cc == dest:
         result = priority - 3
     else:
         for continent in continent_list:
             if parent_cc in continent and dest in continent:
                 result = priority - 1
                 break
     return result + avg_time * 10
Example #10
0
 def get_location(self):
     if time.time() - self.last_ckeck < 60:
         return
     from connection import create_connection
     from httputil import read_reaponse_line, read_headers
     try:
         soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via)
         soc.sendall(b'GET / HTTP/1.0\r\nHost: bot.whatismyipaddress.com\r\n\r\n')
         f = soc.makefile()
         line, version, status, reason = read_reaponse_line(f)
         _, headers = read_headers(f)
         assert status == 200
         ip = soc.recv(int(headers['Content-Length']))
         if not ip:
             soc.close()
             raise ValueError('%s: ip address is empty' % self.name)
         self.country_code = ip_to_country_code(ip)
         soc.close()
     except Exception as e:
         sys.stderr.write(traceback.format_exc())
         sys.stderr.flush()
         self.country_code = None
     self.last_ckeck = time.time()