def get_location(self): if self.country_code: return self.country_code if time.time() - self.last_ckeck < 300: return self.country_code try: self.last_ckeck = time.time() ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0]) if ip.is_loopback or ip.is_private: from connection import create_connection from httputil import read_response_line, read_headers try: soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self) soc.sendall(b'GET / HTTP/1.1\r\nConnection: keep_alive\r\nHost: bot.whatismyipaddress.com\r\nAccept-Encoding: identity\r\nUser-Agent: Python-urllib/2.7\r\n\r\n') f = soc.makefile() line, version, status, reason = read_response_line(f) _, headers = read_headers(f) assert status == 200 ip = soc.recv(int(headers['Content-Length'])) if not ip: soc.close() raise ValueError('%s: ip address is empty' % self.name) self.country_code = ip_to_country_code(ip) soc.close() except Exception: sys.stderr.write(traceback.format_exc()) sys.stderr.flush() self.country_code = None else: self.country_code = ip_to_country_code(ip) finally: return self.country_code
def get_location(self): if time.time() - self.last_ckeck < 60: return ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0]) if ip.is_loopback or ip.is_private: from connection import create_connection from httputil import read_reaponse_line, read_headers try: soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self) soc.sendall( b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n' ) f = soc.makefile() line, version, status, reason = read_reaponse_line(f) _, headers = read_headers(f) assert status == 200 ip = soc.recv(int(headers['Content-Length'])) if not ip: soc.close() raise ValueError('%s: ip address is empty' % self.name) self.country_code = ip_to_country_code(ip) soc.close() except Exception: sys.stderr.write(traceback.format_exc()) sys.stderr.flush() self.country_code = None else: self.country_code = ip_to_country_code(ip) self.last_ckeck = time.time()
def get_location(self): if time.time() - self.last_ckeck < 60: return ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0]) if ip.is_loopback or ip.is_private: from connection import create_connection from httputil import read_reaponse_line, read_headers try: soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via) soc.sendall(b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n') f = soc.makefile() line, version, status, reason = read_reaponse_line(f) _, headers = read_headers(f) assert status == 200 ip = soc.recv(int(headers['Content-Length'])) if not ip: soc.close() raise ValueError('%s: ip address is empty' % self.name) self.country_code = ip_to_country_code(ip) soc.close() except Exception: sys.stderr.write(traceback.format_exc()) sys.stderr.flush() self.country_code = None else: self.country_code = ip_to_country_code(ip) self.last_ckeck = time.time()
def ifhost_in_region(self, host, ip): try: code = ip_to_country_code(ip) if code in self.conf.region: self.logger.info('%s in %s' % (host, code)) return True return False except: pass
def ifhost_in_region(self, host, ip): try: code = ip_to_country_code(ip) if code in self.conf.region: self.logger.info('%s in %s' % (host, code)) return True return False except Exception: pass
def get_proxy(self, uri, host, command, ip, level=1): ''' decide which parentproxy to use. url: 'www.google.com:443' 'http://www.inxian.com' host: ('www.google.com', 443) (without port number is allowed) level: 0 -- direct 1 -- auto: proxy if local_rule, direct if ip in region or override, proxy if gfwlist 2 -- encrypt all: proxy if local_rule, direct if ip in region or override, proxy if gfwlist or not https 3 -- chnroute: proxy if local_rule, direct if ip in region or override, proxy for all 4 -- global: proxy if not local 5 -- global: proxy if not localhost ''' host, port = host ifgfwed = self.ifgfwed(uri, host, port, ip, level) if ifgfwed is False: if ip and ip.is_private: return [ self.conf.parentlist.local or self.conf.parentlist.direct ] return [self.conf.parentlist.direct] parentlist = list(self.conf.parentlist.httpsparents() if command == 'CONNECT' else self.conf.parentlist.httpparents()) if len(parentlist) < self.conf.maxretry: parentlist.extend(parentlist[1:] if not ifgfwed else parentlist) parentlist = parentlist[:self.conf.maxretry] location = ip_to_country_code(ip) or u'None' def priority(parent): return parent.priority(command, host, location) if len(parentlist) > 1: random.shuffle(parentlist) parentlist = sorted(parentlist, key=priority) if ifgfwed: if not parentlist: self.logger.warning( 'No parent proxy available, direct connection is used') return [self.conf.parentlist.get('direct')] else: parentlist.insert(0, self.conf.parentlist.direct) if len(parentlist ) == 1 and parentlist[0] is self.conf.parentlist.get('direct'): return parentlist if len(parentlist) > self.conf.maxretry: parentlist = parentlist[:self.conf.maxretry] return parentlist
def parentproxy(self, uri, host, command, ip, level=1): """ decide which parentproxy to use. url: 'www.google.com:443' 'http://www.inxian.com' host: ('www.google.com', 443) (no port number is allowed) level: 0 -- direct 1 -- auto: proxy if local_rule, direct if ip in region or override, proxy if gfwlist 2 -- encrypt all: proxy if local_rule or not https, direct if ip in region or override, proxy if gfwlist 3 -- chnroute: proxy if local_rule, direct if ip in region or override, proxy if all 4 -- global: proxy if not local 5 -- global: proxy if not localhost """ host, port = host ifgfwed = self.ifgfwed(uri, host, port, ip, level) if ifgfwed is False: if ip and ip.is_private: return [self.conf.parentlist.local or self.conf.parentlist.direct] return [self.conf.parentlist.direct] parentlist = list( self.conf.parentlist.httpsparents() if command == "CONNECT" else self.conf.parentlist.httpparents() ) if len(parentlist) < self.conf.maxretry: parentlist.extend(parentlist[1:] if not ifgfwed else parentlist) parentlist = parentlist[: self.conf.maxretry] location = ip_to_country_code(ip) or u"None" def priority(parent): return parent.priority(command, host, location) if len(parentlist) > 1: random.shuffle(parentlist) parentlist = sorted(parentlist, key=priority) if ifgfwed: if not parentlist: self.logger.warning("No parent proxy available, direct connection is used") return [self.conf.parentlist.get("direct")] else: parentlist.insert(0, self.conf.parentlist.direct) if len(parentlist) == 1 and parentlist[0] is self.conf.parentlist.get("direct"): return parentlist if len(parentlist) > self.conf.maxretry: parentlist = parentlist[: self.conf.maxretry] return parentlist
def get_location(self): if self.country_code: return self.country_code if time.time() - self.last_ckeck < 300: return self.country_code try: self.last_ckeck = time.time() ip = ip_address( socket.getaddrinfo(self.parse.hostname, 0)[0][4][0]) if ip.is_loopback or ip.is_private: from connection import create_connection from httputil import read_response_line, read_headers try: soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self) soc.sendall( b'GET / HTTP/1.1\r\nConnection: keep_alive\r\nHost: bot.whatismyipaddress.com\r\nAccept-Encoding: identity\r\nUser-Agent: Python-urllib/2.7\r\n\r\n' ) f = soc.makefile() line, version, status, reason = read_response_line(f) _, headers = read_headers(f) assert status == 200 ip = soc.recv(int(headers['Content-Length'])) if not ip: soc.close() raise ValueError('%s: ip address is empty' % self.name) self.country_code = ip_to_country_code(ip) soc.close() except Exception: sys.stderr.write(traceback.format_exc()) sys.stderr.flush() self.country_code = None else: self.country_code = ip_to_country_code(ip) finally: return self.country_code
def priority(parent): priority = parent.httpspriority if command == 'CONNECT' else parent.httppriority avg_time = self.STATS.avg_time(parent.name, host) if not ip: return priority + avg_time * 10 result = priority if parent.country_code is None: parent.get_location() if parent.country_code is None: result = priority + 3 parent_cc = parent.country_code dest = '' dest = ip_to_country_code(ip) if parent_cc == dest: result = priority - 3 else: for continent in continent_list: if parent_cc in continent and dest in continent: result = priority - 1 break return result + avg_time * 10
def get_location(self): if time.time() - self.last_ckeck < 60: return from connection import create_connection from httputil import read_reaponse_line, read_headers try: soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via) soc.sendall(b'GET / HTTP/1.0\r\nHost: bot.whatismyipaddress.com\r\n\r\n') f = soc.makefile() line, version, status, reason = read_reaponse_line(f) _, headers = read_headers(f) assert status == 200 ip = soc.recv(int(headers['Content-Length'])) if not ip: soc.close() raise ValueError('%s: ip address is empty' % self.name) self.country_code = ip_to_country_code(ip) soc.close() except Exception as e: sys.stderr.write(traceback.format_exc()) sys.stderr.flush() self.country_code = None self.last_ckeck = time.time()