def get_location(self):
if time.time() - self.last_ckeck < 60:
return
ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])
if ip.is_loopback or ip.is_private:
from connection import create_connection
from httputil import read_reaponse_line, read_headers
try:
soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via)
soc.sendall(b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n')
f = soc.makefile()
line, version, status, reason = read_reaponse_line(f)
_, headers = read_headers(f)
assert status == 200
ip = soc.recv(int(headers['Content-Length']))
if not ip:
soc.close()
raise ValueError('%s: ip address is empty' % self.name)
self.country_code = ip_to_country_code(ip)
soc.close()
except Exception:
sys.stderr.write(traceback.format_exc())
sys.stderr.flush()
self.country_code = None
else:
self.country_code = ip_to_country_code(ip)
self.last_ckeck = time.time()
def get_location(self):
if self.country_code:
return self.country_code
if time.time() - self.last_ckeck < 300:
return self.country_code
try:
self.last_ckeck = time.time()
ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])
if ip.is_loopback or ip.is_private:
from connection import create_connection
from httputil import read_reaponse_line, read_headers
try:
soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self)
soc.sendall(b'GET / HTTP/1.1\r\nConnection: keep_alive\r\nHost: bot.whatismyipaddress.com\r\nAccept-Encoding: identity\r\nUser-Agent: Python-urllib/2.7\r\n\r\n')
f = soc.makefile()
line, version, status, reason = read_reaponse_line(f)
_, headers = read_headers(f)
assert status == 200
ip = soc.recv(int(headers['Content-Length']))
if not ip:
soc.close()
raise ValueError('%s: ip address is empty' % self.name)
self.country_code = ip_to_country_code(ip)
soc.close()
except Exception:
sys.stderr.write(traceback.format_exc())
sys.stderr.flush()
self.country_code = None
else:
self.country_code = ip_to_country_code(ip)
finally:
return self.country_code
def get_location(self):
if time.time() - self.last_ckeck < 60:
return
ip = ip_address(socket.getaddrinfo(self.parse.hostname, 0)[0][4][0])
if ip.is_loopback or ip.is_private:
from connection import create_connection
from httputil import read_reaponse_line, read_headers
try:
soc = create_connection(('bot.whatismyipaddress.com', 80),
ctimeout=None,
parentproxy=self)
soc.sendall(
b'GET / HTTP/1.1\r\nConnection: close\r\nHost: bot.whatismyipaddress.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0\r\n\r\n'
)
f = soc.makefile()
line, version, status, reason = read_reaponse_line(f)
_, headers = read_headers(f)
assert status == 200
ip = soc.recv(int(headers['Content-Length']))
if not ip:
soc.close()
raise ValueError('%s: ip address is empty' % self.name)
self.country_code = ip_to_country_code(ip)
soc.close()
except Exception:
sys.stderr.write(traceback.format_exc())
sys.stderr.flush()
self.country_code = None
else:
self.country_code = ip_to_country_code(ip)
self.last_ckeck = time.time()
def do_tunnel(soc, netloc, pp):
s = ['CONNECT %s:%s HTTP/1.1\r\n' % (netloc[0], netloc[1]), ]
if pp.username:
a = '%s:%s' % (pp.username, pp.password)
s.append('Proxy-Authorization: Basic %s\r\n' % base64.b64encode(a.encode()))
s.append('Host: %s:%s\r\n\r\n' % (netloc[0], netloc[1]))
soc.sendall(''.join(s).encode())
remoterfile = soc.makefile('rb', 0)
line, version, status, reason = read_reaponse_line(remoterfile)
if status != 200:
raise IOError(0, 'create tunnel via %s failed!' % pp.name)
read_header_data(remoterfile)
def do_tunnel(soc, netloc, pp):
s = [
'CONNECT %s:%s HTTP/1.1\r\n' % (netloc[0], netloc[1]),
]
if pp.username:
a = '%s:%s' % (pp.username, pp.password)
s.append('Proxy-Authorization: Basic %s\r\n' %
base64.b64encode(a.encode()))
s.append('Host: %s:%s\r\n\r\n' % (netloc[0], netloc[1]))
soc.sendall(''.join(s).encode())
remoterfile = soc.makefile('rb', 0)
line, version, status, reason = read_reaponse_line(remoterfile)
if status != 200:
raise IOError(0, 'create tunnel via %s failed!' % pp.name)
read_header_data(remoterfile)
def get_location(self):
if time.time() - self.last_ckeck < 60:
return
from connection import create_connection
from httputil import read_reaponse_line, read_headers
try:
soc = create_connection(('bot.whatismyipaddress.com', 80), ctimeout=None, parentproxy=self, via=self.via)
soc.sendall(b'GET / HTTP/1.0\r\nHost: bot.whatismyipaddress.com\r\n\r\n')
f = soc.makefile()
line, version, status, reason = read_reaponse_line(f)
_, headers = read_headers(f)
assert status == 200
ip = soc.recv(int(headers['Content-Length']))
if not ip:
soc.close()
raise ValueError('%s: ip address is empty' % self.name)
self.country_code = ip_to_country_code(ip)
soc.close()
except Exception as e:
sys.stderr.write(traceback.format_exc())
sys.stderr.flush()
self.country_code = None
self.last_ckeck = time.time()
def _do_GET(self, retry=False):
try:
if retry:
if self.remotesoc:
try:
self.remotesoc.close()
except:
pass
self.remotesoc = None
self.failed_parents.append(self.ppname)
self.count += 1
if self.count > 10:
self.logger.error('for some strange reason retry time exceeded 10, pls check!')
return
if not self.retryable:
self.close_connection = 1
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return
if self.getparent():
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return self.send_error(504)
self.upstream_name = self.ppname if self.pproxy.proxy.startswith('http') else self.requesthost
iplist = None
if self.pproxy.name == 'direct' and self.requesthost[0] in self.conf.HOSTS and not self.failed_parents:
iplist = self.conf.HOSTS.get(self.requesthost[0])
self._proxylist.insert(0, self.pproxy)
self.set_timeout()
self.phase = 'http_connect_via_proxy'
self.remotesoc = self._http_connect_via_proxy(self.requesthost, iplist)
self.wbuffer = deque()
self.wbuffer_size = 0
# send request header
self.phase = 'sending request header'
s = []
if self.pproxy.proxy.startswith('http'):
path = self.path
if iplist:
path = self.path.split('/')
path[2] = '%s%s' % (iplist[0][1], ((':%d' % self.requesthost[1]) if self.requesthost[1] != 80 else ''))
path = ''.join(path)
s.append('%s %s %s\r\n' % (self.command, self.path, self.request_version))
if self.pproxy.username:
a = '%s:%s' % (self.pproxy.username, self.pproxy.password)
self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(a.encode())
else:
s.append('%s /%s %s\r\n' % (self.command, '/'.join(self.path.split('/')[3:]), self.request_version))
# Does the client want to close connection after this request?
conntype = self.headers.get('Connection', "")
if self.request_version >= b"HTTP/1.1":
client_close = 'close' in conntype.lower()
else:
client_close = 'keep_alive' in conntype.lower()
if 'Upgrade' in self.headers:
if 'websocket' in self.headers['Upgrade']:
self.headers['Upgrade'] = 'websocket'
client_close = True
else:
self.logger.warning('Upgrade header found! (%s), FW-Lite do not support this...' % self.headers['Upgrade'])
del self.headers['Upgrade']
else:
self.headers['Connection'] = 'keep_alive'
del self.headers['Proxy-Connection']
for k, v in self.headers.items():
if isinstance(v, bytes):
v = v.decode('latin1')
s.append("%s: %s\r\n" % ("-".join([w.capitalize() for w in k.split("-")]), v))
s.append("\r\n")
data = ''.join(s).encode('latin1')
self.remotesoc.sendall(data)
self.traffic_count[0] += len(data)
# Now remotesoc is connected, set read timeout
self.remotesoc.settimeout(self.rtimeout)
remoterfile = self.remotesoc.makefile('rb', 0)
# Expect
skip = False
if 'Expect' in self.headers:
try:
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
except Exception as e:
# TODO: probably the server don't handle Expect well.
self.logger.warning('read response line error: %r' % e)
else:
if response_status == 100:
hdata = read_header_data(remoterfile)
self._wfile_write(response_line + hdata)
else:
skip = True
# send request body
if not skip:
self.phase = 'sending request body'
content_length = int(self.headers.get('Content-Length', 0))
if self.headers.get("Transfer-Encoding") and self.headers.get("Transfer-Encoding") != "identity":
if self.rbuffer:
self.remotesoc.sendall(b''.join(self.rbuffer))
flag = 1
req_body_len = 0
while flag:
trunk_lenth = self.rfile_readline()
if self.retryable:
self.rbuffer.append(trunk_lenth)
req_body_len += len(trunk_lenth)
self.remotesoc.sendall(trunk_lenth)
trunk_lenth = int(trunk_lenth.strip(), 16) + 2
flag = trunk_lenth != 2
data = self.rfile_read(trunk_lenth)
if self.retryable:
self.rbuffer.append(data)
req_body_len += len(data)
self.remotesoc.sendall(data)
if req_body_len > 102400:
self.retryable = False
self.rbuffer = deque()
elif content_length > 0:
if content_length > 102400:
self.retryable = False
if self.rbuffer:
s = b''.join(self.rbuffer)
content_length -= len(s)
self.remotesoc.sendall(s)
while content_length:
data = self.rfile_read(min(self.bufsize, content_length))
if not data:
break
content_length -= len(data)
if self.retryable:
self.rbuffer.append(data)
self.remotesoc.sendall(data)
# read response line
timelog = time.clock()
self.phase = 'reading response_line'
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
rtime = time.clock() - timelog
# read response headers
while response_status == 100:
hdata = read_header_data(remoterfile)
self._wfile_write(response_line + hdata)
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
self.phase = 'reading response header'
header_data, response_header = read_headers(remoterfile)
conntype = response_header.get('Connection', "")
if protocol_version >= b"HTTP/1.1":
remote_close = 'close' in conntype.lower()
else:
remote_close = 'keep_alive' in conntype.lower()
if 'Upgrade' in response_header:
remote_close = True
if "Content-Length" in response_header:
if "," in response_header["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', response_header["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
response_header["Content-Length"])
response_header["Content-Length"] = pieces[0]
content_length = int(response_header["Content-Length"])
else:
content_length = None
buf = io.BytesIO(header_data)
header_data = b''
for line in buf:
if line.startswith('Connection') and 'Upgrade' not in line:
header_data += b'Connection: close\r\n' if client_close else b'Connection: keep_alive\r\n'
else:
header_data += line
self.wfile_write(response_line)
self.wfile_write(header_data)
# verify
if response_status in (301, 302) and self.conf.PARENT_PROXY.bad302(response_header.get('Location')):
raise IOError(0, 'Bad 302!')
# read response body
self.phase = 'reading response body'
if self.command == 'HEAD' or response_status in (204, 205, 304):
pass
elif response_header.get("Transfer-Encoding") and response_header.get("Transfer-Encoding") != "identity":
flag = 1
while flag:
trunk_lenth = remoterfile.readline()
self.wfile_write(trunk_lenth)
trunk_lenth = int(trunk_lenth.strip(), 16) + 2
flag = trunk_lenth != 2
while trunk_lenth:
data = self.remotesoc.recv(min(self.bufsize, trunk_lenth))
trunk_lenth -= len(data)
self.wfile_write(data)
elif content_length is not None:
while content_length:
data = self.remotesoc.recv(min(self.bufsize, content_length))
if not data:
raise IOError(0, 'remote socket closed')
content_length -= len(data)
self.wfile_write(data)
else:
# websocket?
self.close_connection = 1
self.retryable = False
self.wfile_write()
fd = [self.connection, self.remotesoc]
while fd:
ins, _, _ = select.select(fd, [], [], 60)
if not ins:
break
if self.connection in ins:
data = self.connection_recv(self.bufsize)
if data:
self.remotesoc.sendall(data)
else:
fd.remove(self.connection)
self.remotesoc.shutdown(socket.SHUT_WR)
if self.remotesoc in ins:
data = self.remotesoc.recv(self.bufsize)
if data:
self._wfile_write(data)
else:
fd.remove(self.remotesoc)
self.connection.shutdown(socket.SHUT_WR)
self.wfile_write()
self.phase = 'request finish'
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, True if response_status < 400 else False, self.failed_parents, self.ppname, rtime)
self.pproxy.log(self.requesthost[0], rtime)
if remote_close or is_connection_dropped([self.remotesoc]):
self.remotesoc.close()
else:
self.HTTPCONN_POOL.put(self.upstream_name, self.remotesoc, self.ppname if '(pooled)' in self.ppname else (self.ppname + '(pooled)'))
self.remotesoc = None
except ClientError as e:
raise
except NetWorkIOError as e:
return self.on_GET_Error(e)