def _purge(self):
while 1:
time.sleep(self.intv)
pcount = 0
with self.lock:
remove_lst = []
for soc in is_connection_dropped(self.socs.keys()):
soc.close()
remove_lst.append(soc)
pcount += 1
for soc in remove_lst:
self._remove(soc)
remove_lst = []
if pcount:
self.logger.debug('%s closed for connection droped.' % pcount)
self.timerwheel_index = next(self.timerwheel_iter)
for soc in list(self.timerwheel[self.timerwheel_index]):
soc.close()
remove_lst.append(soc)
pcount += 1
for soc in remove_lst:
self._remove(soc)
if pcount:
self.logger.debug('%d remotesoc purged, %d in connection pool.(%s)' % (pcount, len(self.socs), ', '.join([k[0] if isinstance(k, tuple) else k for k, v in self.POOL.items() if v])))
def get(cls, upstream_name):
lst = cls.POOL.get(upstream_name)
while lst:
sock, pproxy = lst.popleft()
if not is_connection_dropped(sock):
return (sock, pproxy)
sock.close()
def get(self, upstream_name):
with self.lock:
lst = self.POOL.get(upstream_name)
while lst:
sock, pproxy = lst.popleft()
if is_connection_dropped([sock]):
sock.close()
self._remove(sock)
continue
self._remove(sock)
return (sock, pproxy)
def get(self, upstream_name):
with self.lock:
lst = self.POOL.get(upstream_name)
while lst:
sock, pproxy = lst.popleft()
if is_connection_dropped([sock]):
sock.close()
self._remove(sock)
continue
self._remove(sock)
return (sock, pproxy)
def _purge(self):
pcount = 0
with self.lock:
for soc in is_connection_dropped(self.socs.keys()):
soc.close()
self._remove(soc)
pcount += 1
self.timerwheel_index = next(self.timerwheel_iter)
for soc in list(self.timerwheel[self.timerwheel_index]):
soc.close()
self._remove(soc)
pcount += 1
if pcount:
self.logger.debug('%d remotesoc purged, %d in connection pool.(%s)' % (pcount, len(self.socs), ', '.join([k[0] if isinstance(k, tuple) else k for k, v in self.POOL.items() if v])))
Timer(30, self._purge, ()).start()
def _do_GET(self, retry=False):
if retry:
if self.remotesoc:
self.remotesoc.close()
self.remotesoc = None
self.failed_parents.append(self.ppname)
if not self.retryable:
self.close_connection = 1
PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return
if self.getparent():
PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return self.send_error(504)
self.upstream_name = self.ppname if self.pproxy.startswith('http') else self.requesthost
try:
self.remotesoc = self._http_connect_via_proxy(self.requesthost)
except NetWorkIOError as e:
return self.on_GET_Error(e)
self.wbuffer = deque()
self.wbuffer_size = 0
# send request header
logging.debug('sending request header')
s = []
if self.pproxy.startswith('http'):
s.append('%s %s %s\r\n' % (self.command, self.path, self.request_version))
if self.pproxyparse.username:
a = '%s:%s' % (self.pproxyparse.username, self.pproxyparse.password)
self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(a.encode())
else:
s.append('%s /%s %s\r\n' % (self.command, '/'.join(self.path.split('/')[3:]), self.request_version))
del self.headers['Proxy-Connection']
for k, v in self.headers.items():
if isinstance(v, bytes):
v = v.decode('latin1')
s.append("%s: %s\r\n" % ("-".join([w.capitalize() for w in k.split("-")]), v))
s.append("\r\n")
try:
self.remotesoc.sendall(''.join(s).encode('latin1'))
except NetWorkIOError as e:
return self.on_GET_Error(e)
logging.debug('sending request body')
# send request body
content_length = int(self.headers.get('Content-Length', 0))
if content_length:
if content_length > 102400:
self.retryable = False
if self.rbuffer:
s = b''.join(self.rbuffer)
content_length -= len(s)
try:
self.remotesoc.sendall(s)
except NetWorkIOError as e:
return self.on_GET_Error(e)
while content_length:
data = self.rfile.read(min(self.bufsize, content_length))
if not data:
break
content_length -= len(data)
if self.retryable:
self.rbuffer.append(data)
try:
self.remotesoc.sendall(data)
except NetWorkIOError as e:
return self.on_GET_Error(e)
# read response line
logging.debug('reading response_line')
remoterfile = self.remotesoc if hasattr(self.remotesoc, 'readline') else self.remotesoc.makefile('rb', 0)
try:
s = response_line = remoterfile.readline()
if not s.startswith(b'HTTP'):
raise OSError(0, 'bad response line: %r' % response_line)
except NetWorkIOError as e:
return self.on_GET_Error(e)
protocol_version, _, response_status = response_line.rstrip(b'\r\n').partition(b' ')
response_status, _, response_reason = response_status.partition(b' ')
response_status = int(response_status)
# read response headers
logging.debug('reading response header')
header_data = []
try:
while True:
line = remoterfile.readline()
header_data.append(line)
if line in (b'\r\n', b'\n', b''): # header ends with a empty line
break
except NetWorkIOError as e:
return self.on_GET_Error(e)
header_data = b''.join(header_data)
response_header = email.message_from_string(header_data)
conntype = response_header.get('Connection', "")
if protocol_version >= b"HTTP/1.1":
self.close_connection = conntype.lower() == 'close'
else:
self.close_connection = conntype.lower() != 'keep_alive'
logging.debug('reading response body')
if "Content-Length" in response_header:
if "," in response_header["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', response_header["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
response_header["Content-Length"])
response_header["Content-Length"] = pieces[0]
content_length = int(response_header["Content-Length"])
else:
content_length = None
self.wfile_write(s)
self.wfile_write(header_data)
# read response body
if self.command == 'HEAD' or 100 <= response_status < 200 or response_status in (204, 304):
pass
elif response_header.get("Transfer-Encoding") and response_header.get("Transfer-Encoding") != "identity":
flag = 1
while flag:
try:
trunk_lenth = remoterfile.readline()
except NetWorkIOError as e:
return self.on_GET_Error(e)
self.wfile_write(trunk_lenth)
trunk_lenth = int(trunk_lenth.strip(), 16) + 2
flag = trunk_lenth != 2
while trunk_lenth:
try:
data = self.remotesoc.recv(min(self.bufsize, trunk_lenth))
except NetWorkIOError as e:
return self.on_GET_Error(e)
trunk_lenth -= len(data)
self.wfile_write(data)
elif content_length is not None:
while content_length:
try:
data = self.remotesoc.recv(min(self.bufsize, content_length))
if not data:
raise OSError(0, 'socket read empty')
except NetWorkIOError as e:
return self.on_GET_Error(e)
content_length -= len(data)
self.wfile_write(data)
else:
self.close_connection = 1
self.retryable = False
while 1:
try:
data = self.remotesoc.recv(self.bufsize)
if not data:
raise
self.wfile_write(data)
except Exception:
break
self.wfile_write()
logging.debug('request finish')
PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, True if response_status < 400 else False, self.failed_parents, self.ppname)
if self.close_connection or is_connection_dropped(self.remotesoc):
self.remotesoc.close()
else:
HTTPCONN_POOL.put(self.upstream_name, self.remotesoc, self.ppname if '(pooled)' in self.ppname else self.ppname + '(pooled)')
self.remotesoc = None
def _do_GET(self, retry=False):
try:
if retry:
if self.remotesoc:
try:
self.remotesoc.close()
except:
pass
self.remotesoc = None
self.failed_parents.append(self.ppname)
self.count += 1
if self.count > 10:
self.logger.error('for some strange reason retry time exceeded 10, pls check!')
return
if not self.retryable:
self.close_connection = 1
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return
if self.getparent():
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname)
return self.send_error(504)
self.upstream_name = self.ppname if self.pproxy.proxy.startswith('http') else self.requesthost
iplist = None
if self.pproxy.name == 'direct' and self.requesthost[0] in self.conf.HOSTS and not self.failed_parents:
iplist = self.conf.HOSTS.get(self.requesthost[0])
self._proxylist.insert(0, self.pproxy)
self.set_timeout()
self.phase = 'http_connect_via_proxy'
self.remotesoc = self._http_connect_via_proxy(self.requesthost, iplist)
self.wbuffer = deque()
self.wbuffer_size = 0
# send request header
self.phase = 'sending request header'
s = []
if self.pproxy.proxy.startswith('http'):
path = self.path
if iplist:
path = self.path.split('/')
path[2] = '%s%s' % (iplist[0][1], ((':%d' % self.requesthost[1]) if self.requesthost[1] != 80 else ''))
path = ''.join(path)
s.append('%s %s %s\r\n' % (self.command, self.path, self.request_version))
if self.pproxy.username:
a = '%s:%s' % (self.pproxy.username, self.pproxy.password)
self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(a.encode())
else:
s.append('%s /%s %s\r\n' % (self.command, '/'.join(self.path.split('/')[3:]), self.request_version))
# Does the client want to close connection after this request?
conntype = self.headers.get('Connection', "")
if self.request_version >= b"HTTP/1.1":
client_close = 'close' in conntype.lower()
else:
client_close = 'keep_alive' in conntype.lower()
if 'Upgrade' in self.headers:
if 'websocket' in self.headers['Upgrade']:
self.headers['Upgrade'] = 'websocket'
client_close = True
else:
self.logger.warning('Upgrade header found! (%s), FW-Lite do not support this...' % self.headers['Upgrade'])
del self.headers['Upgrade']
else:
self.headers['Connection'] = 'keep_alive'
del self.headers['Proxy-Connection']
for k, v in self.headers.items():
if isinstance(v, bytes):
v = v.decode('latin1')
s.append("%s: %s\r\n" % ("-".join([w.capitalize() for w in k.split("-")]), v))
s.append("\r\n")
data = ''.join(s).encode('latin1')
self.remotesoc.sendall(data)
self.traffic_count[0] += len(data)
# Now remotesoc is connected, set read timeout
self.remotesoc.settimeout(self.rtimeout)
remoterfile = self.remotesoc.makefile('rb', 0)
# Expect
skip = False
if 'Expect' in self.headers:
try:
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
except Exception as e:
# TODO: probably the server don't handle Expect well.
self.logger.warning('read response line error: %r' % e)
else:
if response_status == 100:
hdata = read_header_data(remoterfile)
self._wfile_write(response_line + hdata)
else:
skip = True
# send request body
if not skip:
self.phase = 'sending request body'
content_length = int(self.headers.get('Content-Length', 0))
if self.headers.get("Transfer-Encoding") and self.headers.get("Transfer-Encoding") != "identity":
if self.rbuffer:
self.remotesoc.sendall(b''.join(self.rbuffer))
flag = 1
req_body_len = 0
while flag:
trunk_lenth = self.rfile_readline()
if self.retryable:
self.rbuffer.append(trunk_lenth)
req_body_len += len(trunk_lenth)
self.remotesoc.sendall(trunk_lenth)
trunk_lenth = int(trunk_lenth.strip(), 16) + 2
flag = trunk_lenth != 2
data = self.rfile_read(trunk_lenth)
if self.retryable:
self.rbuffer.append(data)
req_body_len += len(data)
self.remotesoc.sendall(data)
if req_body_len > 102400:
self.retryable = False
self.rbuffer = deque()
elif content_length > 0:
if content_length > 102400:
self.retryable = False
if self.rbuffer:
s = b''.join(self.rbuffer)
content_length -= len(s)
self.remotesoc.sendall(s)
while content_length:
data = self.rfile_read(min(self.bufsize, content_length))
if not data:
break
content_length -= len(data)
if self.retryable:
self.rbuffer.append(data)
self.remotesoc.sendall(data)
# read response line
timelog = time.clock()
self.phase = 'reading response_line'
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
rtime = time.clock() - timelog
# read response headers
while response_status == 100:
hdata = read_header_data(remoterfile)
self._wfile_write(response_line + hdata)
response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile)
self.phase = 'reading response header'
header_data, response_header = read_headers(remoterfile)
conntype = response_header.get('Connection', "")
if protocol_version >= b"HTTP/1.1":
remote_close = 'close' in conntype.lower()
else:
remote_close = 'keep_alive' in conntype.lower()
if 'Upgrade' in response_header:
remote_close = True
if "Content-Length" in response_header:
if "," in response_header["Content-Length"]:
# Proxies sometimes cause Content-Length headers to get
# duplicated. If all the values are identical then we can
# use them but if they differ it's an error.
pieces = re.split(r',\s*', response_header["Content-Length"])
if any(i != pieces[0] for i in pieces):
raise ValueError("Multiple unequal Content-Lengths: %r" %
response_header["Content-Length"])
response_header["Content-Length"] = pieces[0]
content_length = int(response_header["Content-Length"])
else:
content_length = None
buf = io.BytesIO(header_data)
header_data = b''
for line in buf:
if line.startswith('Connection') and 'Upgrade' not in line:
header_data += b'Connection: close\r\n' if client_close else b'Connection: keep_alive\r\n'
else:
header_data += line
self.wfile_write(response_line)
self.wfile_write(header_data)
# verify
if response_status in (301, 302) and self.conf.PARENT_PROXY.bad302(response_header.get('Location')):
raise IOError(0, 'Bad 302!')
# read response body
self.phase = 'reading response body'
if self.command == 'HEAD' or response_status in (204, 205, 304):
pass
elif response_header.get("Transfer-Encoding") and response_header.get("Transfer-Encoding") != "identity":
flag = 1
while flag:
trunk_lenth = remoterfile.readline()
self.wfile_write(trunk_lenth)
trunk_lenth = int(trunk_lenth.strip(), 16) + 2
flag = trunk_lenth != 2
while trunk_lenth:
data = self.remotesoc.recv(min(self.bufsize, trunk_lenth))
trunk_lenth -= len(data)
self.wfile_write(data)
elif content_length is not None:
while content_length:
data = self.remotesoc.recv(min(self.bufsize, content_length))
if not data:
raise IOError(0, 'remote socket closed')
content_length -= len(data)
self.wfile_write(data)
else:
# websocket?
self.close_connection = 1
self.retryable = False
self.wfile_write()
fd = [self.connection, self.remotesoc]
while fd:
ins, _, _ = select.select(fd, [], [], 60)
if not ins:
break
if self.connection in ins:
data = self.connection_recv(self.bufsize)
if data:
self.remotesoc.sendall(data)
else:
fd.remove(self.connection)
self.remotesoc.shutdown(socket.SHUT_WR)
if self.remotesoc in ins:
data = self.remotesoc.recv(self.bufsize)
if data:
self._wfile_write(data)
else:
fd.remove(self.remotesoc)
self.connection.shutdown(socket.SHUT_WR)
self.wfile_write()
self.phase = 'request finish'
self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, True if response_status < 400 else False, self.failed_parents, self.ppname, rtime)
self.pproxy.log(self.requesthost[0], rtime)
if remote_close or is_connection_dropped([self.remotesoc]):
self.remotesoc.close()
else:
self.HTTPCONN_POOL.put(self.upstream_name, self.remotesoc, self.ppname if '(pooled)' in self.ppname else (self.ppname + '(pooled)'))
self.remotesoc = None
except ClientError as e:
raise
except NetWorkIOError as e:
return self.on_GET_Error(e)
