def _purge(self): while 1: time.sleep(self.intv) pcount = 0 with self.lock: remove_lst = [] for soc in is_connection_dropped(self.socs.keys()): soc.close() remove_lst.append(soc) pcount += 1 for soc in remove_lst: self._remove(soc) remove_lst = [] if pcount: self.logger.debug('%s closed for connection droped.' % pcount) self.timerwheel_index = next(self.timerwheel_iter) for soc in list(self.timerwheel[self.timerwheel_index]): soc.close() remove_lst.append(soc) pcount += 1 for soc in remove_lst: self._remove(soc) if pcount: self.logger.debug('%d remotesoc purged, %d in connection pool.(%s)' % (pcount, len(self.socs), ', '.join([k[0] if isinstance(k, tuple) else k for k, v in self.POOL.items() if v])))
def get(cls, upstream_name): lst = cls.POOL.get(upstream_name) while lst: sock, pproxy = lst.popleft() if not is_connection_dropped(sock): return (sock, pproxy) sock.close()
def get(self, upstream_name): with self.lock: lst = self.POOL.get(upstream_name) while lst: sock, pproxy = lst.popleft() if is_connection_dropped([sock]): sock.close() self._remove(sock) continue self._remove(sock) return (sock, pproxy)
def _purge(self): pcount = 0 with self.lock: for soc in is_connection_dropped(self.socs.keys()): soc.close() self._remove(soc) pcount += 1 self.timerwheel_index = next(self.timerwheel_iter) for soc in list(self.timerwheel[self.timerwheel_index]): soc.close() self._remove(soc) pcount += 1 if pcount: self.logger.debug('%d remotesoc purged, %d in connection pool.(%s)' % (pcount, len(self.socs), ', '.join([k[0] if isinstance(k, tuple) else k for k, v in self.POOL.items() if v]))) Timer(30, self._purge, ()).start()
def _do_GET(self, retry=False): if retry: if self.remotesoc: self.remotesoc.close() self.remotesoc = None self.failed_parents.append(self.ppname) if not self.retryable: self.close_connection = 1 PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname) return if self.getparent(): PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname) return self.send_error(504) self.upstream_name = self.ppname if self.pproxy.startswith('http') else self.requesthost try: self.remotesoc = self._http_connect_via_proxy(self.requesthost) except NetWorkIOError as e: return self.on_GET_Error(e) self.wbuffer = deque() self.wbuffer_size = 0 # send request header logging.debug('sending request header') s = [] if self.pproxy.startswith('http'): s.append('%s %s %s\r\n' % (self.command, self.path, self.request_version)) if self.pproxyparse.username: a = '%s:%s' % (self.pproxyparse.username, self.pproxyparse.password) self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(a.encode()) else: s.append('%s /%s %s\r\n' % (self.command, '/'.join(self.path.split('/')[3:]), self.request_version)) del self.headers['Proxy-Connection'] for k, v in self.headers.items(): if isinstance(v, bytes): v = v.decode('latin1') s.append("%s: %s\r\n" % ("-".join([w.capitalize() for w in k.split("-")]), v)) s.append("\r\n") try: self.remotesoc.sendall(''.join(s).encode('latin1')) except NetWorkIOError as e: return self.on_GET_Error(e) logging.debug('sending request body') # send request body content_length = int(self.headers.get('Content-Length', 0)) if content_length: if content_length > 102400: self.retryable = False if self.rbuffer: s = b''.join(self.rbuffer) content_length -= len(s) try: self.remotesoc.sendall(s) except NetWorkIOError as e: return self.on_GET_Error(e) while content_length: data = self.rfile.read(min(self.bufsize, content_length)) if not data: break content_length -= len(data) if self.retryable: self.rbuffer.append(data) try: self.remotesoc.sendall(data) except NetWorkIOError as e: return self.on_GET_Error(e) # read response line logging.debug('reading response_line') remoterfile = self.remotesoc if hasattr(self.remotesoc, 'readline') else self.remotesoc.makefile('rb', 0) try: s = response_line = remoterfile.readline() if not s.startswith(b'HTTP'): raise OSError(0, 'bad response line: %r' % response_line) except NetWorkIOError as e: return self.on_GET_Error(e) protocol_version, _, response_status = response_line.rstrip(b'\r\n').partition(b' ') response_status, _, response_reason = response_status.partition(b' ') response_status = int(response_status) # read response headers logging.debug('reading response header') header_data = [] try: while True: line = remoterfile.readline() header_data.append(line) if line in (b'\r\n', b'\n', b''): # header ends with a empty line break except NetWorkIOError as e: return self.on_GET_Error(e) header_data = b''.join(header_data) response_header = email.message_from_string(header_data) conntype = response_header.get('Connection', "") if protocol_version >= b"HTTP/1.1": self.close_connection = conntype.lower() == 'close' else: self.close_connection = conntype.lower() != 'keep_alive' logging.debug('reading response body') if "Content-Length" in response_header: if "," in response_header["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', response_header["Content-Length"]) if any(i != pieces[0] for i in pieces): raise ValueError("Multiple unequal Content-Lengths: %r" % response_header["Content-Length"]) response_header["Content-Length"] = pieces[0] content_length = int(response_header["Content-Length"]) else: content_length = None self.wfile_write(s) self.wfile_write(header_data) # read response body if self.command == 'HEAD' or 100 <= response_status < 200 or response_status in (204, 304): pass elif response_header.get("Transfer-Encoding") and response_header.get("Transfer-Encoding") != "identity": flag = 1 while flag: try: trunk_lenth = remoterfile.readline() except NetWorkIOError as e: return self.on_GET_Error(e) self.wfile_write(trunk_lenth) trunk_lenth = int(trunk_lenth.strip(), 16) + 2 flag = trunk_lenth != 2 while trunk_lenth: try: data = self.remotesoc.recv(min(self.bufsize, trunk_lenth)) except NetWorkIOError as e: return self.on_GET_Error(e) trunk_lenth -= len(data) self.wfile_write(data) elif content_length is not None: while content_length: try: data = self.remotesoc.recv(min(self.bufsize, content_length)) if not data: raise OSError(0, 'socket read empty') except NetWorkIOError as e: return self.on_GET_Error(e) content_length -= len(data) self.wfile_write(data) else: self.close_connection = 1 self.retryable = False while 1: try: data = self.remotesoc.recv(self.bufsize) if not data: raise self.wfile_write(data) except Exception: break self.wfile_write() logging.debug('request finish') PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, True if response_status < 400 else False, self.failed_parents, self.ppname) if self.close_connection or is_connection_dropped(self.remotesoc): self.remotesoc.close() else: HTTPCONN_POOL.put(self.upstream_name, self.remotesoc, self.ppname if '(pooled)' in self.ppname else self.ppname + '(pooled)') self.remotesoc = None
def _do_GET(self, retry=False): try: if retry: if self.remotesoc: try: self.remotesoc.close() except: pass self.remotesoc = None self.failed_parents.append(self.ppname) self.count += 1 if self.count > 10: self.logger.error('for some strange reason retry time exceeded 10, pls check!') return if not self.retryable: self.close_connection = 1 self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname) return if self.getparent(): self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, False, self.failed_parents, self.ppname) return self.send_error(504) self.upstream_name = self.ppname if self.pproxy.proxy.startswith('http') else self.requesthost iplist = None if self.pproxy.name == 'direct' and self.requesthost[0] in self.conf.HOSTS and not self.failed_parents: iplist = self.conf.HOSTS.get(self.requesthost[0]) self._proxylist.insert(0, self.pproxy) self.set_timeout() self.phase = 'http_connect_via_proxy' self.remotesoc = self._http_connect_via_proxy(self.requesthost, iplist) self.wbuffer = deque() self.wbuffer_size = 0 # send request header self.phase = 'sending request header' s = [] if self.pproxy.proxy.startswith('http'): path = self.path if iplist: path = self.path.split('/') path[2] = '%s%s' % (iplist[0][1], ((':%d' % self.requesthost[1]) if self.requesthost[1] != 80 else '')) path = ''.join(path) s.append('%s %s %s\r\n' % (self.command, self.path, self.request_version)) if self.pproxy.username: a = '%s:%s' % (self.pproxy.username, self.pproxy.password) self.headers['Proxy-Authorization'] = 'Basic %s' % base64.b64encode(a.encode()) else: s.append('%s /%s %s\r\n' % (self.command, '/'.join(self.path.split('/')[3:]), self.request_version)) # Does the client want to close connection after this request? conntype = self.headers.get('Connection', "") if self.request_version >= b"HTTP/1.1": client_close = 'close' in conntype.lower() else: client_close = 'keep_alive' in conntype.lower() if 'Upgrade' in self.headers: if 'websocket' in self.headers['Upgrade']: self.headers['Upgrade'] = 'websocket' client_close = True else: self.logger.warning('Upgrade header found! (%s), FW-Lite do not support this...' % self.headers['Upgrade']) del self.headers['Upgrade'] else: self.headers['Connection'] = 'keep_alive' del self.headers['Proxy-Connection'] for k, v in self.headers.items(): if isinstance(v, bytes): v = v.decode('latin1') s.append("%s: %s\r\n" % ("-".join([w.capitalize() for w in k.split("-")]), v)) s.append("\r\n") data = ''.join(s).encode('latin1') self.remotesoc.sendall(data) self.traffic_count[0] += len(data) # Now remotesoc is connected, set read timeout self.remotesoc.settimeout(self.rtimeout) remoterfile = self.remotesoc.makefile('rb', 0) # Expect skip = False if 'Expect' in self.headers: try: response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile) except Exception as e: # TODO: probably the server don't handle Expect well. self.logger.warning('read response line error: %r' % e) else: if response_status == 100: hdata = read_header_data(remoterfile) self._wfile_write(response_line + hdata) else: skip = True # send request body if not skip: self.phase = 'sending request body' content_length = int(self.headers.get('Content-Length', 0)) if self.headers.get("Transfer-Encoding") and self.headers.get("Transfer-Encoding") != "identity": if self.rbuffer: self.remotesoc.sendall(b''.join(self.rbuffer)) flag = 1 req_body_len = 0 while flag: trunk_lenth = self.rfile_readline() if self.retryable: self.rbuffer.append(trunk_lenth) req_body_len += len(trunk_lenth) self.remotesoc.sendall(trunk_lenth) trunk_lenth = int(trunk_lenth.strip(), 16) + 2 flag = trunk_lenth != 2 data = self.rfile_read(trunk_lenth) if self.retryable: self.rbuffer.append(data) req_body_len += len(data) self.remotesoc.sendall(data) if req_body_len > 102400: self.retryable = False self.rbuffer = deque() elif content_length > 0: if content_length > 102400: self.retryable = False if self.rbuffer: s = b''.join(self.rbuffer) content_length -= len(s) self.remotesoc.sendall(s) while content_length: data = self.rfile_read(min(self.bufsize, content_length)) if not data: break content_length -= len(data) if self.retryable: self.rbuffer.append(data) self.remotesoc.sendall(data) # read response line timelog = time.clock() self.phase = 'reading response_line' response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile) rtime = time.clock() - timelog # read response headers while response_status == 100: hdata = read_header_data(remoterfile) self._wfile_write(response_line + hdata) response_line, protocol_version, response_status, response_reason = read_reaponse_line(remoterfile) self.phase = 'reading response header' header_data, response_header = read_headers(remoterfile) conntype = response_header.get('Connection', "") if protocol_version >= b"HTTP/1.1": remote_close = 'close' in conntype.lower() else: remote_close = 'keep_alive' in conntype.lower() if 'Upgrade' in response_header: remote_close = True if "Content-Length" in response_header: if "," in response_header["Content-Length"]: # Proxies sometimes cause Content-Length headers to get # duplicated. If all the values are identical then we can # use them but if they differ it's an error. pieces = re.split(r',\s*', response_header["Content-Length"]) if any(i != pieces[0] for i in pieces): raise ValueError("Multiple unequal Content-Lengths: %r" % response_header["Content-Length"]) response_header["Content-Length"] = pieces[0] content_length = int(response_header["Content-Length"]) else: content_length = None buf = io.BytesIO(header_data) header_data = b'' for line in buf: if line.startswith('Connection') and 'Upgrade' not in line: header_data += b'Connection: close\r\n' if client_close else b'Connection: keep_alive\r\n' else: header_data += line self.wfile_write(response_line) self.wfile_write(header_data) # verify if response_status in (301, 302) and self.conf.PARENT_PROXY.bad302(response_header.get('Location')): raise IOError(0, 'Bad 302!') # read response body self.phase = 'reading response body' if self.command == 'HEAD' or response_status in (204, 205, 304): pass elif response_header.get("Transfer-Encoding") and response_header.get("Transfer-Encoding") != "identity": flag = 1 while flag: trunk_lenth = remoterfile.readline() self.wfile_write(trunk_lenth) trunk_lenth = int(trunk_lenth.strip(), 16) + 2 flag = trunk_lenth != 2 while trunk_lenth: data = self.remotesoc.recv(min(self.bufsize, trunk_lenth)) trunk_lenth -= len(data) self.wfile_write(data) elif content_length is not None: while content_length: data = self.remotesoc.recv(min(self.bufsize, content_length)) if not data: raise IOError(0, 'remote socket closed') content_length -= len(data) self.wfile_write(data) else: # websocket? self.close_connection = 1 self.retryable = False self.wfile_write() fd = [self.connection, self.remotesoc] while fd: ins, _, _ = select.select(fd, [], [], 60) if not ins: break if self.connection in ins: data = self.connection_recv(self.bufsize) if data: self.remotesoc.sendall(data) else: fd.remove(self.connection) self.remotesoc.shutdown(socket.SHUT_WR) if self.remotesoc in ins: data = self.remotesoc.recv(self.bufsize) if data: self._wfile_write(data) else: fd.remove(self.remotesoc) self.connection.shutdown(socket.SHUT_WR) self.wfile_write() self.phase = 'request finish' self.conf.PARENT_PROXY.notify(self.command, self.shortpath, self.requesthost, True if response_status < 400 else False, self.failed_parents, self.ppname, rtime) self.pproxy.log(self.requesthost[0], rtime) if remote_close or is_connection_dropped([self.remotesoc]): self.remotesoc.close() else: self.HTTPCONN_POOL.put(self.upstream_name, self.remotesoc, self.ppname if '(pooled)' in self.ppname else (self.ppname + '(pooled)')) self.remotesoc = None except ClientError as e: raise except NetWorkIOError as e: return self.on_GET_Error(e)