async def handler(conn): print(conn) request = await readall_from_socket(conn) # print(request) http = HTTP() parser = HttpRequestParser(http) parser.feed_data(request) method = parser.get_method().decode() url_path = http.url print(method, url_path) # Attempt 1 # with open('requirements.lock', 'rb') as f: # # only python3.5+, and do not support non-bloking socket # conn.sendfile(f) # Attempt 2 fl = None for mp in mapping: fl = mp.file(url_path or '') if fl: break if fl: if fl.exists(): filepath = fl.path if fl.is_file(): with open(filepath, 'rb') as f: blocksize = os.path.getsize(filepath) conn.send(b'HTTP/1.1 200 OK\r\n') conn.send(f'Content-Length: {blocksize}\r\n'.encode('ascii')) mime = mimetypes.guess_type(filepath)[0] # mime = "text/plain" if mime else "application/octet-stream" mime = mime or "application/octet-stream" conn.send( f'Content-Type: {mime}; charset=utf-8\r\n'.encode('ascii') ) # conn.send(b'Transfer-Encoding: chunked') conn.send(b'\r\n') _ = sendfile(conn.fileno(), f.fileno(), 0, blocksize) elif fl.is_dir(): files = fl.listdir() body = '<br/>'.join( f'<a href="{url_path.rstrip("/")}/{x.basename}{"/" if x.is_dir() else ""}">{x.basename}{"/" if x.is_dir() else ""}</a>' for x in files ).encode('utf8') conn.send(b'HTTP/1.1 200 OK\r\n') conn.send(f'Content-Length: {len(body)}\r\n'.encode('ascii')) conn.send(b'Content-Type: text/html; charset=utf-8\r\n') conn.send(b'\r\n') conn.sendall(body) conn.send(b'HTTP/1.1 404 Not Found\r\n') conn.send(b'Content-Type: text/plain; charset=utf-8\r\n') conn.send(b'\r\n') conn.sendall(b'Not Found') conn.close()
async def _parse_request(self, request_reader: asyncio.StreamReader, response_writer: asyncio.StreamWriter) -> Request: """parse data from StreamReader and build the request object """ limit = 2**16 req = Request() parser = HttpRequestParser(req) while True: data = await request_reader.read(limit) parser.feed_data(data) if req.finished or not data: break elif req.needs_write_continue: response_writer.write(b'HTTP/1.1 100 (Continue)\r\n\r\n') req.reset_state() req.method = touni(parser.get_method()).upper() return req
def handle_request(self, data: bytes): request = _PRequest() parser = HttpRequestParser(request) try: parser.feed_data(data) except HttpParserInvalidMethodError: raise ParseError() if request.ready: return ( b"", ( Request( path=request.path, method=parser.get_method(), headers=tuple(request.headers), body=request.body, protocol=b"HTTP/" + parser.get_http_version().encode(), ), ), False, ) raise ParseError()
async def _parse_request(self, request_reader, response_writer): limit = 2 ** 16 req = Request() parser = HttpRequestParser(req) while True: data = await request_reader.read(limit) parser.feed_data(data) if req.finished or not data: break elif req.needs_write_continue: response_writer.write(b'HTTP/1.1 100 (Continue)\r\n\r\n') req.reset_state() if req.path is None: # connected without a formed HTTP request return handler, args = self.get_handler(req.path) req.method = parser.get_method().decode().upper() req.args = args return req, handler
class _HTTPServerProtocol(asyncio.Protocol): """ HTTP Protocol handler. Should only be used by HTTPServerTransport """ __slots__ = ('_parent', '_transport', 'data', 'http_parser', 'request') def __init__(self, *, parent, loop): self._parent = parent self._transport = None self.data = None self.http_parser = HttpRequestParser(self) self.request = None self._loop = loop """ The next 3 methods are for asyncio.Protocol handling """ def connection_made(self, transport): self._transport = transport self._parent._connections.add(self) def connection_lost(self, exc): self._parent._connections.discard(self) def data_received(self, data): try: self.http_parser.feed_data(data) except HttpParserError as e: traceback.print_exc() print(self.request.__dict__) self.send_response(Response(status=400, body={'reason': 'Invalid HTTP', 'details': str(e)})) """ The following methods are for HTTP parsing (from httptools) """ def on_message_begin(self): self.request = Request() self.data = b'' def on_header(self, name, value): key = name.decode('ASCII').lower() val = value.decode() self.request.headers[key] = val if key == 'x-correlation-id': self.request.correlation_id = val if key == 'content-type': self.request.content_type = val def on_headers_complete(self): self.request.method = self.http_parser.get_method().decode('ASCII') def on_body(self, body: bytes): self.data += body def on_message_complete(self): self.request.body = self.data task = self._loop.create_task( self._parent.handle_incoming_request(self.request) ) task.add_done_callback(self.handle_response) def on_url(self, url): url = parse_url(url) if url.query: self.request.query_string = url.query.decode('ASCII') self.request.path = url.path.decode('ASCII') """ End parsing methods """ def handle_response(self, future): try: self.send_response(future.result()) except Exception: traceback.print_exc() self.send_response( Response(status=500, body={'reason': 'Something really bad happened'})) def send_response(self, response): headers = 'HTTP/1.1 {status_code} {status_message}\r\n'.format( status_code=response.status.value, status_message=response.status.phrase, ) if self._parent.shutting_down: headers += 'Connection: close\r\n' else: headers += 'Connection: keep-alive\r\n' if response.data: headers += 'Content-Type: {}\r\n'.format(response.content_type) headers += 'Content-Length: {}\r\n'.format(len(response.data)) if ('transfer-encoding' in response.headers or 'Transfer-Encoding' in response.headers): print('Httptoolstransport currently doesnt support ' 'chunked mode, attempting without.') response.headers.pop('transfer-encoding', None) response.headers.pop('Transfer-Encoding', None) else: headers += 'Content-Length: {}\r\n'.format(0) for header, value in response.headers.items(): headers += '{header}: {value}\r\n'.format(header=header, value=value) result = headers.encode('ASCII') + b'\r\n' if response.data: result += response.data self._transport.write(result) self.request = 0 self.data = 0 def attempt_close(self): if self.request == 0: self._transport.close()
class Connection(object): def __init__(self, sock, addr, app): self._sock = sock # addr = (hostaddr, port) self._addr = addr self._fd = self._sock.fileno() self.parser = None self.url = None self.headers = None self._header_fragment = b"" self.request = None self.app = app self.has_sent_size = 0 self.send_body_size = 0 self._read_buffer = bytearray() self._write_buffer = bytes() # - on_message_begin() # - on_url(url: bytes) # - on_header(name: bytes, value: bytes) # - on_headers_complete() # - on_body(body: bytes) # - on_message_complete() # - on_chunk_header() # - on_chunk_complete() # - on_status(status: bytes) def get_address(self): return self._addr def close(self): KBEngine.deregisterReadFileDescriptor(self._fd) self._sock.close() self._fd = 0 self._sock = None self.cleanup() def cleanup(self): self.request = None self.parser = None self.headers = [] self.app = None self._read_buffer.clear() self._write_buffer = None def data_received(self, fd): # DEBUG_MSG("data_received, fd: %s, id: %s" % (fd, id(self))) if self.parser is None: self.parser = HttpRequestParser(self) self.headers = [] while True: try: data = self._sock.recv(4096) # DEBUG_MSG("data_received, fd: %s, data len: %s" % # (fd, len(data))) # 客户端关闭了链接 if not data: ERROR_MSG("data_received, data len is 0, close") self.close() return self._read_buffer += data except (socket.error, IOError, OSError) as e: _errno = errno_from_exception(e) # 系统调用被signal中断 if _errno == errno.EINTR: continue # 此次的recv数据读完了,recv抛出下面的异常 # 此次recv数据读完了,但不表示该连接的一次发包数据读完了,一次发包可能 # 触发多次epoll 事件 elif _errno in ERRNO_WOULDBLOCK: DEBUG_MSG("data_received, done") break ERROR_MSG("socket recv error: %s" % str(e)) self.close() return except Exception as e: ERROR_MSG("data_received exception, e: %s" % str(e)) return if self._read_buffer: try: self.parser.feed_data(bytes(self._read_buffer)) # 注意要在feed之后清,这次请求可能会被feed 多次,因此每次feed 完要清掉 self._read_buffer.clear() except HttpParserError as e: ERROR_MSG( "Connection::data_received feed_data error. error: %s" " \n %s \n id: %s" % (str(e), traceback.format_exc(), id(self))) # ERROR_MSG( # "Connection::data_received feed_data error. read_buffer: %s" # " \n" % str(bytes(self._read_buffer)) # # ) # TODO 给客户端回错误 def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, key, value): """ :param key: :param value: bytes :return: """ self._header_fragment += key if value is not None: try: value = value.decode() except UnicodeDecodeError: value = value.decode("latin_1") self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b"" def on_headers_complete(self): DEBUG_MSG("Connection::on_header_complete") self.request = Request( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), connection=self, ) def on_body(self, data): DEBUG_MSG("Connection::on_body, data len: %s" % len(data)) # DEBUG_MSG("data: %s" % str(data)) self.request.body.append(data) def on_message_complete(self): INFO_MSG("Connection request: %s" % str(self.request)) self.request.body = b"".join(self.request.body) self.handle_request() def handle_request(self): request_handler_class = self.app.router.get(self.request.path) if not request_handler_class: response = HTTPResponse("NotFound handler", status=500) self.write_response(response) return try: handler = request_handler_class(self.request) # TODO url 参数等数据后面加上 handler.execute_handler() except BaseException_ as e: response = HTTPResponse("An error occurred, error: %s" % str(e), status=e.status_code or 500) self.write_response(response) except Exception as e: ERROR_MSG("Internal Server Error: %s" % str(e)) response = HTTPResponse("Internal Server Error", status=500) self.write_response(response) def write_response(self, response): self._write_buffer = response.output() self.send_body_size = len(self._write_buffer) self._write_to_fd(False, self._fd) if self._check_send_finish(): self.write_completed() else: # 一次没有发完,注册fd,交给epoll DEBUG_MSG("Connection::register write fd: %s" % self._fd) KBEngine.registerWriteFileDescriptor( self._fd, Functor(self._write_to_fd, True)) def _write_to_fd(self, in_poller, fd): try: send_size = self._sock.send( self._write_buffer[self.has_sent_size:]) self.has_sent_size += send_size DEBUG_MSG("_write_to_fd, in_poller: %s, fd: %s" % (in_poller, fd)) if in_poller and self._check_send_finish(): self.write_completed(in_poller) except (socket.error, IOError, OSError) as e: ERROR_MSG("write to fd error: %s" % str(e)) self.close() def _check_send_finish(self): return self.has_sent_size >= self.send_body_size def write_completed(self, in_poller=False): if in_poller: KBEngine.deregisterWriteFileDescriptor(self._fd) DEBUG_MSG("write_completed, in_poller: %s, fd: %s" % (in_poller, self._fd)) self.close()
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', # enable or disable access log / error log purpose 'has_log', 'log', 'netlog', # connection management '_is_upgrade', '_total_request_size', '_timeout_handler', '_last_request_time', 'body_channel', 'message') def __init__(self, *, loop, request_handler: Awaitable, log=None, signal=None, connections=set(), request_timeout=60, request_max_size=None, has_log=True, keep_alive=True, netlog=None): '''signal is shared''' self.loop = loop self.transport = None self.parser = None self.url = None self.headers = None self.body_channel = None self.message = None self.signal = signal self.has_log = has_log self.log = log self.netlog = netlog self.connections = connections self.request_handler = request_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None self._request_stream_task = None self._is_upgrade = False # config.KEEP_ALIVE or not check_headers()['connection_close'] self._keep_alive = keep_alive @property def keep_alive(self): return (self._keep_alive and not self.signal.stopped and self.parser and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later(self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = get_current_time() def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() def connection_timeout(self): # Check if time_elapsed = get_current_time() - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = (self.loop.call_later( time_left, self.connection_timeout)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() exception = (408, 'Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data: bytes): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = (413, 'Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: self.headers = [] self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserUpgrade: upgrade_to_websocket(self) except HttpParserError: exception = (400, 'Bad Request') self.write_error(exception) def on_url(self, url: bytes): self.url = url def on_header(self, name: bytes, value: bytes): # for websocket name = name.lower() if name == b'content-length' and int(value) > self.request_max_size: exception = (413, 'Payload Too Large') self.write_error(exception) if name == b'upgrade': self._is_upgrade = True self.headers.append([name, value]) def on_headers_complete(self): if self._is_upgrade: return channels = {} self.message = self.get_message(self.transport, self.parser.get_http_version(), self.parser.get_method(), self.url, self.headers) channels['body'] = BodyChannel(self.transport) channels['reply'] = ReplyChannel(self) self.body_channel = channels['body'] self._request_handler_task = self.loop.create_task( self.request_handler(self.message, channels)) def on_body(self, body: bytes): if self._is_upgrade: return body_chunk = self.get_request_body_chunk(body, False, True) self._request_stream_task = self.loop.create_task( self.body_channel.send(body_chunk)) def on_message_complete(self): if self._is_upgrade: return body_chunk = self.get_request_body_chunk(b'', False, False) self._request_stream_task = self.loop.create_task( self.body_channel.send(body_chunk)) def get_request_body_chunk(self, content: bytes, closed: bool, more_content: bool) -> Dict[str, Any]: ''' http://channels.readthedocs.io/en/stable/asgi/www.html#request-body-chunk ''' return { 'content': content, 'closed': closed, 'more_content': more_content } def get_message(self, transport, http_version: str, method: bytes, url: bytes, headers: List[List[bytes]]) -> Dict[str, Any]: ''' http://channels.readthedocs.io/en/stable/asgi/www.html#request ''' url_obj = parse_url(url) if url_obj.schema is None: if transport.get_extra_info('sslcontext'): scheme = 'https' else: scheme = 'http' else: scheme = url_obj.schema.decode() path = '' if url_obj.path is None else url_obj.path.decode('utf-8') query = b'' if url_obj.query is None else url_obj.query return { 'channel': 'http.request', 'reply_channel': None, 'http_version': http_version, 'method': method.decode(), 'scheme': scheme, 'path': path, 'query_string': query, 'root_path': '', 'headers': headers, 'body': b'', 'body_channel': None, 'client': transport.get_extra_info('peername'), 'server': transport.get_extra_info('sockname') } def check_headers(self, headers: List[List[bytes]]) -> Dict[str, bool]: connection_close = False content_length = False for key, value in headers: if key == b'Connection' and value == b'close': connection_close = True if key == b'Content-Length': content_length = True return { 'connection_close': connection_close, 'content_length': content_length } def after_write(self, more_content, keep_alive): if not more_content and not keep_alive: self.transport.close() elif not more_content and keep_alive: self._last_request_time = get_current_time() self.cleanup() def is_response_chunk(self, message: Dict[str, Any]) -> bool: return 'status' not in message and 'headers' not in message def make_header_content(self, headers, result_headers, content, more_content): header_content = b'' if headers is not None: _header_content = [] if not more_content and not result_headers['content_length']: _header_content.extend( [b'Content-Length: ', str(len(content)).encode(), b'\r\n']) for key, value in headers: if key == b'Connection': continue _header_content.extend([key, b': ', value, b'\r\n']) header_content = b''.join(_header_content) return header_content def send(self, message: Dict[str, Any]): transport = self.transport status = message.get('status') headers = message.get('headers') content = message.get('content') more_content = message.get('more_content', False) if headers is not None: result_headers = self.check_headers(headers) if result_headers['connection_close'] is True: self._keep_alive = False keep_alive = self.keep_alive if self.is_response_chunk(message): content_length = len(content) if more_content and content_length > 0: transport.write(b'%x\r\n%b\r\n' % (content_length, content)) self.after_write(more_content, keep_alive) elif more_content is False: transport.write(b'0\r\n\r\n') self.after_write(more_content, keep_alive) return keep_alive_timeout = self.request_timeout timeout_header = b'' if keep_alive and keep_alive_timeout is not None: timeout_header = b'Keep-Alive: %d\r\n' % keep_alive_timeout header_content = self.make_header_content(headers, result_headers, content, more_content) response = (b'HTTP/1.1 %d %b\r\n' b'Connection: %b\r\n' b'%b' b'%b\r\n' b'%b') % (status, ALL_STATUS_CODES[status], b'keep-alive' if keep_alive else b'close', timeout_header, header_content, content) transport.write(response) self.after_write(more_content, keep_alive) def write_error(self, exception): try: status, content = exception content = 'Error: {}'.format(content).encode() headers = [] self.send({ 'status': status, 'headers': headers, 'content': content, 'more_content': False }) except RuntimeError: self.log.error('Connection lost before error written.') except Exception as e: self.bail_out('Writing error failed, connection closed {}'.format( repr(e)), from_error=True) finally: self.transport.close() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): self.log.error(('Transport closed @ {} and exception ' 'experienced during error handling').format( self.transport.get_extra_info('peername'))) self.log.debug('Exception:\n{}'.format(traceback.format_exc())) else: exception = (500, message) self.write_error(exception) self.log.error(message) def cleanup(self): self.parser = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self.body_channel = None self.message = None def close_if_idle(self): '''Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open ''' if not self.parser: self.transport.close() return True return False
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, request_max_size=None): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.signal = signal self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later( self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = ( self.loop.call_later(time_left, self.connection_timeout)) else: if self._request_handler_task: self._request_handler_task.cancel() exception = RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: exception = InvalidUsage('Bad Request') self.write_error(exception) def on_url(self, url): self.url = url def on_header(self, name, value): if name == b'Content-Length' and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append((name.decode().casefold(), value.decode())) def on_headers_complete(self): self.request = Request( url_bytes=self.url, headers=CIDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport ) def on_body(self, body): self.request.body.append(body) def on_message_complete(self): if self.request.body: self.request.body = b''.join(self.request.body) self._request_handler_task = self.loop.create_task( self.request_handler( self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): """ Writes response content synchronously to the transport. """ try: keep_alive = ( self.parser.should_keep_alive() and not self.signal.stopped) self.transport.write( response.output( self.request.version, keep_alive, self.request_timeout)) except AttributeError: log.error( ('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error( 'Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ try: keep_alive = ( self.parser.should_keep_alive() and not self.signal.stopped) response.transport = self.transport await response.stream( self.request.version, keep_alive, self.request_timeout) except AttributeError: log.error( ('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error( 'Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() def write_error(self, exception): try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: log.error( 'Connection lost before error written @ {}'.format( self.request.ip if self.request else 'Unknown')) except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(repr(e)), from_error=True) finally: self.transport.close() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): log.error( ("Transport closed @ {} and exception " "experienced during error handling").format( self.transport.get_extra_info('peername'))) log.debug( 'Exception:\n{}'.format(traceback.format_exc())) else: exception = ServerError(message) self.write_error(exception) log.error(message) def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._total_request_size = 0 def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'response_timeout', 'keep_alive_timeout', 'request_max_size', 'request_class', 'is_request_stream', 'router', # enable or disable access log purpose 'access_log', # connection management '_total_request_size', '_request_timeout_handler', '_response_timeout_handler', '_keep_alive_timeout_handler', '_last_request_time', '_last_response_time', '_is_stream_handler') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.access_log = access_log self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.response_timeout = response_timeout self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b'' self.state = state if state else {} if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug @property def keep_alive(self): return (self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._request_timeout_handler = self.loop.call_later( self.request_timeout, self.request_timeout_callback) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) if self._request_timeout_handler: self._request_timeout_handler.cancel() if self._response_timeout_handler: self._response_timeout_handler.cancel() if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = (self.loop.call_later( time_left, self.request_timeout_callback)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise RequestTimeout('Request Timeout') except RequestTimeout as exception: self.write_error(exception) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = current_time - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = (self.loop.call_later( time_left, self.response_timeout_callback)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise ServiceUnavailable('Response Timeout') except ServiceUnavailable as exception: self.write_error(exception) def keep_alive_timeout_callback(self): # Check if elapsed time since last response exceeds our configured # maximum keep alive timeout value time_elapsed = current_time - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = (self.loop.call_later( time_left, self.keep_alive_timeout_callback)) else: logger.debug('KeepAlive Timeout. Closing connection.') self.transport.close() self.transport = None # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if self._header_fragment == b'Content-Length' \ and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) try: value = value.decode() except UnicodeDecodeError: value = value.decode('latin_1') self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b'' def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(body)) return self.request.body.append(body) def on_message_complete(self): # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body = b''.join(self.request.body) self.execute_request_handler() def execute_request_handler(self): self._response_timeout_handler = self.loop.call_later( self.response_timeout, self.response_timeout_callback) self._last_request_time = current_time self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def log_response(self, response): if self.access_log: extra = { 'status': getattr(response, 'status', 0), } if isinstance(response, HTTPResponse): extra['byte'] = len(response.body) else: extra['byte'] = -1 extra['host'] = 'UNKNOWN' if self.request is not None: if self.request.ip: extra['host'] = '{0}:{1}'.format(self.request.ip, self.request.port) extra['request'] = '{0} {1}'.format(self.request.method, self.request.url) else: extra['request'] = 'nil' access_logger.info('', extra=extra) def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error( 'Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.transport = self.transport await response.stream(self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) except AttributeError: logger.error( 'Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = current_time self.cleanup() def write_error(self, exception): # An error _is_ a response. # Don't throw a response timeout, when a response _is_ given. if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: if self._debug: logger.error('Connection lost before error written @ %s', self.request.ip if self.request else 'Unknown') except Exception as e: self.bail_out("Writing error failed, connection closed {}".format( repr(e)), from_error=True) finally: if self.parser and (self.keep_alive or getattr(response, 'status', 0) == 408): self.log_response(response) try: self.transport.close() except AttributeError as e: logger.debug('Connection lost before server could close it.') def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): logger.error( "Transport closed @ %s and exception " "experienced during error handling", self.transport.get_extra_info('peername')) logger.debug('Exception:\n%s', traceback.format_exc()) else: exception = ServerError(message) self.write_error(exception) logger.error(message) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class HttpProtocol(SanicHttpProtocol): __slots__ = tuple() def __init__(self, *, loop, request_handler, error_handler, signal=None, connections=set(), request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): signal = signal or Signal() request_class = request_class or Request super(HttpProtocol, self).\ __init__(loop=loop, request_handler=request_handler, error_handler=error_handler, signal=signal, connections=connections, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, state=state, debug=debug, **kwargs) def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = server.current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = (self.loop.call_later( time_left, self.request_timeout_callback)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise RequestTimeout('Request Timeout') except RequestTimeout as exception: self.write_error(exception) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = server.current_time - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = (self.loop.call_later( time_left, self.response_timeout_callback)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise ServiceUnavailable('Response Timeout') except ServiceUnavailable as exception: self.write_error(exception) def keep_alive_timeout_callback(self): # Check if elapsed time since last response exceeds our configured # maximum keep alive timeout value time_elapsed = server.current_time - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = (self.loop.call_later( time_left, self.keep_alive_timeout_callback)) else: logger.debug('KeepAlive Timeout. Closing connection.') self.transport.close() self.transport = None def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def on_header(self, name, value): self._header_fragment += name if value is not None: if self._header_fragment == b'Content-Length' \ and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) try: value = value.decode() except UnicodeDecodeError: value = value.decode('latin_1') self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b'' def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error( 'Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = server.current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.transport = self.transport await response.stream(self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) except AttributeError: logger.error( 'Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = server.current_time self.cleanup() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): logger.error( "Transport closed @ %s and exception " "experienced during error handling", self.transport.get_extra_info('peername')) logger.debug('Exception:\n%s', traceback.format_exc()) else: exception = ServerError(message) self.write_error(exception) logger.error(message)
def data_received(self, data): hrp = HttpRequestParser(self) hrp.feed_data(data) self.http_version = hrp.get_http_version() self.method = hrp.get_method() before_requests = asyncio. async (self.call_before_requests())
class HttpProtocol(asyncio.Protocol): __slots__ = ("_route_", "_loop", "_transport", "_parser", "_request") def __init__(self, event_loop=None, route=None, objt_request=None): self._route = route self._loop = event_loop self._transport = None self._parser = HttpRequestParser(self) self._request = objt_request() def connection_made(self, transport): self._transport = transport def data_received(self, data): try: self._parser.feed_data(data) except HttpParserError: Logger.info("bad request") def connection_lost(self, exc): self._transport.close() def on_url(self, uri): self._request.uri = uri.decode() def on_header(self, name, value): self._request.headers[name] = value def on_headers_complete(self): self._request.version = self._parser.get_http_version() self._request.method = self._parser.get_method().decode() def on_body(self, body): self._request.body.append(body) def on_message_complete(self): if self._request.body: self._request.body = b"".join(self._request.body) self._loop.create_task( self.start_response(request=self._request, transport=self._transport)) async def start_response(self, transport, request): view_func = self._route._router.get(request.url, None) if view_func is None: transport.write( b'HTTP/1.1 404 Not Found\r\nServer: aquarius\r\nContent-Length:9\r\n\r\nNot Found\r\n\r\n' ) else: view_obj = view_func.__dict__ if request.method not in view_obj.get("allowed_method"): transport.close() if view_obj.get("async"): content = await view_func(request) else: content = view_func(request) try: transport.write(content) except Exception as e: transport.close() if request.version == "1.0": transport.close()
class HttpProtocol(asyncio.Protocol): """ This class provides a basic HTTP implementation of the sanic framework. """ __slots__ = ( # app "app", # event loop, connection "loop", "transport", "connections", "signal", # request params "parser", "request", "url", "headers", # request config "request_handler", "request_timeout", "response_timeout", "keep_alive_timeout", "request_max_size", "request_buffer_queue_size", "request_class", "is_request_stream", "router", "error_handler", # enable or disable access log purpose "access_log", # connection management "_total_request_size", "_request_timeout_handler", "_response_timeout_handler", "_keep_alive_timeout_handler", "_last_request_time", "_last_response_time", "_is_stream_handler", "_not_paused", "_request_handler_task", "_request_stream_task", "_keep_alive", "_header_fragment", "state", "_debug", ) def __init__( self, *, loop, app, request_handler, error_handler, signal=Signal(), connections=None, request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_buffer_queue_size=100, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs ): self.loop = loop self.app = app self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.access_log = access_log self.connections = connections if connections is not None else set() self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_buffer_queue_size = request_buffer_queue_size self.response_timeout = response_timeout self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._not_paused = asyncio.Event(loop=loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b"" self.state = state if state else {} if "requests_count" not in self.state: self.state["requests_count"] = 0 self._debug = debug self._not_paused.set() @property def keep_alive(self): """ Check if the connection needs to be kept alive based on the params attached to the `_keep_alive` attribute, :attr:`Signal.stopped` and :func:`HttpProtocol.parser.should_keep_alive` :return: ``True`` if connection is to be kept alive ``False`` else """ return ( self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive() ) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._request_timeout_handler = self.loop.call_later( self.request_timeout, self.request_timeout_callback ) self.transport = transport self._last_request_time = time() def connection_lost(self, exc): self.connections.discard(self) if self._request_handler_task: self._request_handler_task.cancel() if self._request_stream_task: self._request_stream_task.cancel() if self._request_timeout_handler: self._request_timeout_handler.cancel() if self._response_timeout_handler: self._response_timeout_handler.cancel() if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() def pause_writing(self): self._not_paused.clear() def resume_writing(self): self._not_paused.set() def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = self.loop.call_later( time_left, self.request_timeout_callback ) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() self.write_error(RequestTimeout("Request Timeout")) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = self.loop.call_later( time_left, self.response_timeout_callback ) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() self.write_error(ServiceUnavailable("Response Timeout")) def keep_alive_timeout_callback(self): """ Check if elapsed time since last response exceeds our configured maximum keep alive timeout value and if so, close the transport pipe and let the response writer handle the error. :return: None """ time_elapsed = time() - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = self.loop.call_later( time_left, self.keep_alive_timeout_callback ) else: logger.debug("KeepAlive Timeout. Closing connection.") self.transport.close() self.transport = None # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: self.write_error(PayloadTooLarge("Payload Too Large")) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state["requests_count"] = self.state["requests_count"] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = "Bad Request" if self._debug: message += "\n" + traceback.format_exc() self.write_error(InvalidUsage(message)) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if ( self._header_fragment == b"Content-Length" and int(value) > self.request_max_size ): self.write_error(PayloadTooLarge("Payload Too Large")) try: value = value.decode() except UnicodeDecodeError: value = value.decode("latin_1") self.headers.append( (self._header_fragment.decode().casefold(), value) ) self._header_fragment = b"" def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport, app=self.app, ) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request ) if self._is_stream_handler: self.request.stream = StreamBuffer( self.request_buffer_queue_size ) self.execute_request_handler() def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.body_append(body) ) else: self.request.body_push(body) async def body_append(self, body): if self.request.stream.is_full(): self.transport.pause_reading() await self.request.stream.put(body) self.transport.resume_reading() else: await self.request.stream.put(body) def on_message_complete(self): # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None) ) return self.request.body_finish() self.execute_request_handler() def execute_request_handler(self): """ Invoke the request handler defined by the :func:`sanic.app.Sanic.handle_request` method :return: None """ self._response_timeout_handler = self.loop.call_later( self.response_timeout, self.response_timeout_callback ) self._last_request_time = time() self._request_handler_task = self.loop.create_task( self.request_handler( self.request, self.write_response, self.stream_response ) ) # -------------------------------------------- # # Responding # -------------------------------------------- # def log_response(self, response): """ Helper method provided to enable the logging of responses in case if the :attr:`HttpProtocol.access_log` is enabled. :param response: Response generated for the current request :type response: :class:`sanic.response.HTTPResponse` or :class:`sanic.response.StreamingHTTPResponse` :return: None """ if self.access_log: extra = {"status": getattr(response, "status", 0)} if isinstance(response, HTTPResponse): extra["byte"] = len(response.body) else: extra["byte"] = -1 extra["host"] = "UNKNOWN" if self.request is not None: if self.request.ip: extra["host"] = "{0}:{1}".format( self.request.ip, self.request.port ) extra["request"] = "{0} {1}".format( self.request.method, self.request.url ) else: extra["request"] = "nil" access_logger.info("", extra=extra) def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output( self.request.version, keep_alive, self.keep_alive_timeout ) ) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self._debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(repr(e)) ) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback ) self._last_response_time = time() self.cleanup() async def drain(self): await self._not_paused.wait() def push_data(self, data): self.transport.write(data) async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.protocol = self await response.stream( self.request.version, keep_alive, self.keep_alive_timeout ) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self._debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(repr(e)) ) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback ) self._last_response_time = time() self.cleanup() def write_error(self, exception): # An error _is_ a response. # Don't throw a response timeout, when a response _is_ given. if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else "1.1" self.transport.write(response.output(version)) except RuntimeError: if self._debug: logger.error( "Connection lost before error written @ %s", self.request.ip if self.request else "Unknown", ) except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(repr(e)), from_error=True, ) finally: if self.parser and ( self.keep_alive or getattr(response, "status", 0) == 408 ): self.log_response(response) try: self.transport.close() except AttributeError: logger.debug("Connection lost before server could close it.") def bail_out(self, message, from_error=False): """ In case if the transport pipes are closed and the sanic app encounters an error while writing data to the transport pipe, we log the error with proper details. :param message: Error message to display :param from_error: If the bail out was invoked while handling an exception scenario. :type message: str :type from_error: bool :return: None """ if from_error or self.transport is None or self.transport.is_closing(): logger.error( "Transport closed @ %s and exception " "experienced during error handling", ( self.transport.get_extra_info("peername") if self.transport is not None else "N/A" ), ) logger.debug("Exception:", exc_info=True) else: self.write_error(ServerError(message)) logger.error(message) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class _HTTPServerProtocol(asyncio.Protocol): """ HTTP Protocol handler. Should only be used by HTTPServerTransport """ __slots__ = ('_parent', '_transport', '_task', 'data', 'http_parser', 'request') def __init__(self, *, parent, loop): self._parent = parent self._transport = None self.data = None self.http_parser = HttpRequestParser(self) self.request = None self._loop = loop self._task: asyncio.Task = None """ The next 3 methods are for asyncio.Protocol handling """ def connection_made(self, transport): self._transport = transport self._parent._connections.add(self) def connection_lost(self, exc): self._parent._connections.discard(self) if self._task: self._task.cancel() self._transport = None def data_received(self, data): try: self.http_parser.feed_data(data) except HttpParserError as e: traceback.print_exc() logger.error('Bad http: %s', self.request) if self._transport: self.send_response( Response(status=400, body={ 'reason': 'Invalid HTTP', 'details': str(e) })) """ The following methods are for HTTP parsing (from httptools) """ def on_message_begin(self): self.request = Request() self.data = b'' def on_header(self, name, value): key = name.decode('latin-1').lower() if not value: value = b'' val = value.decode() self.request.headers[key] = val if key == 'x-correlation-id': self.request.correlation_id = val if key == 'content-type': self.request.content_type = val def on_headers_complete(self): self.request.method = self.http_parser.get_method().decode('latin-1') def on_body(self, body: bytes): self.data += body def on_message_complete(self): self.request.body = self.data task = self._loop.create_task( self._parent.handle_incoming_request(self.request)) task.add_done_callback(self.handle_response) self._task = task def on_url(self, url): url = url.replace(b'//', b'/') url = parse_url(url) if url.query: # query = urllib.parse.unquote(url.query.decode('latin-1')) self.request.query_string = url.query.decode('latin-1') path = urllib.parse.unquote(url.path.decode('latin-1')) if path.startswith(self._parent.prefix): path = path[len(self._parent.prefix):] self.request.path = path """ End parsing methods """ def handle_response(self, future): try: self.send_response(future.result()) except Exception: traceback.print_exc() self.send_response( Response(status=500, body={'reason': 'Something really bad happened'}, content_type=self.request.app.default_content_type)) def send_response(self, response): if response is None: # connection closed, no response return headers = 'HTTP/1.1 {status_code} {status_message}\r\n'.format( status_code=response.status.value, status_message=response.status.phrase, ) headers += 'Connection: close\r\n' # if self._parent.shutting_down: # headers += 'Connection: close\r\n' # else: # headers += 'Connection: keep-alive\r\n' # headers += 'Keep-Alive: timeout=5, max=50\r\n' if response.raw_body: headers += 'Content-Type: {}\r\n'.format(response.content_type) headers += 'Content-Length: {}\r\n'.format(len(response.raw_body)) if ('transfer-encoding' in response.headers or 'Transfer-Encoding' in response.headers): print('Httptoolstransport currently doesnt support ' 'chunked mode, attempting without.') response.headers.pop('transfer-encoding', None) response.headers.pop('Transfer-Encoding', None) else: headers += 'Content-Length: {}\r\n'.format(0) for header, value in response.headers.items(): if header in ('Content-Length', 'content-lenth'): continue headers += '{header}: {value}\r\n'.format(header=header, value=value) result = headers.encode('latin-1') + b'\r\n' if response.raw_body: result += response.raw_body try: self._transport.write(result) except AttributeError: # "NoneType has no attribute 'write'" because transport is closed logger.debug( 'Connection closed prematurely, most likely by client') self.request = 0 self.data = 0 self.attempt_close() def attempt_close(self): if self.request == 0 and self._transport: self._transport.close()
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, request_max_size=None): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.signal = signal self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later( self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = ( self.loop.call_later(time_left, self.connection_timeout)) else: if self._request_handler_task: self._request_handler_task.cancel() exception = RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: exception = InvalidUsage('Bad Request') self.write_error(exception) def on_url(self, url): self.url = url def on_header(self, name, value): if name == b'Content-Length' and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append((name.decode().casefold(), value.decode())) def on_headers_complete(self): self.request = Request( url_bytes=self.url, headers=CIDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport ) def on_body(self, body): self.request.body.append(body) def on_message_complete(self): if self.request.body: self.request.body = b''.join(self.request.body) self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): try: keep_alive = ( self.parser.should_keep_alive() and not self.signal.stopped) self.transport.write( response.output( self.request.version, keep_alive, self.request_timeout)) except RuntimeError: log.error( 'Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(e)) finally: if not keep_alive: self.transport.close() else: # Record that we received data self._last_request_time = current_time self.cleanup() def write_error(self, exception): try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: log.error( 'Connection lost before error written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(e), from_error=True) finally: self.transport.close() def bail_out(self, message, from_error=False): if from_error and self.transport.is_closing(): log.error( ("Transport closed @ {} and exception " "experienced during error handling").format( self.transport.get_extra_info('peername'))) log.debug( 'Exception:\n{}'.format(traceback.format_exc())) else: exception = ServerError(message) self.write_error(exception) log.error(message) def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._total_request_size = 0 def close_if_idle(self): """ Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class LuyProtocol(asyncio.Protocol): def __init__(self, app, loop=None, keep_alive=True): self.parser = None self.url = None self._request_handler_task = None self.loop = loop self.header = {} self.app = app self.keep_alive = keep_alive def connection_made(self, transport): self.transport = transport if self.parser is None: self.parser = HttpRequestParser(self) def connection_lost(self, exc): self.transport.close() self.refresh() self.transport = None #------------------------------------- # parsing #------------------------------------- def data_received(self, data): try: self.parser.feed_data(data) except HttpParserError as e: print('出错', e) finally: pass def on_message_begin(self): # print('on_message_begin') pass def on_url(self, url): # storing the url from web if not self.url: self.url = url else: self.url += url def on_header(self, name, value): ''' Content-Length cannot be too long, protect the server ''' if value is not None: if name == b'Content-Length' and int(value) > 1500: self.write_error() self.header[name.decode().casefold()] = value.decode() def on_headers_complete(self): self.request = request_class(url=self.url.decode(), header=self.header, version=self.parser.get_http_version(), method=self.parser.get_method().decode()) def on_body(self, body): self.request.body.append(body) def on_message_complete(self): # print('on_message_complete') self._request_handler_task = self.loop.create_task( self.app.request_handler(self.request, self.write_response, None)) #--------------------------- # error handling #--------------------------- def write_error(self): response = html('bad connecton', status=400) self.write_response(response) self.transport.close() #------------------------------------- # write response #------------------------------------- def write_response(self, response): ''' the writing phase is very fast so may not have to use coroutine ''' try: keep_alive = self.keep_alive self.transport.write(response.drain(keep_alive=keep_alive)) if keep_alive: self.refresh() else: self.transport.close() except AttributeError as e: print('AttributeError????', e) self.transport.close() except RuntimeError as e: print('RuntimeError????', e) self.transport.close() except Exception as e: print('Exception????', e) self.transport.close() def refresh(self): ''' refresh the server state prepare for next incoming request ''' self.url = None self.header = {}
class HttpProtocol(asyncio.Protocol): """ HTTP 协议 """ # 插槽 __slots__ = ( # 事件循环, 连接 'loop', 'transport', 'connections', 'signal', # 请求参数 'parser', 'request', 'url', 'headers', # 请求配置 'request_handler', 'request_timeout', 'request_max_size', # 连接管理 '_total_request_size', '_timeout_handler', '_last_communication_time') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections={}, request_timeout=60, request_max_size=None): self.loop = loop # 事件循环 self.transport = None self.request = None # 请求 self.parser = None self.url = None # 预留的路径 self.headers = None # 请求头 self.signal = signal # 标志是否结束 self.connections = connections # 连接集合 self.request_handler = request_handler # 请求处理器 self.error_handler = error_handler # 出错处理器 self.request_timeout = request_timeout # 请求超时时间 self.request_max_size = request_max_size # 请求最大大小 self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None # -------------------------------------------- # # 连接部分 # -------------------------------------------- # def connection_made(self, transport): """ 创建连接 """ self.connections.add(self) self._timeout_handler = self.loop.call_later( self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): """ 丢失连接 """ self.connections.discard(self) self._timeout_handler.cancel() self.cleanup() def connection_timeout(self): """ 连接超时 """ time_elapsed = current_time - self._last_request_time # 计算与上次请求间隔 if time_elapsed < self.request_timeout: # 未超时 time_left = self.request_timeout - time_elapsed self._timeout_handler = \ self.loop.call_later(time_left, self.connection_timeout) else: # 超时 if self._request_handler_task: self._request_handler_task.cancel() exception = RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # 解析部分 # -------------------------------------------- # def data_received(self, data): """ 接受数据 """ self._total_request_size += len(data) if self._total_request_size > self.request_max_size: # 请求数据过大 # 在`exceptions.py`中添加 PayloadTooLarge 错误 exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # 如果是第一次接受数据,创建 parser if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # 解析请求 try: self.parser.feed_data(data) except HttpParserError: exception = InvalidUsage('Bad Request') self.write_error(exception) def on_url(self, url): """ 获得 url """ self.url = url def on_header(self, name, value): """ 补全 HTTP 请求的 head 信息 """ if name == b'Content-Length' and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append((name.decode(), value.decode('utf-8'))) def on_headers_complete(self): """ 写入 HTTP 请求 head 信息 """ # 远程地址 remote_addr = self.transport.get_extra_info('peername') if remote_addr: self.headers.append(('Remote-Addr', '%s:%s' % remote_addr)) # HTTP 请求 head self.request = Request( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode() ) def on_body(self, body): """ 写入 HTTP 请求 body """ if self.request.body: self.request.body += body else: self.request.body = body def on_message_complete(self): """ 创建 task """ self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response)) # -------------------------------------------- # # 响应部分 # -------------------------------------------- # def write_response(self, response): """ 编写 HTTP 响应 """ try: keep_alive = self.parser.should_keep_alive() \ and not self.signal.stopped # 输出响应 self.transport.write( response.output( self.request.version, keep_alive, self.request_timeout)) if not keep_alive: self.transport.close() else: # 记录接收到的数据 self._last_request_time = current_time self.cleanup() except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(e)) def write_error(self, exception): """ 编写 HTTP 错误响应 """ try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) self.transport.close() except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(e)) def bail_out(self, message): """ 记录异常辅助方法 """ exception = ServerError(message) self.write_error(exception) log.error(message) def cleanup(self): """ 清空请求字段 """ self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._total_request_size = 0 def close_if_idle(self): """ 若没有发生或接受请求,则关闭连接 :return: boolean - True 为关, false 为保持开启 """ if not self.parser: self.transport.close() return True return False
class BaseServer(asyncio.Protocol): def __init__(self, loop, requesthandle, toggle, connections=set(), request_timeout=10): self.loop = loop self.requesthandle = requesthandle self.toggle = toggle self.connections = connections self.request_timeout = request_timeout self.timehandle = None self.requesthandletask = None # 默认请求参数 self.ip = None self.parse = None self.url = None self.headers = {} self.body = None self.httpversion = None self.method = None self.request = None self.contentlength = 0 self.keep_alive = False # print("enter init") ####################### # 连接建立 def connection_made(self, transport): # 将本连接加入全局的连接集合 print("connect start") self.connections.add(self) self.transport = transport self.ip = transport.get_extra_info("peername") # self.timehandle = self.loop.call_later(self.request_timeout, self.teardown) def connection_lost(self, exc): print("connect end") self.connections.remove(self) # self.timehandle.cancel() self.clean() ######################## # 解析数据 def data_received(self, data): pprint(data.decode("utf-8")) self.contentlength += len(data) if not self.parse: # HttpRequestParser响应当前对象的以下方法 # - on_url(url:byte) # - on_header(name: bytes, value: bytes) # - on_headers_complete() # - on_body(body: bytes) # - on_message_complete() # get_http_version(self) -> str # def should_keep_alive(self) -> bool: self.parse = HttpRequestParser(self) try: self.parse.feed_data(data) except HttpRequestParser as e: pass def on_url(self, url): self.url = url def on_header(self, name, value): self.headers[name] = value def on_headers_complete(self): self.method = self.parse.get_method() self.httpversion = self.parse.get_http_version() self.keep_alive = self.parse.should_keep_alive() self.request = Request(self.ip, self.url, self.headers, self.method, self.httpversion, self.request_timeout if self.keep_alive else None) def on_body(self, body): # print(body) self.body = body self.request.setbody(body) def on_message_complete(self): print("parser complete") self.requesthandletask = self.loop.create_task(self.requesthandle(self.request, self.write)) ######################### # 返回响应 def write(self, response): print("start write") self.transport.write(response.make_response()) print("end write") keep_alive = self.keep_alive and not self.toggle[0] if not keep_alive: self.transport.close() else: self.clean() ######################## # 清理或者关闭连接 def clean(self): self.requsettask = None self.parse = None self.url = None self.headers = {} self.body = None self.httpversion = None self.methods = None self.request = None self.contentlength = 0 self.keep_alive = False def teardown(self): if not self.parse: self.transport.close() return True return False
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections={}, request_timeout=60, request_max_size=None): self.loop = loop self.transport = None self.request = None # 请求 self.parser = None self.url = None self.headers = None # 请求头 self.signal = signal self.connections = connections self.request_handler = request_handler # 请求处理器 self.error_handler = error_handler # 出错处理器 self.request_timeout = request_timeout self.request_max_size = request_max_size self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later(self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() self.cleanup() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = \ self.loop.call_later(time_left, self.connection_timeout) else: if self._request_handler_task: self._request_handler_task.cancel() exception = RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: exception = InvalidUsage('Bad Request') self.write_error(exception) def on_url(self, url): self.url = url # # HTTP 请求: 补全 head 信息 # - 更新 headers 字段 # def on_header(self, name, value): if name == b'Content-Length' and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append((name.decode(), value.decode('utf-8'))) # # HTTP 请求: 写入 head 信息 # def on_headers_complete(self): remote_addr = self.transport.get_extra_info('peername') if remote_addr: self.headers.append(('Remote-Addr', '%s:%s' % remote_addr)) # # 构建 HTTP 请求 # self.request = Request(url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode()) # # HTTP 请求: 写入 body 部分 # def on_body(self, body): if self.request.body: self.request.body += body else: self.request.body = body def on_message_complete(self): # # 任务创建: # self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response)) # -------------------------------------------- # # Responding # - HTTP 响应部分 # -------------------------------------------- # # # HTTP 响应: 正常响应 # - 写出 HTTP 响应 # - 长连接, 更新连接时间 # def write_response(self, response): try: keep_alive = self.parser.should_keep_alive( ) and not self.signal.stopped # # 输出 HTTP 响应 # self.transport.write( response.output( # HTTP Response, 写一个响应 self.request.version, keep_alive, self.request_timeout)) if not keep_alive: # 非长连接, 关闭 self.transport.close() else: # Record that we received data self._last_request_time = current_time self.cleanup() except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(e)) # # HTTP 响应: 出错响应 # def write_error(self, exception): try: response = self.error_handler.response(self.request, exception) # 出错响应处理 version = self.request.version if self.request else '1.1' # HTTP 协议版本 self.transport.write( response.output(version)) # HTTP Response, 写一个响应 self.transport.close() except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(e)) # # 异常记录: # def bail_out(self, message): exception = ServerError(message) self.write_error(exception) log.error(message) # # 清理: # - 将字段复位为空 # def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._total_request_size = 0 def close_if_idle(self): """ Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class SimpleHttp(Protocol): ''' simple-http protocol is a light http protocol, you can set proxy or gateway for it ''' __slots__ = ("loop", "conns", "router", "transport", "request", "response", "parser", "request_limit_size", "request_cur_size", "request_timeout", "response_timeout", "keep_alive", "last_request_time", "remote_addr", "request_timeout_task", "response_timeout_task", "conn_timeout_task") def __init__( self, *, loop, conns, # server.conns router, # path handler mgr request_limit_size=1024 * 1024 * 1, # 1M request_timeout=60, response_timeout=60, keep_alive=10): self.loop = loop self.conns = conns self.router = router self.transport = None self.request = Request(self) self.response = Response(self) self.response.set_keep_alive() self.parser = HttpRequestParser(self) self.request_limit_size = request_limit_size self.request_cur_size = 0 self.request_timeout = request_timeout self.response_timeout = response_timeout self.keep_alive = keep_alive self.last_request_time = 0 self.remote_addr = None self.request_timeout_task = None self.response_timeout_task = None self.conn_timeout_task = None self.route_mgr = None def connection_made(self, transport): self.transport = transport self.remote_addr = transport.get_extra_info("peername") self.conns.add(self) self.last_request_time = time() self.request_timeout_task = self.loop.call_later( self.request_timeout, self.request_timeout_handler) def data_received(self, data): self.request_cur_size += len(data) if self.request_cur_size > self.request_limit_size: self.response.error(UnlightException(413)) try: self.parser.feed_data(data) except HttpParserError: self.response.error(UnlightException(401)) traceback.print_exc() def connection_lost(self, err): self.conns.discard(self) self._cancel_request_timeout_task() self._cancel_response_timeout_task() self._cancel_conn_timeout_task() @property def is_keep_alive(self): return self.keep_alive and self.parser.should_keep_alive def write(self, enc_data): ''' write and try keep alive ''' try: self.transport.write(enc_data) except RuntimeError: unlight_logger.error( "Connection lost before response written @ %s", self.remote_addr if self.remote_addr else "Unknown") finally: if self.is_keep_alive: self._cancel_conn_timeout_task() self.conn_timeout_task = self.loop.call_later( self.keep_alive, self.keep_alive_timeout_handler) self.reset() else: self.transport.close() self.transport = None def fatal(self, enc_err): ''' wirte and close ''' try: self.transport.write(enc_err) except RuntimeError: unlight_logger.error( "Connection lost before error written @ %s", self.remote_addr if self.remote_addr else "Unknown") finally: try: self.transport.close() self.transport = None except AttributeError: unlight_logger.error( "Connection lost before server could close it.") def reset(self): self.request_cur_size = 0 self.request.reset() self.response.reset() self._cancel_response_timeout_task() def request_timeout_handler(self): self._cancel_request_timeout_task() self.response.error(UnlightException(408)) def response_timeout_handler(self): self._cancel_response_timeout_task() self.response.error(UnlightException(502)) def keep_alive_timeout_handler(self): self._cancel_request_timeout_task() self.transport.close() self.transport = None def _cancel_request_timeout_task(self): if self.request_timeout_task: self.request_timeout_task.cancel() self.request_timeout_task = None def _cancel_response_timeout_task(self): if self.response_timeout_task: self.response_timeout_task.cancel() self.response_timeout_task = None def _cancel_conn_timeout_task(self): if self.conn_timeout_task: self.conn_timeout_task.cancel() self.conn_timeout_task = None def on_url(self, burl): self.request.add_burl(burl) def on_header(self, bname, bvalue): self.request.add_bheader(bname, bvalue) if bname.lower( ) == b"content-length" and int(bvalue) > self.request_limit_size: self.response.error(UnlightException(413)) if bname.lower() == b"expect" and bvalue.lower() == b"100-continue": self.response.error(UnlightException(100)) def on_header_complete(self): self.response_timeout_task = self.loop.call_later( self.response_timeout, self.response_timeout_handler) self._cancel_request_timeout_task() def on_body(self, bbody): if self.request.add_bbody(bbody) == 2: self.response.error(UnlightException(400)) # parse err def on_message_complete(self): self.request.set_method(self.parser.get_method().decode()) self.loop.create_task( self.router.handle_request(self.request, self.response))
class HttpProtocol(asyncio.Protocol): __slots__ = ("_route", "_loop", "_transport", "_parser", "_request") def __init__(self, event_loop=None, route=None, re_route=None): self._route = route self._re_route = re_route self._loop = event_loop self._transport = None self._parser = HttpRequestParser(self) self._request = Request() def connection_made(self, transport): self._transport = transport def data_received(self, data): try: self._parser.feed_data(data) except HttpParserError: pass def connection_lost(self, exc): self._transport.close() def on_url(self, uri): self._request.uri = uri.decode() def on_header(self, name, value): self._request.headers[name] = value def on_headers_complete(self): self._request.version = self._parser.get_http_version() self._request.method = self._parser.get_method().decode() def on_body(self, body): self._request.body.append(body) def on_message_complete(self): if self._request.body: self._request.body = b"".join(self._request.body) self._loop.create_task( self.start_response(request=self._request, transport=self._transport)) async def start_response(self, transport, request): try: _view = self._route.get(request.url, self._re_route) if isinstance(_view, list): for _re_route_tuple in _view: regex, nums, view = _re_route_tuple _re_uri = re.match(regex, request.url) if _re_uri: args = [_re_uri.group(i + 1) for i in range(nums)] content = await view(request, *args) break else: print(request.url) raise RouterError("404 %s" % request.url) else: content = await _view(request) transport.write(content) except RouterError: transport.write( b'HTTP/1.1 404 Not Found\r\nServer: aquarius\r\nContent-Length:9\r\n\r\nNot Found\r\n\r\n' ) except ValueError: transport.write( b'HTTP/1.1 404 Not Found\r\nServer: aquarius\r\nContent-Length:9\r\n\r\nNot Found\r\n\r\n' ) except AttributeError: transport.write( b'HTTP/1.1 404 Not Found\r\nServer: aquarius\r\nContent-Length:9\r\n\r\nNot Found\r\n\r\n' ) if request.version == "1.0": transport.close()
class HTTP1Protocol(AbstractProtocol): __slots__ = ("parser", "headers", "request", "response", "url", "body_parser") parser: HttpRequestParser body_parser: BodyParser request: Request response: Response headers: Headers url: Any def __init__(self): super().__init__() self.parser = HttpRequestParser(self) self.headers = Headers() self.body_parser = None # ---------------- # # PROTOCOL METHODS # # ---------------- # def connection_lost(self, exc): # print("HTTP1Protocol.connection_lost") pass def pause_writing(self): print("HTTP1Protocol.pause_writing") def resume_writing(self): print("HTTP1Protocol.resume_writing") def data_received(self, data): self.parser.feed_data(data) def eof_received(self): print("HTTP1Protocol.eof_received") # --------------------- # # PARSER EVENT HANDLERS # # --------------------- # def on_url(self, url: bytes) -> None: self.url = parse_url(url) def on_header(self, name: bytes, value: bytes) -> None: self.headers[name] = value def on_headers_complete(self) -> None: injector = self.injector.descend() method = self.parser.get_method() if method == b"POST": if b"content-type" in self.headers: ct, params = parse_header( self.headers[b"content-type"].decode("ASCII")) if ct == "multipart/form-data": self.body_parser = FormDataParser( params["boundary"].encode("ASCII")) if self.body_parser is None: self.body_parser = RawBody() injector[BodyParser] = self.body_parser response = self.response = injector[Response] = injector[Response] request = self.request = injector[Request] = injector[Request] request.method = method request.version = response.version = self.parser.get_http_version() request.url = self.url request.headers = self.headers injector[Cookie] = injector[Cookie] task = self.loop.create_task(request()) task.injector = injector task.add_done_callback(self.__finalize_task) request.on_headers.set() # def on_message_begin(self): # print("on_message_begin") def on_body(self, body: bytes): if self.body_parser is not None: self.body_parser.feed(body) def on_message_complete(self): self.body_parser.process() self.body_parser = None self.request.on_body.set() def on_chunk_header(self): print("on_chunk_header") def on_chunk_complete(self): print("on_chunk_header") def __finalize_task(self, task): if task.cancelled(): response = task.injector[Response] if not response.headers_sent: self.loop.create_task(response.begin(503)) else: exc = task.exception() if exc is not None: self.loop.create_task(handle_error(task.injector, exc))
class ProxyProtocol(HttpProtocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'response_timeout', 'keep_alive_timeout', 'request_max_size', 'request_class', 'is_request_stream', 'router', # enable or disable access log purpose 'access_log', # connection management '_total_request_size', '_request_timeout_handler', '_response_timeout_handler', '_keep_alive_timeout_handler', '_last_request_time', '_last_response_time', '_is_stream_handler', '_not_paused') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.access_log = access_log self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.response_timeout = response_timeout self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or ProxyRequest self.is_request_stream = is_request_stream self._is_stream_handler = False self._not_paused = asyncio.Event(loop=loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b'' self.state = state if state else {} if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug self._not_paused.set() self._raw = asyncio.Queue() # 原始数据都保存 self._is_proxy = False # for proxy def data_received(self, data): self._raw.put_nowait(data) # 原始数据都保存 # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) if self._is_proxy: # 如果是代理过程那么就不要再去parse了 return # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserInvalidMethodError as e: # CONNECT 包 pass except HttpParserUpgrade: # CONNECT 包 pass except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False self._raw = asyncio.Queue() # for proxy 清理 self._is_proxy = False # for proxy def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport) self.request['_raw'] = self._raw # 原始数据包交给request处理函数 # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def on_message_complete(self): # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body = b''.join(self.request.body) if self.request.method == 'CONNECT' or self.request.raw_url.startswith( b'http'): self._is_proxy = True # CONNET 包或者url是完整url的包就是代理包 self.execute_request_handler() def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error( 'Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = sanic_server.current_time if self.request.method == 'CONNECT': return #https 代理的握手请求完了,可不能清理,后面还要代理的 self.cleanup()
class HttpProtocol(asyncio.Protocol): # http://book.pythontips.com/en/latest/__slots__magic.html # def __init__(self, *, loop, request_handler, error_handler, signal, connections, request_timeout) -> None: def __init__(self, params: panic_datatypes.ServerParams): self.params = params self.loop = params.loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.signal = params.signal self.connections = params.connections self.request_handler = params.request_handler self.request_timeout = params.request_timeout self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None self._identity = uuid.uuid4() # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later(self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = datetime.datetime.utcnow() def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() self.cleanup() def connection_timeout(self): time_elapsed = datetime.datetime.utcnow() - self._last_request_time try: if time_elapsed.seconds < self.request_timeout: time_left = self.request_timeout - time_elapsed.seconds self._timeout_handler = self.loop.call_later(time_left, self.connection_timeout) except Exception as err: print(err) import ipdb; ipdb.set_trace() pass else: if self._request_handler_task: self._request_handler_task.cancel() exception = panic_exceptions.RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits # TODO: ^ self._total_request_size += len(data) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = panic_datatypes.HTTPHeaders() self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError as err: import ipdb;ipdb.set_trace() exception = panic_exceptions.InvalidUsage('Bad Request') self.write_error(exception) def on_url(self, url): self.url = url def on_header(self, name, value): #if name == b'Content-Length' and int(value) > 1000: # exception = PayloadTooLarge('Payload Too Large') # self.write_error(exception) self.headers.append(name.decode(), value.decode('utf-8')) def on_headers_complete(self): remote_addr = self.transport.get_extra_info('peername') if remote_addr: self.headers.append(remote_addr[0], str(remote_addr[1])) self.request = panic_request.Request( url = self.url, headers = self.headers, version = self.parser.get_http_version(), method = panic_datatypes.HTTPMethod.Match(self.parser.get_method().decode()) ) def on_body(self, body): self.request.body.append(body) def on_message_complete(self): self._request_handler_task = self.loop.create_task(self.request_handler(self.request, self.write_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): if self.parser: keep_alive = self.parser.should_keep_alive() and not self.signal.stopped else: keep_alive = False try: self.transport.write(response.output(getattr(self.request, 'version', '1.1'))) except RuntimeError as err: logger.error(err) except Exception as err: import ipdb; ipdb.set_trace() pass if keep_alive: self._last_request_time = datetime.datetime.utcnow() self.cleanup() else: self.transport.close() def write_error(self, exception): try: response = self.params.error_handler(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(float(version))) self.transport.close() except panic_exceptions.RequestTimeout: exception = panic_exceptions.ServerError('RT') exception.status = 408 response = self.params.error_handler(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(float(version))) self.transport.close() #self.write_error(exception) except Exception as err: # logger.exception(err) import traceback traceback.print_stack() import ipdb;ipdb.set_trace() import sys; sys.exit(1) self.bail_out("Writing error failed, connection closed {}".format(e)) def bail_out(self, message): exception = ServerError(message) self.write_error(exception) logger.error(message) def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None def close_if_idle(self): """ Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'response_timeout', 'keep_alive_timeout', 'request_max_size', 'request_class', 'is_request_stream', 'router', # enable or disable access log purpose 'access_log', # connection management '_total_request_size', '_request_timeout_handler', '_response_timeout_handler', '_keep_alive_timeout_handler', '_last_request_time', '_last_response_time', '_is_stream_handler') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.access_log = access_log self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.response_timeout = response_timeout self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b'' self.state = state if state else {} if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug @property def keep_alive(self): return ( self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._request_timeout_handler = self.loop.call_later( self.request_timeout, self.request_timeout_callback) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) if self._request_timeout_handler: self._request_timeout_handler.cancel() if self._response_timeout_handler: self._response_timeout_handler.cancel() if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = ( self.loop.call_later(time_left, self.request_timeout_callback) ) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise RequestTimeout('Request Timeout') except RequestTimeout as exception: self.write_error(exception) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = current_time - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = ( self.loop.call_later(time_left, self.response_timeout_callback) ) else: try: raise ServiceUnavailable('Response Timeout') except ServiceUnavailable as exception: self.write_error(exception) def keep_alive_timeout_callback(self): # Check if elapsed time since last response exceeds our configured # maximum keep alive timeout value time_elapsed = current_time - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = ( self.loop.call_later(time_left, self.keep_alive_timeout_callback) ) else: logger.info('KeepAlive Timeout. Closing connection.') self.transport.close() self.transport = None # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if self._header_fragment == b'Content-Length' \ and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) try: value = value.decode() except UnicodeDecodeError: value = value.decode('latin_1') self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b'' def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport ) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(body)) return self.request.body.append(body) def on_message_complete(self): # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body = b''.join(self.request.body) self.execute_request_handler() def execute_request_handler(self): self._response_timeout_handler = self.loop.call_later( self.response_timeout, self.response_timeout_callback) self._last_request_time = current_time self._request_handler_task = self.loop.create_task( self.request_handler( self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def log_response(self, response): if self.access_log: extra = { 'status': getattr(response, 'status', 0), } if isinstance(response, HTTPResponse): extra['byte'] = len(response.body) else: extra['byte'] = -1 extra['host'] = 'UNKNOWN' if self.request is not None: if self.request.ip: extra['host'] = '{0[0]}:{0[1]}'.format(self.request.ip) extra['request'] = '{0} {1}'.format(self.request.method, self.request.url) else: extra['request'] = 'nil' access_logger.info('', extra=extra) def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output( self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error('Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.transport = self.transport await response.stream( self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) except AttributeError: logger.error('Invalid response object for url %s, ' 'Expected Type: HTTPResponse, Actual Type: %s', self.url, type(response)) self.write_error(ServerError('Invalid response type')) except RuntimeError: if self._debug: logger.error('Connection lost before response written @ %s', self.request.ip) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = current_time self.cleanup() def write_error(self, exception): # An error _is_ a response. # Don't throw a response timeout, when a response _is_ given. if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: if self._debug: logger.error('Connection lost before error written @ %s', self.request.ip if self.request else 'Unknown') except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format( repr(e)), from_error=True ) finally: if self.parser and (self.keep_alive or getattr(response, 'status', 0) == 408): self.log_response(response) self.transport.close() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): logger.error("Transport closed @ %s and exception " "experienced during error handling", self.transport.get_extra_info('peername')) logger.debug('Exception:\n%s', traceback.format_exc()) else: exception = ServerError(message) self.write_error(exception) logger.error(message) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class BaseHTTPRequestHandler(BaseRequestHandler): request_klass = HTTPRequest response_klass = HTTPResponse def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.parser = HttpRequestParser(self) self._url: bytes = b"" self._headers: tp.Dict[bytes, bytes] = {} self._body: bytes = b"" self._parsed = False def handle(self) -> None: request = self.parse_request() if request: try: response = self.handle_request(request) except Exception: print("Ooops... Error 500") response = self.response_klass(status=500, headers={}, body=b"") else: response = self.response_klass(status=400, headers={}, body=b"") self.handle_response(response) self.close() def parse_request(self) -> tp.Optional[HTTPRequest]: while not self._parsed: try: data = self.socket.recv(1024) if data == b"": print("Ok. Parsed done successfully") break self.parser.feed_data(data) except socket.timeout: print("Ooops... Timeout!") break except ( HttpParserError, HttpParserCallbackError, HttpParserInvalidStatusError, HttpParserInvalidMethodError, HttpParserInvalidURLError, HttpParserUpgrade, ): print("Ooops... Parser error") break if self._parsed: return self.request_klass( method=self.parser.get_method(), url=self._url, headers=self._headers, body=self._body, ) return None def handle_request(self, request: HTTPRequest) -> HTTPResponse: return self.response_klass(status=405, headers={}, body=b"") def handle_response(self, response: HTTPResponse) -> None: self.socket.sendall(response.to_http1()) def on_url(self, url: bytes) -> None: self._url = url def on_header(self, name: bytes, value: bytes) -> None: self._headers[name] = value def on_body(self, body: bytes) -> None: self._body = body def on_message_complete(self) -> None: self._parsed = True
class HttpProtocol(asyncio.Protocol): """ This class provides a basic HTTP implementation of the sanic framework. """ __slots__ = ( # app "app", # event loop, connection "loop", "transport", "connections", "signal", # request params "parser", "request", "url", "headers", # request config "request_handler", "request_timeout", "response_timeout", "keep_alive_timeout", "request_max_size", "request_buffer_queue_size", "request_class", "is_request_stream", "router", "error_handler", # enable or disable access log purpose "access_log", # connection management "_total_request_size", "_request_timeout_handler", "_response_timeout_handler", "_keep_alive_timeout_handler", "_last_request_time", "_last_response_time", "_is_stream_handler", "_not_paused", "_request_handler_task", "_request_stream_task", "_keep_alive", "_header_fragment", "state", "_debug", ) def __init__(self, *, loop, app, request_handler, error_handler, signal=Signal(), connections=None, request_timeout=60, response_timeout=60, keep_alive_timeout=5, request_max_size=None, request_buffer_queue_size=100, request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.app = app self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.access_log = access_log self.connections = connections if connections is not None else set() self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_buffer_queue_size = request_buffer_queue_size self.response_timeout = response_timeout self.keep_alive_timeout = keep_alive_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._not_paused = asyncio.Event(loop=loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b"" self.state = state if state else {} if "requests_count" not in self.state: self.state["requests_count"] = 0 self._debug = debug self._not_paused.set() @property def keep_alive(self): """ Check if the connection needs to be kept alive based on the params attached to the `_keep_alive` attribute, :attr:`Signal.stopped` and :func:`HttpProtocol.parser.should_keep_alive` :return: ``True`` if connection is to be kept alive ``False`` else """ return (self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._request_timeout_handler = self.loop.call_later( self.request_timeout, self.request_timeout_callback) self.transport = transport self._last_request_time = time() def connection_lost(self, exc): self.connections.discard(self) if self._request_handler_task: self._request_handler_task.cancel() if self._request_stream_task: self._request_stream_task.cancel() if self._request_timeout_handler: self._request_timeout_handler.cancel() if self._response_timeout_handler: self._response_timeout_handler.cancel() if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() def pause_writing(self): self._not_paused.clear() def resume_writing(self): self._not_paused.set() def request_timeout_callback(self): # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = self.loop.call_later( time_left, self.request_timeout_callback) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() self.write_error(RequestTimeout("Request Timeout")) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = self.loop.call_later( time_left, self.response_timeout_callback) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() self.write_error(ServiceUnavailable("Response Timeout")) def keep_alive_timeout_callback(self): """ Check if elapsed time since last response exceeds our configured maximum keep alive timeout value and if so, close the transport pipe and let the response writer handle the error. :return: None """ time_elapsed = time() - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = self.loop.call_later( time_left, self.keep_alive_timeout_callback) else: logger.debug("KeepAlive Timeout. Closing connection.") self.transport.close() self.transport = None # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: self.write_error(PayloadTooLarge("Payload Too Large")) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state["requests_count"] = self.state["requests_count"] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = "Bad Request" if self._debug: message += "\n" + traceback.format_exc() self.write_error(InvalidUsage(message)) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if (self._header_fragment == b"Content-Length" and int(value) > self.request_max_size): self.write_error(PayloadTooLarge("Payload Too Large")) try: value = value.decode() except UnicodeDecodeError: value = value.decode("latin_1") self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b"" def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=Header(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport, app=self.app, ) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.request.headers.get(EXPECT_HEADER): self.expect_handler() if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = StreamBuffer( self.request_buffer_queue_size) self.execute_request_handler() def expect_handler(self): """ Handler for Expect Header. """ expect = self.request.headers.get(EXPECT_HEADER) if self.request.version == "1.1": if expect.lower() == "100-continue": self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n") else: self.write_error( HeaderExpectationFailed( "Unknown Expect: {expect}".format(expect=expect))) def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.body_append(body)) else: self.request.body_push(body) async def body_append(self, body): if self.request.stream.is_full(): self.transport.pause_reading() await self.request.stream.put(body) self.transport.resume_reading() else: await self.request.stream.put(body) def on_message_complete(self): # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body_finish() self.execute_request_handler() def execute_request_handler(self): """ Invoke the request handler defined by the :func:`sanic.app.Sanic.handle_request` method :return: None """ self._response_timeout_handler = self.loop.call_later( self.response_timeout, self.response_timeout_callback) self._last_request_time = time() self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def log_response(self, response): """ Helper method provided to enable the logging of responses in case if the :attr:`HttpProtocol.access_log` is enabled. :param response: Response generated for the current request :type response: :class:`sanic.response.HTTPResponse` or :class:`sanic.response.StreamingHTTPResponse` :return: None """ if self.access_log: extra = {"status": getattr(response, "status", 0)} if isinstance(response, HTTPResponse): extra["byte"] = len(response.body) else: extra["byte"] = -1 extra["host"] = "UNKNOWN" if self.request is not None: if self.request.ip: extra["host"] = "{0}:{1}".format(self.request.ip, self.request.port) extra["request"] = "{0} {1}".format(self.request.method, self.request.url) else: extra["request"] = "nil" access_logger.info("", extra=extra) def write_response(self, response): """ Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self._debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = time() self.cleanup() async def drain(self): await self._not_paused.wait() async def push_data(self, data): self.transport.write(data) async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.protocol = self await response.stream(self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self._debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = time() self.cleanup() def write_error(self, exception): # An error _is_ a response. # Don't throw a response timeout, when a response _is_ given. if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else "1.1" self.transport.write(response.output(version)) except RuntimeError: if self._debug: logger.error( "Connection lost before error written @ %s", self.request.ip if self.request else "Unknown", ) except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(repr(e)), from_error=True, ) finally: if self.parser and (self.keep_alive or getattr(response, "status", 0) == 408): self.log_response(response) try: self.transport.close() except AttributeError: logger.debug("Connection lost before server could close it.") def bail_out(self, message, from_error=False): """ In case if the transport pipes are closed and the sanic app encounters an error while writing data to the transport pipe, we log the error with proper details. :param message: Error message to display :param from_error: If the bail out was invoked while handling an exception scenario. :type message: str :type from_error: bool :return: None """ if from_error or self.transport is None or self.transport.is_closing(): logger.error( "Transport closed @ %s and exception " "experienced during error handling", (self.transport.get_extra_info("peername") if self.transport is not None else "N/A"), ) logger.debug("Exception:", exc_info=True) else: self.write_error(ServerError(message)) logger.error(message) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class HttpProtocol(asyncio.Protocol): """ This class provides a basic HTTP implementation of the web framework. """ __slots__ = ( # app "app", # event loop, connection "loop", "transport", "connections", "signal", # request params "parser", "request", "url", "headers", # request config "request_handler", "request_timeout", "response_timeout", "keep_alive_timeout", "request_max_size", "request_buffer_queue_size", "request_class", "is_request_stream", "error_handler", # enable or disable access log purpose "access_log", # connection management "_total_request_size", "_request_timeout_handler", "_response_timeout_handler", "_keep_alive_timeout_handler", "_last_request_time", "_last_response_time", "_is_stream_handler", "_not_paused", "_request_handler_task", "_request_stream_task", "_keep_alive", "_header_fragment", "state", "_body_chunks", ) def __init__( self, *, loop, app, signal=Signal(), connections=None, state=None, **kwargs, ): asyncio.set_event_loop(loop) self.loop = loop deprecated_loop = self.loop if sys.version_info < (3, 7) else None self.app = app self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.signal = signal self.access_log = self.app.config.ACCESS_LOG self.connections = connections if connections is not None else set() self.request_handler = self.app.handle_request self.error_handler = self.app.error_handler self.request_timeout = self.app.config.REQUEST_TIMEOUT self.request_buffer_queue_size = ( self.app.config.REQUEST_BUFFER_QUEUE_SIZE) self.response_timeout = self.app.config.RESPONSE_TIMEOUT self.keep_alive_timeout = self.app.config.KEEP_ALIVE_TIMEOUT self.request_max_size = self.app.config.REQUEST_MAX_SIZE self.request_class = self.app.request_class or Request self.is_request_stream = self.app.is_request_stream self._is_stream_handler = False self._not_paused = asyncio.Event(loop=deprecated_loop) self._total_request_size = 0 self._request_timeout_handler = None self._response_timeout_handler = None self._keep_alive_timeout_handler = None self._last_request_time = None self._last_response_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = self.app.config.KEEP_ALIVE self._header_fragment = b"" self.state = state if state else {} if "requests_count" not in self.state: self.state["requests_count"] = 0 # 设置Event对象内部的信号标志为真 self._not_paused.set() self._body_chunks = deque() @property def keep_alive(self): """ Check if the connection needs to be kept alive based on the params attached to the `_keep_alive` attribute, :attr:`Signal.stopped` and :func:`HttpProtocol.parser.should_keep_alive` :return: ``True`` if connection is to be kept alive ``False`` else """ return (self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): """创建连接时执行 .""" self.connections.add(self) # 设置请求超时处理函数 self._request_timeout_handler = self.loop.call_later( self.request_timeout, self.request_timeout_callback) self.transport = transport # 记录请求时间 self._last_request_time = time() def connection_lost(self, exc): """连接丢失时执行 .""" self.connections.discard(self) if self._request_handler_task: self._request_handler_task.cancel() if self._request_stream_task: self._request_stream_task.cancel() if self._request_timeout_handler: self._request_timeout_handler.cancel() if self._response_timeout_handler: self._response_timeout_handler.cancel() if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() def pause_writing(self): self._not_paused.clear() def resume_writing(self): self._not_paused.set() def request_timeout_callback(self): """请求超时处理函数 .""" # See the docstring in the RequestTimeout exception, to see # exactly what this timeout is checking for. # Check if elapsed time since request initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time # 判断请求是否超时 # 没有超时,继续判断 if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._request_timeout_handler = self.loop.call_later( time_left, self.request_timeout_callback) else: # 超时取消流任务 if self._request_stream_task: self._request_stream_task.cancel() # 超时取消Handler请求 if self._request_handler_task: self._request_handler_task.cancel() self.write_error(RequestTimeout("Request Timeout")) def response_timeout_callback(self): # Check if elapsed time since response was initiated exceeds our # configured maximum request timeout value time_elapsed = time() - self._last_request_time if time_elapsed < self.response_timeout: time_left = self.response_timeout - time_elapsed self._response_timeout_handler = self.loop.call_later( time_left, self.response_timeout_callback) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() self.write_error(ServiceUnavailable("Response Timeout")) def keep_alive_timeout_callback(self): """ Check if elapsed time since last response exceeds our configured maximum keep alive timeout value and if so, close the transport pipe and let the response writer handle the error. :return: None """ time_elapsed = time() - self._last_response_time if time_elapsed < self.keep_alive_timeout: time_left = self.keep_alive_timeout - time_elapsed self._keep_alive_timeout_handler = self.loop.call_later( time_left, self.keep_alive_timeout_callback) else: logger.debug("KeepAlive Timeout. Closing connection.") # 关闭连接 self.transport.close() self.transport = None # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): """接受到HTTP请求时调用 .""" # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) # 限制请求内容的长度 if self._total_request_size > self.request_max_size: self.write_error(PayloadTooLarge("Payload Too Large")) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state["requests_count"] = self.state["requests_count"] + 1 # Parse request chunk or close connection try: # 解析HTTP协议 self.parser.feed_data(data) except HttpParserError: # 如果不是合法的HTTP协议,返回400错误 message = "Bad Request" if self.app.debug: message += "\n" + traceback.format_exc() self.write_error(InvalidUsage(message)) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if (self._header_fragment == b"Content-Length" and int(value) > self.request_max_size): self.write_error(PayloadTooLarge("Payload Too Large")) try: value = value.decode() except UnicodeDecodeError: value = value.decode("latin_1") self.headers.append( (self._header_fragment.decode().casefold(), value)) self._header_fragment = b"" def on_headers_complete(self): """在服务器接收到头部后,创建一个请求对象 .""" self.request = self.request_class( url_bytes=self.url, headers=Header(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport, app=self.app, ) # Remove any existing KeepAlive handler here, # It will be recreated if required on the new request. if self._keep_alive_timeout_handler: self._keep_alive_timeout_handler.cancel() self._keep_alive_timeout_handler = None if self.request.headers.get(EXPECT_HEADER): self.expect_handler() if self.is_request_stream: self._is_stream_handler = self.app.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = StreamBuffer( self.request_buffer_queue_size) self.execute_request_handler() def expect_handler(self): """ Handler for Expect Header. """ expect = self.request.headers.get(EXPECT_HEADER) if self.request.version == "1.1": if expect.lower() == "100-continue": self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n") else: self.write_error( HeaderExpectationFailed(f"Unknown Expect: {expect}")) def on_body(self, body): """服务器接收到body后,将其存放到请求对象中 .""" if self.is_request_stream and self._is_stream_handler: # body chunks can be put into asyncio.Queue out of order if # multiple tasks put concurrently and the queue is full in python # 3.7. so we should not create more than one task putting into the # queue simultaneously. self._body_chunks.append(body) if (not self._request_stream_task or self._request_stream_task.done()): self._request_stream_task = self.loop.create_task( self.stream_append()) else: self.request.body_push(body) async def body_append(self, body): if (self.request is None or self._request_stream_task is None or self._request_stream_task.cancelled()): return if self.request.stream.is_full(): self.transport.pause_reading() await self.request.stream.put(body) self.transport.resume_reading() else: await self.request.stream.put(body) async def stream_append(self): while self._body_chunks: body = self._body_chunks.popleft() if self.request.stream.is_full(): self.transport.pause_reading() await self.request.stream.put(body) self.transport.resume_reading() else: await self.request.stream.put(body) def on_message_complete(self): """服务器收到全部消息 .""" # Entire request (headers and whole body) is received. # We can cancel and remove the request timeout handler now. if self._request_timeout_handler: self._request_timeout_handler.cancel() self._request_timeout_handler = None if self.is_request_stream and self._is_stream_handler: self._body_chunks.append(None) if (not self._request_stream_task or self._request_stream_task.done()): self._request_stream_task = self.loop.create_task( self.stream_append()) return # 服务器收到全部请求后,将完整的body放到请求对象中 self.request.body_finish() # 执行请求处理器 self.execute_request_handler() def execute_request_handler(self): """ Invoke the request handler defined by the :func:`sanic.app.Sanic.handle_request` method :return: None """ self._response_timeout_handler = self.loop.call_later( self.response_timeout, self.response_timeout_callback) self._last_request_time = time() # 创建执行请求处理器任务 self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def log_response(self, response): """ Helper method provided to enable the logging of responses in case if the :attr:`HttpProtocol.access_log` is enabled. :param response: Response generated for the current request :type response: :class:`sanic.response.HTTPResponse` or :class:`sanic.response.StreamingHTTPResponse` :return: None """ if self.access_log: extra = {"status": getattr(response, "status", 0)} if isinstance(response, HTTPResponse): extra["byte"] = len(response.body) else: extra["byte"] = -1 extra["host"] = "UNKNOWN" if self.request is not None: if self.request.ip: extra["host"] = f"{self.request.ip}:{self.request.port}" extra["request"] = f"{self.request.method} {self.request.url}" else: extra["request"] = "nil" access_logger.info("", extra=extra) def write_response(self, response): """处理HTTP响应 Writes response content synchronously to the transport. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.keep_alive_timeout)) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self.app.debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out(f"Writing response failed, connection closed {e!r}") finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = time() self.cleanup() async def drain(self): await self._not_paused.wait() async def push_data(self, data): self.transport.write(data) async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None try: keep_alive = self.keep_alive response.protocol = self await response.stream(self.request.version, keep_alive, self.keep_alive_timeout) self.log_response(response) except AttributeError: logger.error( "Invalid response object for url %s, " "Expected Type: HTTPResponse, Actual Type: %s", self.url, type(response), ) self.write_error(ServerError("Invalid response type")) except RuntimeError: if self.app.debug: logger.error( "Connection lost before response written @ %s", self.request.ip, ) keep_alive = False except Exception as e: self.bail_out(f"Writing response failed, connection closed {e!r}") finally: if not keep_alive: self.transport.close() self.transport = None else: self._keep_alive_timeout_handler = self.loop.call_later( self.keep_alive_timeout, self.keep_alive_timeout_callback) self._last_response_time = time() self.cleanup() def write_error(self, exception): # An error _is_ a response. # Don't throw a response timeout, when a response _is_ given. if self._response_timeout_handler: self._response_timeout_handler.cancel() self._response_timeout_handler = None response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else "1.1" self.transport.write(response.output(version)) except RuntimeError: if self.app.debug: logger.error( "Connection lost before error written @ %s", self.request.ip if self.request else "Unknown", ) except Exception as e: self.bail_out( f"Writing error failed, connection closed {e!r}", from_error=True, ) finally: if self.parser and (self.keep_alive or getattr(response, "status", 0) == 408): self.log_response(response) try: self.transport.close() except AttributeError: logger.debug("Connection lost before server could close it.") def bail_out(self, message, from_error=False): """ In case if the transport pipes are closed and the sanic app encounters an error while writing data to the transport pipe, we log the error with proper details. :param message: Error message to display :param from_error: If the bail out was invoked while handling an exception scenario. :type message: str :type from_error: bool :return: None """ if from_error or self.transport is None or self.transport.is_closing(): logger.error( "Transport closed @ %s and exception " "experienced during error handling", (self.transport.get_extra_info("peername") if self.transport is not None else "N/A"), ) logger.debug("Exception:", exc_info=True) else: self.write_error(ServerError(message)) logger.error(message) def cleanup(self): """This is called when KeepAlive feature is used, it resets the connection in order for it to be able to handle receiving another request on the same connection.""" self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser and self.transport is not None: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class Channel: __slots__ = ( 'parser', 'request', 'complete', 'headers_complete', 'socket', 'reader', ) def __init__(self, socket): self.complete = False self.headers_complete = False self.parser = HttpRequestParser(self) self.request = None self.socket = socket self.reader = self._reader() def data_received(self, data: bytes): try: self.parser.feed_data(data) except HttpParserUpgrade: self.request.upgrade = True except (HttpParserError, HttpParserInvalidMethodError) as exc: # We should log the exc. raise HTTPError( HTTPStatus.BAD_REQUEST, 'Unparsable request.') async def read(self, parse: bool=True) -> bytes: data = await self.socket.recv(1024) if data: if parse: self.data_received(data) return data async def _reader(self) -> bytes: while not self.complete: data = await self.read() if not data: break yield data async def _drainer(self) -> bytes: while True: data = await self.read(parse=False) if not data: break yield data def on_header(self, name: bytes, value: bytes): value = value.decode() if value: name = name.decode().title() if name in self.request.headers: self.request.headers[name] += ', {}'.format(value) else: self.request.headers[name] = value def on_body(self, data: bytes): self.request.body += data def on_message_begin(self): self.complete = False self.request = Request(self.socket, self.reader) def on_message_complete(self): self.complete = True def on_url(self, url: bytes): self.request.url = url parsed = parse_url(url) self.request.path = unquote(parsed.path.decode()) self.request.query_string = (parsed.query or b'').decode() def on_headers_complete(self): self.request.keep_alive = self.parser.should_keep_alive() self.request.method = self.parser.get_method().decode().upper() self.headers_complete = True async def __aiter__(self): keep_alive = True while keep_alive: data = await self.read() if data is None: break if self.headers_complete: yield self.request keep_alive = self.request.keep_alive if keep_alive: if not self.complete: await self.reader.aclose() # We drain if there's an uncomplete request. async for _ in self._drainer(): pass self.request = None self.complete = False self.headers_complete = False self.reader = self._reader()
class Protocol(asyncio.Protocol): """Responsible of parsing the request and writing the response. You can subclass it to set your own `Query`, `Request` or `Response` classes. """ __slots__ = ('app', 'req', 'parser', 'resp', 'writer') Query = Query Request = Request Response = Response def __init__(self, app): self.app = app self.parser = HttpRequestParser(self) def data_received(self, data: bytes): try: self.parser.feed_data(data) except HttpParserError: # If the parsing failed before on_message_begin, we don't have a # response. self.response = Response() self.response.status = HTTPStatus.BAD_REQUEST self.response.body = b'Unparsable request' self.write() def connection_made(self, transport): self.writer = transport # All on_xxx methods are in use by httptools parser. # See https://github.com/MagicStack/httptools#apis def on_header(self, name: bytes, value: bytes): self.request.headers[name.decode()] = value.decode() def on_body(self, body: bytes): self.request.body += body def on_url(self, url: bytes): self.request.url = url parsed = parse_url(url) self.request.path = parsed.path.decode() self.request.query_string = (parsed.query or b'').decode() parsed_qs = parse_qs(self.request.query_string, keep_blank_values=True) self.request.query = self.Query(parsed_qs) def on_message_begin(self): self.request = self.Request() self.response = self.Response() def on_message_complete(self): self.request.method = self.parser.get_method().decode().upper() task = self.app.loop.create_task(self.app(self.request, self.response)) task.add_done_callback(self.write) # May or may not have "future" as arg. def write(self, *args): # Appends bytes for performances. payload = b'HTTP/1.1 %a %b\r\n' % ( self.response.status.value, self.response.status.phrase.encode()) if not isinstance(self.response.body, bytes): self.response.body = self.response.body.encode() if 'Content-Length' not in self.response.headers: length = len(self.response.body) self.response.headers['Content-Length'] = str(length) for key, value in self.response.headers.items(): payload += b'%b: %b\r\n' % (key.encode(), str(value).encode()) payload += b'\r\n%b' % self.response.body self.writer.write(payload) if not self.parser.should_keep_alive(): self.writer.close()
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', 'request_class', 'is_request_stream', 'router', # enable or disable access log / error log purpose 'has_log', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time', '_is_stream_handler') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, request_max_size=None, request_class=None, has_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.has_log = has_log self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b'' self.state = state if state else {} if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug @property def keep_alive(self): return (self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later(self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = (self.loop.call_later( time_left, self.connection_timeout)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() exception = RequestTimeout('Request Timeout') self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if self._header_fragment == b'Content-Length' \ and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append( (self._header_fragment.decode().casefold(), value.decode())) self._header_fragment = b'' def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport) if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(body)) return self.request.body.append(body) def on_message_complete(self): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body = b''.join(self.request.body) self.execute_request_handler() def execute_request_handler(self): self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): """ Writes response content synchronously to the transport. """ try: keep_alive = self.keep_alive self.transport.write( response.output(self.request.version, keep_alive, self.request_timeout)) if self.has_log: netlog.info('', extra={ 'status': response.status, 'byte': len(response.body), 'host': '{0}:{1}'.format(self.request.ip[0], self.request.ip[1]), 'request': '{0} {1}'.format(self.request.method, self.request.url) }) except AttributeError: log.error(('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error('Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ try: keep_alive = self.keep_alive response.transport = self.transport await response.stream(self.request.version, keep_alive, self.request_timeout) if self.has_log: netlog.info('', extra={ 'status': response.status, 'byte': -1, 'host': '{0}:{1}'.format(self.request.ip[0], self.request.ip[1]), 'request': '{0} {1}'.format(self.request.method, self.request.url) }) except AttributeError: log.error(('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error('Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() def write_error(self, exception): try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: log.error('Connection lost before error written @ {}'.format( self.request.ip if self.request else 'Unknown')) except Exception as e: self.bail_out("Writing error failed, connection closed {}".format( repr(e)), from_error=True) finally: if self.has_log: extra = { 'status': response.status, 'host': '', 'request': str(self.request) + str(self.url) } if response and isinstance(response, HTTPResponse): extra['byte'] = len(response.body) else: extra['byte'] = -1 if self.request: extra['host'] = '%s:%d' % self.request.ip, extra['request'] = '%s %s' % (self.request.method, self.url) netlog.info('', extra=extra) self.transport.close() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): log.error(("Transport closed @ {} and exception " "experienced during error handling").format( self.transport.get_extra_info('peername'))) log.debug('Exception:\n{}'.format(traceback.format_exc())) else: exception = ServerError(message) self.write_error(exception) log.error(message) def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', 'request_class', 'is_request_stream', 'router', # enable or disable access log / error log purpose 'has_log', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time', '_is_stream_handler') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections=set(), request_timeout=60, request_max_size=None, request_class=None, has_log=True, keep_alive=True, is_request_stream=False, router=None, state=None, debug=False, **kwargs): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.router = router self.signal = signal self.has_log = has_log self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self.request_class = request_class or Request self.is_request_stream = is_request_stream self._is_stream_handler = False self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None self._request_stream_task = None self._keep_alive = keep_alive self._header_fragment = b'' self.state = state if state else {} if 'requests_count' not in self.state: self.state['requests_count'] = 0 self._debug = debug @property def keep_alive(self): return ( self._keep_alive and not self.signal.stopped and self.parser.should_keep_alive()) # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections.add(self) self._timeout_handler = self.loop.call_later( self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): self.connections.discard(self) self._timeout_handler.cancel() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = ( self.loop.call_later(time_left, self.connection_timeout)) else: if self._request_stream_task: self._request_stream_task.cancel() if self._request_handler_task: self._request_handler_task.cancel() try: raise RequestTimeout('Request Timeout') except RequestTimeout as exception: self.write_error(exception) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # requests count self.state['requests_count'] = self.state['requests_count'] + 1 # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError: message = 'Bad Request' if self._debug: message += '\n' + traceback.format_exc() exception = InvalidUsage(message) self.write_error(exception) def on_url(self, url): if not self.url: self.url = url else: self.url += url def on_header(self, name, value): self._header_fragment += name if value is not None: if self._header_fragment == b'Content-Length' \ and int(value) > self.request_max_size: exception = PayloadTooLarge('Payload Too Large') self.write_error(exception) self.headers.append( (self._header_fragment.decode().casefold(), value.decode())) self._header_fragment = b'' def on_headers_complete(self): self.request = self.request_class( url_bytes=self.url, headers=CIDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode(), transport=self.transport ) if self.is_request_stream: self._is_stream_handler = self.router.is_stream_handler( self.request) if self._is_stream_handler: self.request.stream = asyncio.Queue() self.execute_request_handler() def on_body(self, body): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(body)) return self.request.body.append(body) def on_message_complete(self): if self.is_request_stream and self._is_stream_handler: self._request_stream_task = self.loop.create_task( self.request.stream.put(None)) return self.request.body = b''.join(self.request.body) self.execute_request_handler() def execute_request_handler(self): self._request_handler_task = self.loop.create_task( self.request_handler( self.request, self.write_response, self.stream_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): """ Writes response content synchronously to the transport. """ try: keep_alive = self.keep_alive self.transport.write( response.output( self.request.version, keep_alive, self.request_timeout)) if self.has_log: netlog.info('', extra={ 'status': response.status, 'byte': len(response.body), 'host': '{0}:{1}'.format(self.request.ip[0], self.request.ip[1]), 'request': '{0} {1}'.format(self.request.method, self.request.url) }) except AttributeError: log.error( ('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error( 'Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() async def stream_response(self, response): """ Streams a response to the client asynchronously. Attaches the transport to the response so the response consumer can write to the response as needed. """ try: keep_alive = self.keep_alive response.transport = self.transport await response.stream( self.request.version, keep_alive, self.request_timeout) if self.has_log: netlog.info('', extra={ 'status': response.status, 'byte': -1, 'host': '{0}:{1}'.format(self.request.ip[0], self.request.ip[1]), 'request': '{0} {1}'.format(self.request.method, self.request.url) }) except AttributeError: log.error( ('Invalid response object for url {}, ' 'Expected Type: HTTPResponse, Actual Type: {}').format( self.url, type(response))) self.write_error(ServerError('Invalid response type')) except RuntimeError: log.error( 'Connection lost before response written @ {}'.format( self.request.ip)) except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format( repr(e))) finally: if not keep_alive: self.transport.close() else: self._last_request_time = current_time self.cleanup() def write_error(self, exception): response = None try: response = self.error_handler.response(self.request, exception) version = self.request.version if self.request else '1.1' self.transport.write(response.output(version)) except RuntimeError: log.error( 'Connection lost before error written @ {}'.format( self.request.ip if self.request else 'Unknown')) except Exception as e: self.bail_out( "Writing error failed, connection closed {}".format(repr(e)), from_error=True) finally: if self.has_log: extra = dict() if isinstance(response, HTTPResponse): extra['status'] = response.status extra['byte'] = len(response.body) else: extra['status'] = 0 extra['byte'] = -1 if self.request: extra['host'] = '%s:%d' % self.request.ip, extra['request'] = '%s %s' % (self.request.method, self.url) else: extra['host'] = 'UNKNOWN' extra['request'] = 'nil' if self.parser and not (self.keep_alive and extra['status'] == 408): netlog.info('', extra=extra) self.transport.close() def bail_out(self, message, from_error=False): if from_error or self.transport.is_closing(): log.error( ("Transport closed @ {} and exception " "experienced during error handling").format( self.transport.get_extra_info('peername'))) log.debug( 'Exception:\n{}'.format(traceback.format_exc())) else: exception = ServerError(message) self.write_error(exception) log.error(message) def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._request_stream_task = None self._total_request_size = 0 self._is_stream_handler = False def close_if_idle(self): """Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False def close(self): """ Force close the connection. """ if self.transport is not None: self.transport.close() self.transport = None
class HttpProtocol(asyncio.Protocol): __slots__ = ( # event loop, connection 'loop', 'transport', 'connections', 'signal', # request params 'parser', 'request', 'url', 'headers', # request config 'request_handler', 'request_timeout', 'request_max_size', # connection management '_total_request_size', '_timeout_handler', '_last_communication_time') def __init__(self, *, loop, request_handler, error_handler, signal=Signal(), connections={}, request_timeout=60, request_max_size=None): self.loop = loop self.transport = None self.request = None self.parser = None self.url = None self.headers = None self.signal = signal self.connections = connections self.request_handler = request_handler self.error_handler = error_handler self.request_timeout = request_timeout self.request_max_size = request_max_size self._total_request_size = 0 self._timeout_handler = None self._last_request_time = None self._request_handler_task = None # -------------------------------------------- # # Connection # -------------------------------------------- # def connection_made(self, transport): self.connections[self] = True self._timeout_handler = self.loop.call_later(self.request_timeout, self.connection_timeout) self.transport = transport self._last_request_time = current_time def connection_lost(self, exc): del self.connections[self] self._timeout_handler.cancel() self.cleanup() def connection_timeout(self): # Check if time_elapsed = current_time - self._last_request_time if time_elapsed < self.request_timeout: time_left = self.request_timeout - time_elapsed self._timeout_handler = \ self.loop.call_later(time_left, self.connection_timeout) else: if self._request_handler_task: self._request_handler_task.cancel() response = self.error_handler.response( self.request, RequestTimeout('Request Timeout')) self.write_response(response) # -------------------------------------------- # # Parsing # -------------------------------------------- # def data_received(self, data): # Check for the request itself getting too large and exceeding # memory limits self._total_request_size += len(data) if self._total_request_size > self.request_max_size: return self.bail_out( "Request too large ({}), connection closed".format( self._total_request_size)) # Create parser if this is the first time we're receiving data if self.parser is None: assert self.request is None self.headers = [] self.parser = HttpRequestParser(self) # Parse request chunk or close connection try: self.parser.feed_data(data) except HttpParserError as e: self.bail_out( "Invalid request data, connection closed ({})".format(e)) def on_url(self, url): self.url = url def on_header(self, name, value): if name == b'Content-Length' and int(value) > self.request_max_size: return self.bail_out( "Request body too large ({}), connection closed".format(value)) self.headers.append((name.decode(), value.decode('utf-8'))) def on_headers_complete(self): remote_addr = self.transport.get_extra_info('peername') if remote_addr: self.headers.append(('Remote-Addr', '%s:%s' % remote_addr)) self.request = Request(url_bytes=self.url, headers=CIMultiDict(self.headers), version=self.parser.get_http_version(), method=self.parser.get_method().decode()) def on_body(self, body): if self.request.body: self.request.body += body else: self.request.body = body def on_message_complete(self): self._request_handler_task = self.loop.create_task( self.request_handler(self.request, self.write_response)) # -------------------------------------------- # # Responding # -------------------------------------------- # def write_response(self, response): try: keep_alive = self.parser.should_keep_alive() \ and not self.signal.stopped self.transport.write( response.output(self.request.version, keep_alive, self.request_timeout)) if not keep_alive: self.transport.close() else: # Record that we received data self._last_request_time = current_time self.cleanup() except Exception as e: self.bail_out( "Writing response failed, connection closed {}".format(e)) def bail_out(self, message): log.debug(message) self.transport.close() def cleanup(self): self.parser = None self.request = None self.url = None self.headers = None self._request_handler_task = None self._total_request_size = 0 def close_if_idle(self): """ Close the connection if a request is not being sent or received :return: boolean - True if closed, false if staying open """ if not self.parser: self.transport.close() return True return False
class LuyProtocol(asyncio.Protocol): def __init__(self, app, loop=None, keep_alive=True, request_max_size=None, has_stream=False, debug=True): self.parser = None self.url = None self._request_handler_task = None self._request_stream_task = None self.request_max_size = request_max_size self._total_request_size = 0 self.loop = loop self.header = {} self.app = app self.keep_alive = keep_alive self.has_stream = has_stream self.stream_handler = None def connection_made(self, transport): self.transport = transport if self.parser is None: self.parser = HttpRequestParser(self) def connection_lost(self, exc): self.transport.close() self.refresh() self.transport = None #------------------------------------- # parsing #------------------------------------- def data_received(self, data): ''' reveiving data from network. it is a streaming. has to check the data size for pretecting memory limits. ''' if self.request_max_size: self._total_request_size += len(data) if self._total_request_size > self.request_max_size: # todo:payload too large,have to implement a method represent PAYLOAD TOO LARGE self.write_error() try: self.parser.feed_data(data) except HttpParserError as e: print('出错', e) finally: pass def on_message_begin(self): # print('on_message_begin') pass def on_url(self, url): # storing the url from web if not self.url: self.url = url else: self.url += url def on_header(self, name, value): ''' Content-Length cannot be too long, protect the server ''' if value is not None: if name == b'Content-Length' and int(value) > 1500: self.write_error() self.header[name.decode().casefold()] = value.decode() def on_headers_complete(self): self.request = request_class(url_bytes=self.url, header=self.header, version=self.parser.get_http_version(), method=self.parser.get_method().decode()) # here is where we deal with "Expect: 100-Continue" # when user upload some big file or big things # their client would send a header with 'Expect: 100-Continue' # to if check the server can be accepted. if self.has_stream: self.stream_handler, kw = self.app.router.get_mapped_handle( self.request) if self.stream_handler and kw['stream']: # here is a coroutine queue # when await get() self.request.stream = asyncio.Queue() self.execute_request_handler() def on_body(self, body): if self.has_stream and self.stream_handler: self.loop.create_task(self.request.stream.put(body)) return self.request.body.append(body) def on_message_complete(self): # None is the signal for task to stop if self.has_stream and self.stream_handler: self.loop.create_task(self.request.stream.put(None)) return self.execute_request_handler() def execute_request_handler(self): self._request_handler_task = self.loop.create_task( self.app.request_handler(self.request, self.write_response, self.stream_callback)) #--------------------------- # error handling #--------------------------- def write_error(self): response = html('bad connecton', status=400) self.write_response(response) self.transport.close() #------------------------------------- # write response #------------------------------------- def write_response(self, response): ''' the writing phase is very fast so may not have to use coroutine ''' try: keep_alive = self.keep_alive self.transport.write(response.drain(keep_alive=keep_alive)) if keep_alive: self.refresh() else: self.transport.close() except AttributeError as e: print('AttributeError????', e) self.transport.close() except RuntimeError as e: print('RuntimeError????', e) self.transport.close() except Exception as e: print('Exception????', e) self.transport.close() async def stream_callback(self, response): ''' ''' try: keep_alive = self.keep_alive response.transport = self.transport await response.stream_output(self.request.version, keep_alive) if keep_alive: self.refresh() else: self.transport.close() except AttributeError as e: print('AttributeError????', e) self.transport.close() except RuntimeError as e: print('RuntimeError????', e) self.transport.close() except Exception as e: print('Exception????', e) self.transport.close() def refresh(self): ''' refresh the server state prepare for next incoming request ''' self.url = None self.header = {} self._request_handler_task = None self.stream_handler = None