def __init__(self, conn, handle_request): self.conn = conn self.server_loop = conn.server_loop self.max_header_line_size = self.server_loop.opts.max_header_line_size self.scheme = 'http' if self.server_loop.ssl_context is None else 'https' self.inheaders = MultiDict() self.outheaders = MultiDict() self.handle_request = handle_request self.request_line = None self.path = () self.qs = MultiDict() """When True, the request has been parsed and is ready to begin generating the response. When False, signals the calling Connection that the response should not be generated and the connection should close, immediately after parsing the request.""" self.ready = False """Signals the calling Connection that the request should close. This does not imply an error! The client and/or server may each request that the connection be closed, after the response.""" self.close_connection = False self.started_request = False self.reponse_protocol = HTTP1 self.status_code = None self.sent_headers = False self.request_content_length = 0 self.chunked_read = False
def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if path is None: raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component") if b'#' in path: raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join( unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query
def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if b'#' in path: raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query
def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() return self.simple_response(httplib.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( self.method, self.path, self.query, inheaders, request_body_file, outheaders, self.response_protocol, self.static_cache, self.opts, self.remote_addr, self.remote_port, self.translator_cache, self.tdir ) self.queue_job(self.run_request_handler, data)
def parse_request_line(self, buf, event, first=False): # {{{ line = self.readline(buf) if line is None: return if line == b'\r\n': # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] scheme, authority, path = parse_request_uri(uri) if b'#' in path: return self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: self.scheme = scheme.decode('ascii') except ValueError: return self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme') qs = b'' if b'?' in path: path, qs = path.split(b'?', 1) try: self.query = MultiDict.create_from_query_string(qs) except Exception: return self.simple_response(httplib.BAD_REQUEST, 'Unparseable query string') try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: return self.simple_response(httplib.BAD_REQUEST, as_unicode(e)) self.path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE self.request_line = line.rstrip() self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
def __init__(self): self.hdict = MultiDict() self.lines = [] self.finished = False
class HTTPHeaderParser(object): ''' Parse HTTP headers. Use this class by repeatedly calling the created object with a single line at a time and checking the finished attribute. Can raise ValueError for malformed headers, in which case you should probably return BAD_REQUEST. Headers which are repeated are folded together using a comma if their specification so dictates. ''' __slots__ = ('hdict', 'lines', 'finished') def __init__(self): self.hdict = MultiDict() self.lines = [] self.finished = False def push(self, *lines): for line in lines: self(line) def __call__(self, line): 'Process a single line' def safe_decode(hname, value): try: return value.decode('utf-8') except UnicodeDecodeError: if hname in decoded_headers: raise return value def commit(): if not self.lines: return line = b' '.join(self.lines) del self.lines[:] k, v = line.partition(b':')[::2] key = normalize_header_name(k.strip().decode('ascii')) val = safe_decode(key, v.strip()) if not key or not val: raise ValueError('Malformed header line: %s' % reprlib.repr(line)) if key in comma_separated_headers: existing = self.hdict.pop(key) if existing is not None: val = existing + ', ' + val self.hdict[key] = val if self.finished: raise ValueError('Header block already terminated') if line == b'\r\n': # Normal end of headers commit() self.finished = True return if line and line[0] in b' \t': # It's a continuation line. if not self.lines: raise ValueError('Orphaned continuation line') self.lines.append(line.lstrip()) else: commit() self.lines.append(line)
def parse_request_line(self, buf, event, first=False): # {{{ line = self.readline(buf) if line is None: return if line == b'\r\n': # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) return self.simple_response( httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] scheme, authority, path = parse_request_uri(uri) if b'#' in path: return self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: self.scheme = scheme.decode('ascii') except ValueError: return self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme') qs = b'' if b'?' in path: path, qs = path.split(b'?', 1) try: self.query = MultiDict.create_from_query_string(qs) except Exception: return self.simple_response(httplib.BAD_REQUEST, 'Unparseable query string') try: path = '%2F'.join( unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: return self.simple_response(httplib.BAD_REQUEST, as_unicode(e)) self.path = tuple( filter(None, (x.replace('%2F', '/') for x in path.split('/')))) self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE self.request_line = line.rstrip() self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() return self.simple_response(httplib.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( self.method, self.path, self.query, inheaders, request_body_file, outheaders, self.response_protocol, self.static_cache, self.opts, self.remote_addr, self.remote_port ) try: output = self.request_handler(data) except HTTP404 as e: return self.simple_response(httplib.NOT_FOUND, msg=e.message or '', close_after_response=False) output = self.finalize_output(output, data, self.method is HTTP1) if output is None: return outheaders.set('Date', http_date(), replace_all=True) outheaders.set('Server', 'calibre %s' % __version__, replace_all=True) keep_alive = not self.close_after_response and self.opts.timeout > 0 if keep_alive: outheaders.set('Keep-Alive', 'timeout=%d' % int(self.opts.timeout)) if 'Connection' not in outheaders: if self.response_protocol is HTTP11: if self.close_after_response: outheaders.set('Connection', 'close') else: if not self.close_after_response: outheaders.set('Connection', 'Keep-Alive') ct = outheaders.get('Content-Type', '') if ct.startswith('text/') and 'charset=' not in ct: outheaders.set('Content-Type', ct + '; charset=UTF-8') buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] for header, value in sorted(outheaders.iteritems(), key=itemgetter(0)): buf.append('%s: %s' % (header, value)) buf.append('') self.response_ready(BytesIO(b''.join((x + '\r\n').encode('ascii') for x in buf)), output=output)
def read_request_line(self): request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size) # Set started_request to True so http_communicate() knows to send 408 # from here on out. self.started_request = True if not request_line: return False if request_line == b'\r\n': # RFC 2616 sec 4.1: "...if the server is reading the protocol # stream at the beginning of a message and receives a CRLF # first, it should ignore the CRLF." # But only ignore one leading line! else we enable a DoS. request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size) if not request_line: return False if not request_line.endswith(b'\r\n'): self.simple_response( httplib.BAD_REQUEST, "HTTP requires CRLF terminators") return False self.request_line = request_line try: method, uri, req_protocol = request_line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii') except (ValueError, IndexError): self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") return False try: self.request_protocol = protocol_map[rp] except KeyError: self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) return False scheme, authority, path = parse_request_uri(uri) if b'#' in path: self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") return False if scheme: try: self.scheme = scheme.decode('ascii') except ValueError: self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme') return False qs = b'' if b'?' in path: path, qs = path.split(b'?', 1) try: self.qs = MultiDict.create_from_query_string(qs) except Exception: self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line", 'Unparseable query string') return False try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: self.simple_response(httplib.BAD_REQUEST, as_unicode(e)) return False self.path = tuple(x.replace('%2F', '/') for x in path.split('/')) self.response_protocol = protocol_map[min((1, 1), rp)] return True
class HTTPPair(object): ''' Represents a HTTP request/response pair ''' def __init__(self, conn, handle_request): self.conn = conn self.server_loop = conn.server_loop self.max_header_line_size = self.server_loop.opts.max_header_line_size self.scheme = 'http' if self.server_loop.ssl_context is None else 'https' self.inheaders = MultiDict() self.outheaders = MultiDict() self.handle_request = handle_request self.request_line = None self.path = () self.qs = MultiDict() """When True, the request has been parsed and is ready to begin generating the response. When False, signals the calling Connection that the response should not be generated and the connection should close, immediately after parsing the request.""" self.ready = False """Signals the calling Connection that the request should close. This does not imply an error! The client and/or server may each request that the connection be closed, after the response.""" self.close_connection = False self.started_request = False self.reponse_protocol = HTTP1 self.status_code = None self.sent_headers = False self.request_content_length = 0 self.chunked_read = False def parse_request(self): """Parse the next HTTP request start-line and message-headers.""" try: if not self.read_request_line(): return except MaxSizeExceeded: self.simple_response( httplib.REQUEST_URI_TOO_LONG, "The Request-URI sent with the request exceeds the maximum allowed bytes.") return try: if not self.read_request_headers(): return except MaxSizeExceeded: self.simple_response( httplib.REQUEST_ENTITY_TOO_LARGE, "The headers sent with the request exceed the maximum allowed bytes.") return self.ready = True def read_request_line(self): request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size) # Set started_request to True so http_communicate() knows to send 408 # from here on out. self.started_request = True if not request_line: return False if request_line == b'\r\n': # RFC 2616 sec 4.1: "...if the server is reading the protocol # stream at the beginning of a message and receives a CRLF # first, it should ignore the CRLF." # But only ignore one leading line! else we enable a DoS. request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size) if not request_line: return False if not request_line.endswith(b'\r\n'): self.simple_response( httplib.BAD_REQUEST, "HTTP requires CRLF terminators") return False self.request_line = request_line try: method, uri, req_protocol = request_line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii') except (ValueError, IndexError): self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") return False try: self.request_protocol = protocol_map[rp] except KeyError: self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) return False scheme, authority, path = parse_request_uri(uri) if b'#' in path: self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") return False if scheme: try: self.scheme = scheme.decode('ascii') except ValueError: self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme') return False qs = b'' if b'?' in path: path, qs = path.split(b'?', 1) try: self.qs = MultiDict.create_from_query_string(qs) except Exception: self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line", 'Unparseable query string') return False try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: self.simple_response(httplib.BAD_REQUEST, as_unicode(e)) return False self.path = tuple(x.replace('%2F', '/') for x in path.split('/')) self.response_protocol = protocol_map[min((1, 1), rp)] return True def read_request_headers(self): # then all the http headers try: self.inheaders = read_headers(partial(self.conn.socket_file.readline, maxsize=self.max_header_line_size)) self.request_content_length = int(self.inheaders.get('Content-Length', 0)) except ValueError as e: self.simple_response(httplib.BAD_REQUEST, as_unicode(e)) return False if self.request_content_length > self.server_loop.opts.max_request_body_size: self.simple_response( httplib.REQUEST_ENTITY_TOO_LARGE, "The entity sent with the request exceeds the maximum " "allowed bytes (%d)." % self.server_loop.opts.max_request_body_size) return False # Persistent connection support if self.response_protocol is HTTP11: # Both server and client are HTTP/1.1 if self.inheaders.get("Connection", "") == "close": self.close_connection = True else: # Either the server or client (or both) are HTTP/1.0 if self.inheaders.get("Connection", "") != "Keep-Alive": self.close_connection = True # Transfer-Encoding support te = () if self.response_protocol is HTTP11: rte = self.inheaders.get("Transfer-Encoding") if rte: te = [x.strip().lower() for x in rte.split(",") if x.strip()] self.chunked_read = False if te: for enc in te: if enc == "chunked": self.chunked_read = True else: # Note that, even if we see "chunked", we must reject # if there is an extension we don't recognize. self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) self.close_connection = True return False if self.inheaders.get("Expect", '').lower() == "100-continue": # Don't use simple_response here, because it emits headers # we don't want. msg = HTTP11 + " 100 Continue\r\n\r\n" self.flushed_write(msg.encode('ascii')) return True def simple_response(self, status_code, msg=""): abort = status_code in (httplib.REQUEST_ENTITY_TOO_LARGE, httplib.REQUEST_URI_TOO_LONG) if abort: self.close_connection = True if self.reponse_protocol is HTTP1: # HTTP/1.0 has no 413/414 codes status_code = httplib.BAD_REQUEST msg = msg.encode('utf-8') buf = [ '%s %d %s' % (self.reponse_protocol, status_code, httplib.responses[status_code]), "Content-Length: %s" % len(msg), "Content-Type: text/plain; charset=UTF-8" ] if abort and self.reponse_protocol is HTTP11: buf.append("Connection: close") buf.append('') buf = [(x + '\r\n').encode('ascii') for x in buf] buf.append(msg) self.flushed_write(b''.join(buf)) def flushed_write(self, data): self.conn.socket_file.write(data) self.conn.socket_file.flush() def repr_for_log(self): return 'HTTPPair: %r\nPath:%r\nQuery:\n%s\nIn Headers:\n%s\nOut Headers:\n%s' % ( self.request_line, self.path, self.qs.pretty('\t'), self.inheaders.pretty('\t'), self.outheaders.pretty('\t') ) def generate_static_output(self, name, generator): return generate_static_output(self.server_loop.gso_cache, self.server_loop.gso_lock, name, generator) def response(self): if self.chunked_read: self.input_reader = ChunkedReader(self.conn.socket_file, self.server_loop.opts.max_request_body_size) else: self.input_reader = FixedSizeReader(self.conn.socket_file, self.request_content_length) output = self.handle_request(self) if self.status_code is None: raise Exception('Request handler did not set status_code') # Read and discard any remaining body from the HTTP request self.input_reader.read() self.status_code, output = finalize_output(output, self.inheaders, self.outheaders, self.status_code) self.send_headers() if self.method != 'HEAD': output.commit(self.conn.socket_file) self.conn.socket_file.flush() def send_headers(self): self.sent_headers = True