def _parse_headers(self, data): idx = data.find(b("\r\n\r\n")) if idx < 0: # we don't have all headers return False # Split lines on \r\n keeping the \r\n on each line lines = [bytes_to_str(line) + "\r\n" for line in data[:idx].split(b("\r\n"))] # Parse headers into key/value pairs paying attention # to continuation lines. while len(lines): # Parse initial header name : value pair. curr = lines.pop(0) if curr.find(":") < 0: raise InvalidHeader("invalid line %s" % curr.strip()) name, value = curr.split(":", 1) name = name.rstrip(" \t").upper() if HEADER_RE.search(name): raise InvalidHeader("invalid header name %s" % name) name, value = name.strip(), [value.lstrip()] # Consume value continuation lines while len(lines) and lines[0].startswith((" ", "\t")): value.append(lines.pop(0)) value = ''.join(value).rstrip() # store new header value self._headers.add_header(name, value) # update WSGI environ key = 'HTTP_%s' % name.upper().replace('-', '_') self._environ[key] = value # detect now if body is sent by chunks. clen = self._headers.get('content-length') te = self._headers.get('transfer-encoding', '').lower() if clen is not None: try: self._clen_rest = self._clen = int(clen) except ValueError: pass else: self._chunked = (te == 'chunked') if not self._chunked: self._clen_rest = MAXSIZE # detect encoding and set decompress object encoding = self._headers.get('content-encoding') if self.decompress: if encoding == "gzip": self.__decompress_obj = zlib.decompressobj(16+zlib.MAX_WBITS) elif encoding == "deflate": self.__decompress_obj = zlib.decompressobj() rest = data[idx+4:] self._buf = [rest] self.__on_headers_complete = True return len(rest)
def _parse_headers(self, data): idx = data.find(b("\r\n\r\n")) if idx < 0: # we don't have all headers return False # Split lines on \r\n keeping the \r\n on each line lines = [(str(line, 'unicode_escape') if PY3 else line) + "\r\n" for line in data[:idx].split(b("\r\n"))] # Parse headers into key/value pairs paying attention # to continuation lines. while len(lines): # Parse initial header name : value pair. curr = lines.pop(0) if curr.find(":") < 0: raise InvalidHeader("invalid line %s" % curr.strip()) name, value = curr.split(":", 1) name = name.rstrip(" \t").upper() if HEADER_RE.search(name): raise InvalidHeader("invalid header name %s" % name) name, value = name.strip(), [value.lstrip()] # Consume value continuation lines while len(lines) and lines[0].startswith((" ", "\t")): value.append(lines.pop(0)) value = ''.join(value).rstrip() # store new header value self._headers.add_header(name, value) # update WSGI environ key = 'HTTP_%s' % name.upper().replace('-', '_') self._environ[key] = value # detect now if body is sent by chunks. clen = self._headers.get('content-length') te = self._headers.get('transfer-encoding', '').lower() if clen is not None: try: self._clen_rest = self._clen = int(clen) except ValueError: pass else: self._chunked = (te == 'chunked') if not self._chunked: self._clen_rest = MAXSIZE # detect encoding and set decompress object encoding = self._headers.get('content-encoding') if self.decompress: if encoding == "gzip": self.__decompress_obj = zlib.decompressobj(16 + zlib.MAX_WBITS) elif encoding == "deflate": self.__decompress_obj = zlib.decompressobj() rest = data[idx + 4:] self._buf = [rest] self.__on_headers_complete = True return len(rest)
def canonical(self): '''Canonicalize this url. This includes reordering parameters and args to have a consistent ordering''' self._query = b('&').join( sorted([q for q in self._query.split(b('&'))])) self._params = b(';').join( sorted([q for q in self._params.split(b(';'))])) return self
def canonical(self): '''Canonicalize this url. This includes reordering parameters and args to have a consistent ordering''' self._query = b('&').join( sorted([q for q in self._query.split(b('&'))]) ) self._params = b(';').join( sorted([q for q in self._params.split(b(';'))]) ) return self
def test_parsemsg(): s = b(":foo!bar@localhost NICK foobar") source, command, args = parsemsg(s) assert source == (u("foo"), u("bar"), u("localhost")) assert command == "NICK" assert args == [u("foobar")] s = b("") source, command, args = parsemsg(s) assert source == (None, None, None) assert command is None assert args == []
def test_request_body(webapp): connection = HTTPConnection(webapp.server.host, webapp.server.port) connection.connect() body = b("ä") connection.request("GET", "/request_body", body) response = connection.getresponse() assert response.status == 200 assert response.reason == "OK" s = response.read() assert s == b("ä") connection.close()
def _parse_chunk_size(self, data): idx = data.find(b("\r\n")) if idx < 0: return None, None line, rest_chunk = data[:idx], data[idx+2:] chunk_size = line.split(b(";"), 1)[0].strip() try: chunk_size = int(chunk_size, 16) except ValueError: raise InvalidChunkSize(chunk_size) if chunk_size == 0: self._parse_trailers(rest_chunk) return 0, None return chunk_size, rest_chunk
def _parse_chunk_size(self, data): idx = data.find(b("\r\n")) if idx < 0: return None, None line, rest_chunk = data[:idx], data[idx + 2:] chunk_size = line.split(b(";"), 1)[0].strip() try: chunk_size = int(chunk_size, 16) except ValueError: raise InvalidChunkSize(chunk_size) if chunk_size == 0: self._parse_trailers(rest_chunk) return 0, None return chunk_size, rest_chunk
def test_string(tmpdir): args = [sys.executable, exitcodeapp.__file__, "foobar"] p = Popen(args, env={"PYTHONPATH": ":".join(sys.path)}, stderr=PIPE) status = p.wait() assert status == 1 assert p.stderr.read() == b("foobar\n")
def _parse_body(self): if not self._chunked: body_part = b("").join(self._buf) self._clen_rest -= len(body_part) # maybe decompress if self.__decompress_obj is not None: body_part = self.__decompress_obj.decompress(body_part) self._partial_body = True self._body.append(body_part) self._buf = [] if self._clen_rest <= 0: self.__on_message_complete = True return else: data = b("").join(self._buf) try: size, rest = self._parse_chunk_size(data) except InvalidChunkSize as e: self.errno = INVALID_CHUNK self.errstr = "invalid chunk size [%s]" % str(e) return -1 if size == 0: return size if size is None or len(rest) < size: return None body_part, rest = rest[:size], rest[size:] if len(rest) < 2: self.errno = INVALID_CHUNK self.errstr = "chunk missing terminator [%s]" % data return -1 # maybe decompress if self.__decompress_obj is not None: body_part = self.__decompress_obj.decompress(body_part) self._partial_body = True self._body.append(body_part) self._buf = [rest[2:]] return len(rest)
def encode(self, encoding): """Return the url in an arbitrary encoding""" netloc = self._host if self._port: netloc += b(":") + bytes(self._port) result = urlunparse((self._scheme, netloc, self._path, self._params, self._query, self._fragment)) return result.decode("utf-8").encode(encoding)
def abspath(self): '''Clear out any '..' and excessive slashes from the path''' # Remove double forward-slashes from the path path = re.sub(b('\/{2,}'), b('/'), self._path) # With that done, go through and remove all the relative references unsplit = [] directory = False for part in path.split(b('/')): # If we encounter the parent directory, and there's # a segment to pop off, then we should pop it off. if part == b('..') and (not unsplit or unsplit.pop() is not None): directory = True elif part != b('.'): directory = False unsplit.append(part) else: directory = True # With all these pieces, assemble! if directory: # If the path ends with a period, then it refers to a directory, # not a file path unsplit.append(b('/')) self._path = b('/').join(unsplit) return self
def encode(self, encoding): '''Return the url in an arbitrary encoding''' netloc = self._host if self._port: netloc += (b(':') + bytes(self._port)) result = urlunparse((self._scheme, netloc, self._path, self._params, self._query, self._fragment)) return result.decode('utf-8').encode(encoding)
def encode(self, encoding): '''Return the url in an arbitrary encoding''' netloc = self._host if self._port: netloc += (b(':') + bytes(self._port)) result = urlunparse(( self._scheme, netloc, self._path, self._params, self._query, self._fragment )) return result.decode('utf-8').encode(encoding)
def test_bad_header(webapp): connection = HTTPConnection(webapp.server.host, webapp.server.port) connection.connect() connection.putrequest("GET", "/", "HTTP/1.1") connection.putheader("Connection", "close") connection._output(b("X-Foo")) # Bad Header connection.endheaders() response = connection.getresponse() assert response.status == 400 assert response.reason == "Bad Request" connection.close()
def test_request_headers(webapp): connection = HTTPConnection(webapp.server.host, webapp.server.port) connection.connect() body = b("") headers = {"A": "ä"} connection.request("GET", "/request_headers", body, headers) response = connection.getresponse() assert response.status == 200 assert response.reason == "OK" s = response.read() assert s == u"ä".encode('utf-8') connection.close()
def recv_body_into(self, barray): """ Receive the last chunk of the parsed bodyand store the data in a buffer rather than creating a new string. """ l = len(barray) body = b("").join(self._body) m = min(len(body), l) data, rest = body[:m], body[m:] barray[0:m] = data if not rest: self._body = [] self._partial_body = False else: self._body = [rest] return m
def _on_request(self, event, request, response): if self._path is not None and not request.path.startswith(self._path): return self._protocol_version = 13 headers = request.headers sec_key = headers.get("Sec-WebSocket-Key", "").encode("utf-8") subprotocols = headers.elements("Sec-WebSocket-Protocol") connection_tokens = [ s.strip() for s in headers.get("Connection", "").lower().split(",") ] try: if ("Host" not in headers or headers.get("Upgrade", "").lower() != "websocket" or "upgrade" not in connection_tokens or sec_key is None or len(base64.b64decode(sec_key)) != 16): return httperror(request, response, code=400) if headers.get("Sec-WebSocket-Version", "") != "13": response.headers["Sec-WebSocket-Version"] = "13" return httperror(request, response, code=400) # Generate accept header information msg = sec_key + b("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") hasher = hashlib.sha1() hasher.update(msg) accept = base64.b64encode(hasher.digest()) # Successful completion response.status = 101 response.close = False try: del response.headers["Content-Type"] except KeyError: pass response.headers["Upgrade"] = "WebSocket" response.headers["Connection"] = "Upgrade" response.headers["Sec-WebSocket-Accept"] = accept.decode() if subprotocols: response.headers[ "Sec-WebSocket-Protocol"] = self.select_subprotocol( subprotocols) codec = WebSocketCodec(request.sock, channel=self._wschannel) self._codecs[request.sock] = codec codec.register(self) return response finally: event.stop()
def test_response_headers(webapp): client = Client() client.start() client.fire( request( "GET", "http://%s:%s/response_headers" % (webapp.server.host, webapp.server.port))) while client.response is None: pass assert client.response.status == 200 assert client.response.reason == 'OK' s = client.response.read() a = client.response.headers.get('A') assert a == "ä" assert s == b("ä")
def abspath(self): """Clear out any '..' and excessive slashes from the path""" # Remove double forward-slashes from the path path = re.sub(b("\/{2,}"), b("/"), self._path) # With that done, go through and remove all the relative references unsplit = [] for part in path.split(b("/")): # If we encounter the parent directory, and there's # a segment to pop off, then we should pop it off. if part == b("..") and (not unsplit or unsplit.pop() is not None): pass elif part != b("."): unsplit.append(part) # With all these pieces, assemble! if self._path.endswith(b(".")): # If the path ends with a period, then it refers to a directory, # not a file path self._path = b("/").join(unsplit) + b("/") else: self._path = b("/").join(unsplit) return self
def _on_request(self, event, request, response): if self._path is not None and not request.path.startswith(self._path): return self._protocol_version = 13 headers = request.headers sec_key = headers.get("Sec-WebSocket-Key", "").encode("utf-8") connection_tokens = [s.strip() for s in headers.get("Connection", "").lower().split(",")] try: if ("Host" not in headers or headers.get("Upgrade", "").lower() != "websocket" or "upgrade" not in connection_tokens or sec_key is None or len(base64.b64decode(sec_key)) != 16): return httperror(request, response, code=400) if headers.get("Sec-WebSocket-Version", "") != "13": response.headers["Sec-WebSocket-Version"] = "13" return httperror(request, response, code=400) # Generate accept header information msg = sec_key + b("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") hasher = hashlib.sha1() hasher.update(msg) accept = base64.b64encode(hasher.digest()) # Successful completion response.status = 101 response.close = False try: del response.headers["Content-Type"] except KeyError: pass response.headers["Upgrade"] = "WebSocket" response.headers["Connection"] = "Upgrade" response.headers["Sec-WebSocket-Accept"] = accept.decode() codec = WebSocketCodec(request.sock, channel=self._wschannel) self._codecs[request.sock] = codec codec.register(self) return response finally: event.stop()
def test_response_headers(webapp): client = Client() client.start() client.fire( request( "GET", "http://%s:%s/response_headers" % ( webapp.server.host, webapp.server.port ) ) ) while client.response is None: pass assert client.response.status == 200 assert client.response.reason == 'OK' s = client.response.read() a = client.response.headers.get('A') assert a == "ä" assert s == b("ä")
def execute(self, data, length): # end of body can be passed manually by putting a length of 0 if length == 0: self.on_message_complete = True return length # start to parse nb_parsed = 0 while True: if not self.__on_firstline: idx = data.find(b("\r\n")) if idx < 0: self._buf.append(data) return len(data) else: self.__on_firstline = True self._buf.append(data[:idx]) first_line = b("").join(self._buf) if PY3: first_line = str(first_line, 'unicode_escape') nb_parsed = nb_parsed + idx + 2 rest = data[idx + 2:] data = b("") if self._parse_firstline(first_line): self._buf = [rest] else: return nb_parsed elif not self.__on_headers_complete: if data: self._buf.append(data) data = b("") try: to_parse = b("").join(self._buf) ret = self._parse_headers(to_parse) if not ret: return length nb_parsed = nb_parsed + (len(to_parse) - ret) except InvalidHeader as e: self.errno = INVALID_HEADER self.errstr = str(e) return nb_parsed elif not self.__on_message_complete: if not self.__on_message_begin: self.__on_message_begin = True if data: self._buf.append(data) data = b("") ret = self._parse_body() if ret is None: return length elif ret < 0: return ret elif ret == 0: self.__on_message_complete = True return length else: nb_parsed = max(length, ret) else: return 0
def __init__(self, scheme, host, port, path, params=b"", query=b"", fragment=b""): assert not type(port) is int self._scheme = scheme self._host = host self._port = port self._path = path or b("/") self._params = re.sub(b("^;+"), b(""), params) self._params = re.sub(b("^;|;$"), b(""), re.sub(b(";{2,}"), b(";"), self._params)) # Strip off extra leading ?'s self._query = query.lstrip(b("?")) self._query = re.sub(b("^&|&$"), b(""), re.sub(b("&{2,}"), b("&"), self._query)) self._fragment = fragment
def _parse_trailers(self, data): idx = data.find(b("\r\n\r\n")) if data[:2] == b("\r\n"): self._trailers = self._parse_headers(data[:idx])
def __init__(self, scheme, host, port, path, params=b"", query=b"", fragment=b""): assert not type(port) is int self._scheme = scheme self._host = host self._port = port self._path = path or b('/') self._params = re.sub(b('^;+'), b(''), params) self._params = re.sub( b('^;|;$'), b(''), re.sub(b(';{2,}'), b(';'), self._params) ) # Strip off extra leading ?'s self._query = query.lstrip(b('?')) self._query = re.sub( b('^&|&$'), b(''), re.sub(b('&{2,}'), b('&'), self._query) ) self._fragment = fragment
def recv_body(self): """ return last chunk of the parsed body""" body = b("").join(self._body) self._body = [] self._partial_body = False return body
"""Line Protocol This module implements the basic Line protocol. This module can be used in both server and client implementations. """ import re from circuits.core import BaseComponent, Event, handler from circuits.six import b LINESEP = re.compile(b("\r?\n")) def splitLines(s, buffer): """splitLines(s, buffer) -> lines, buffer Append s to buffer and find any new lines of text in the string splitting at the standard IRC delimiter CRLF. Any new lines found, return them as a list and the remaining buffer for further processing. """ lines = LINESEP.split(buffer + s) return lines[:-1], lines[-1] class line(Event): """line Event"""
assert False def test_args(webapp): args = ("1", "2", "3") kwargs = {"1": "one", "2": "two", "3": "three"} url = "%s/test_args/%s" % (webapp.server.http.base, "/".join(args)) data = urlencode(kwargs).encode('utf-8') f = urlopen(url, data) data = f.read().split(b"\n") assert eval(data[0]) == args assert eval(data[1]) == kwargs @pytest.mark.parametrize("data,expected", [ ((["1"], {}), b("a=1\nb=None")), ((["1", "2"], {}), b("a=1\nb=2")), ((["1"], { "b": "2" }), b("a=1\nb=2")), ]) def test_default_args(webapp, data, expected): args, kwargs = data url = u("{0:s}/test_default_args/{1:s}".format(webapp.server.http.base, u("/").join(args))) data = urlencode(kwargs).encode("utf-8") f = urlopen(url, data) assert f.read() == expected def test_redirect(webapp):
def __bytes__(self): return b(self.__str__())
def test_index(webapp): f = urlopen(webapp.server.http.base) s = f.read() assert s == b("Hello World!")
def execute(self, data, length): # end of body can be passed manually by putting a length of 0 if length == 0: self.on_message_complete = True return length # start to parse nb_parsed = 0 while True: if not self.__on_firstline: idx = data.find(b("\r\n")) if idx < 0: self._buf.append(data) return len(data) else: self.__on_firstline = True self._buf.append(data[:idx]) first_line = bytes_to_str(b("").join(self._buf)) nb_parsed = nb_parsed + idx + 2 rest = data[idx+2:] data = b("") if self._parse_firstline(first_line): self._buf = [rest] else: return nb_parsed elif not self.__on_headers_complete: if data: self._buf.append(data) data = b("") try: to_parse = b("").join(self._buf) ret = self._parse_headers(to_parse) if not ret: return length nb_parsed = nb_parsed + (len(to_parse) - ret) except InvalidHeader as e: self.errno = INVALID_HEADER self.errstr = str(e) return nb_parsed elif not self.__on_message_complete: if not self.__on_message_begin: self.__on_message_begin = True if data: self._buf.append(data) data = b("") ret = self._parse_body() if ret is None: return length elif ret < 0: return ret elif ret == 0: self.__on_message_complete = True return length else: nb_parsed = max(length, ret) else: return 0
assert False def test_args(webapp): args = ("1", "2", "3") kwargs = {"1": "one", "2": "two", "3": "three"} url = "%s/test_args/%s" % (webapp.server.http.base, "/".join(args)) data = urlencode(kwargs).encode('utf-8') f = urlopen(url, data) data = f.read().split(b"\n") assert eval(data[0]) == args assert eval(data[1]) == kwargs @pytest.mark.parametrize("data,expected", [ ((["1"], {}), b("a=1\nb=None")), ((["1", "2"], {}), b("a=1\nb=2")), ((["1"], {"b": "2"}), b("a=1\nb=2")), ]) def test_default_args(webapp, data, expected): args, kwargs = data url = u("{0:s}/test_default_args/{1:s}".format( webapp.server.http.base, u("/").join(args) )) data = urlencode(kwargs).encode("utf-8") f = urlopen(url, data) assert f.read() == expected def test_redirect(webapp):
def __init__(self, scheme, host, port, path, params=b"", query=b"", fragment=b""): assert not type(port) is int self._scheme = scheme self._host = host self._port = port self._path = path or b('/') self._params = re.sub(b('^;+'), b(''), params) self._params = re.sub(b('^;|;$'), b(''), re.sub(b(';{2,}'), b(';'), self._params)) # Strip off extra leading ?'s self._query = query.lstrip(b('?')) self._query = re.sub(b('^&|&$'), b(''), re.sub(b('&{2,}'), b('&'), self._query)) self._fragment = fragment