def __init__(self, data):
parser = HttpResponseParser(self)
self.headers = {}
self.body = b""
self.data = data
parser.feed_data(self.data)
self.status_code = parser.get_status_code()
def __init__(self):
self.reader = None
self.writer = None
self.http_parser = HttpResponseParser(self)
self.response = None
self._data = b''
self._done = False
async def send(self, method, host, path, headers=None, data=None):
"""
Send the request (usually called by the Session object).
"""
self.__keep_alive = None
self.__gzipped = None
self.headers_complete = False
self.contains_body = False
self.body_done = True
self.__text = b''
self.content = b''
self.__headers = {}
self.__header_dict = None
self.parser = HttpResponseParser(self)
self.method = method
self.request_headers = {
b"Host": host,
b"User-Agent": b"uvloop http client"
}
if data:
self.request_headers[b"Content-Length"] = str(len(data)).encode()
if headers:
self.request_headers.update(headers)
request = b"\r\n".join(
[ b" ".join([method, path, b"HTTP/1.1"]) ] +
[ b": ".join(header) for header in self.request_headers.items() ] +
[ b"\r\n" ]
)
if data:
request += data
await self.connection.send(request)
try:
await self.fetch()
except EOFError as e:
if self.headers[b'transfer-encoding'] \
or self.headers[b'content-encoding'] or self.headers[b'content-length']:
raise e
self.status_code = self.parser.get_status_code()
def __init__(self, parseType="request"):
# create abstract parser
self.parser = AbstractParser()
# determine httptools parser
if parseType.lower() == "request":
self.parsed = HttpRequestParser(self.parser)
else:
self.parsed = HttpResponseParser(self.parser)
# set null url info
self.url = None
self.host = None
self.schema = None
self.port = None
self.path = None
self.params = {}
class ApricotParser(object):
''' Apricot Http Parser '''
def __init__(self, parseType="request"):
# create abstract parser
self.parser = AbstractParser()
# determine httptools parser
if parseType.lower() == "request":
self.parsed = HttpRequestParser(self.parser)
else:
self.parsed = HttpResponseParser(self.parser)
# set null url info
self.url = None
self.host = None
self.schema = None
self.port = None
self.path = None
self.params = {}
async def feed_async(self, data=b''):
self.feed(data)
def feed(self, data=b''):
''' feed data to parser '''
self.raw_data = data
self.parsed.feed_data(memoryview(data))
self.set_attributes() # set apricot attributes
def set_attributes(self):
''' set ApricotParser attributes '''
# check parser type
if isinstance(self.parsed, HttpRequestParser):
self.method = self.parsed.get_method().decode()
else:
self.status = self.parsed.get_status_code()
# set basic attr's
try:
self.headers = self.parser.headers
self.body = self.parser.body
self.http_ver = self.parsed.get_http_version()
self.keep_alive = self.parsed.should_keep_alive()
except Exception as e:
print("error in parser: " + str(e))
if self.keep_alive == None: self.keep_alive = False
# parse url and get info
self.get_url_info()
def get_url_info(self):
''' Parse url info '''
if 'Host' in self.headers:
try:
# make url
self.url = b'http://'
self.url += self.headers['Host'].encode()
# add path if request
if isinstance(self.parsed, HttpRequestParser):
path = self.raw_data.split(b'\r\n')[0].split()[1]
self.url += path
self.path = path
# parse url and get basic info
URL = parse_url(self.url)
self.schema = URL.schema.decode()
self.host = URL.host.decode()
self.port = URL.port
# get url query parameters if any
if URL.query is not None:
for param in URL.query.split(b'&'):
parts = param.split(b'=')
key = parts[0].decode()
value = b'='.join(parts[1:]).decode()
self.params[key] = value
# decode url from bytes to string
self.url = self.url.decode()
except:
pass
def __init__(self, data, protocol):
self.parser = HttpResponseParser(self)
self.headers = {}
self.body = b""
self.data = data
self.protocol = protocol
def receive(self):
http_response = HttpResonse(cookies=self.cookies,
encoding=self.encoding)
http_response_parser = HttpResponseParser(http_response)
conn = self.connection
# TODO, handle Maximum amount of incoming data to buffer
chucks = b''
while True:
chuck = yield from conn.readline()
chucks += chuck
if chuck == b'\r\n':
break
http_response_parser.feed_data(chucks)
self.status_code = http_response_parser.get_status_code()
headers = http_response.headers
self.headers = headers
# (protocol, status_code, ok), headers, cookies = parse_headers(chucks)
# self.headers = headers
# self.status_code = status_code
# self.cookies = cookies
# TODO, handle redirect
body = b''
if self.method.lower() == 'head': # HEAD
self.content = body
return None
nbytes = headers.get('Content-Length')
if nbytes:
nbytes = int(nbytes)
if nbytes:
body += yield from conn.read(nbytes)
else:
if headers.get('Transfer-Encoding') == 'chunked':
blocks = []
while True:
size_header = yield from conn.readline()
if not size_header:
# logging
break
parts = size_header.split(b';')
size = int(parts[0], 16)
if size:
block = yield from conn.read(size)
assert len(block) == size, (
'[Response.receive] [Transfer-Encoding]',
len(block), size)
blocks.append(block)
crlf = yield from conn.readline()
assert crlf == b'\r\n', repr(crlf)
if not size:
break
body += b''.join(blocks)
else:
# reading until EOF
pass
# body += yield from conn.read(-1)
if body and self.headers.get('Content-Encoding', '').lower() == 'gzip':
self.content = decode_gzip(body)
elif body and self.headers.get('Content-Encoding',
'').lower() == 'deflate':
self.content = decode_deflate(body)
else:
self.content = body
class HttpProtocol:
"""The protocol class constructs the incoming request and response."""
def __init__(self, sock: socket.socket = None):
self._sock = sock
self._bufsize = 2048
self._data_hooks = [] # type: List[Callable]
self._url_hooks = [] # type: List[Callable]
self._parser = None # type: Union[HttpRequestParser, HttpResponseParser]
self._is_done = False
self.msg = None # type: Union[Request, Response]
def add_url_hook(self, func: Callable):
"""Add hook to be executed during on_url()"""
self._url_hooks.append(func)
def add_data_hook(self, func: Callable):
"""Add hook to be executed during recv()"""
self._data_hooks.append(func)
def set_bufsize(self, bufsize: int):
"""Set the bytes size to read from the socket"""
self._bufsize = bufsize
def set_request_parser(self):
"""Set a new request parser with self as callback"""
self._parser = HttpRequestParser(self)
def set_response_parser(self):
"""Set a new response parser with self as callback"""
self._parser = HttpResponseParser(self)
def set_request(self, request: Request):
"""Set new request"""
self.msg = request
def set_response(self, response: Response):
"""Set new response"""
self.msg = response
def feed(self, data: bytes):
"""feed data to the underlying parser"""
for hook in self._data_hooks:
hook(data)
self._parser.feed_data(data)
def recv(self):
"""Receive all incoming http data on a socket and execute hooks"""
while not self._is_done:
data = self._sock.recv(self._bufsize)
self.feed(data)
# HTTPTOOLS CALLBACK METHODS
def on_message_begin(self):
"""Triggered on first bytes"""
def on_status(self, status: bytes):
"""Status integer callback"""
if self.msg is not None:
self.msg.status = self._parser.get_status_code()
def on_url(self, url: bytes):
"""Request message attribute will have url and method at this point"""
url = URL(url)
if self.msg is not None:
self.msg.url = url
self.msg.method = self._parser.get_method().decode()
for hook in self._url_hooks:
hook(url)
def on_header(self, name: bytes, value: bytes):
"""Alter request header as they come in if necessary"""
if self.msg is not None:
self.msg.headers[name.decode()] = value.decode()
def on_headers_complete(self):
"""End of headers trigger"""
def on_body(self, body: bytes):
"""Optionally access body stream"""
if self.msg is not None:
self.msg.body += body
def on_message_complete(self):
"""Trigger when message finishes"""
self._is_done = True
def on_chunk_header(self):
"""Chunked message header"""
def on_chunk_complete(self):
"""Chunked response end"""
def set_response_parser(self):
"""Set a new response parser with self as callback"""
self._parser = HttpResponseParser(self)
def set_request_parser(self):
"""Set a new request parser with self as callback"""
self._parser = HttpRequestParser(self)
class HttpProtocol:
"""The protocol class constructs the incoming request and response."""
def __init__(self, bufsize: int = 2048):
self._bufsize = bufsize
self._data_hooks = [] # type: List[Callable]
self._url_hooks = [] # type: List[Callable]
self._parser = None # type: PARSER_TYPE
self._is_done = False
self._body = bytes()
self.msg = None
def add_url_hook(self, func: Callable):
"""Add hook to be executed during on_url()"""
self._url_hooks.append(func)
def add_data_hook(self, func: Callable):
"""Add hook to be executed during recv()"""
self._data_hooks.append(func)
def set_bufsize(self, bufsize: int):
"""Set the bytes size to read from the socket"""
self._bufsize = bufsize
def set_request_parser(self):
"""Set a new request parser with self as callback"""
self._parser = HttpRequestParser(self)
def set_response_parser(self):
"""Set a new response parser with self as callback"""
self._parser = HttpResponseParser(self)
def set_request(self, request: dict):
"""Set new request"""
request['version'] = 'HTTP/1.1'
request['method'] = 'GET'
request['headers'] = {}
request['body'] = {}
request['url'] = {
'scheme': '',
'netloc': '',
'path': '',
'params': '',
'query': {},
'fragment': ''
}
self.msg = request
def set_response(self, response: dict):
"""Set new response"""
response['version'] = 'HTTP/1.1'
response['status'] = 200
response['reason'] = 'OK'
response['headers'] = {}
response['body'] = {}
self.msg = response
def feed(self, data: bytes):
"""feed data to the underlying parser"""
for hook in self._data_hooks:
hook(data)
self._parser.feed_data(data)
def recv(self, sock: socket.socket):
"""Receive all incoming http data on a socket and execute hooks"""
while not self._is_done:
data = sock.recv(self._bufsize)
self.feed(data)
# HTTPTOOLS CALLBACK METHODS
def on_message_begin(self):
"""Triggered on first bytes"""
def on_status(self, status: bytes):
"""Status integer callback"""
if self.msg is not None:
self.msg['status'] = self._parser.get_status_code()
self.msg['reason'] = status.decode()
def on_url(self, url: bytes):
"""Request message attribute will have url and method at this point"""
url = parse_url(url)
if self.msg is not None:
self.msg['url'] = url
self.msg['method'] = self._parser.get_method().decode()
for hook in self._url_hooks:
hook(url)
def on_header(self, name: bytes, value: bytes):
"""Alter request header as they come in if necessary"""
if self.msg is not None:
self.msg['headers'][name.decode().lower()] = value.decode()
def on_headers_complete(self):
"""End of headers trigger"""
def on_body(self, body: bytes):
"""Optionally access body stream"""
self._body += body
def on_message_complete(self):
"""Trigger when message finishes"""
if self.msg is not None:
self.msg['version'] = 'HTTP/' + self._parser.get_http_version()
if self.msg['headers'].get('content-encoding', '') == 'gzip':
self._body = gzip.decompress(self._body)
self.msg['body'] = load_json(self._body)
self._is_done = True
def on_chunk_header(self):
"""Chunked message header"""
def on_chunk_complete(self):
"""Chunked response end"""
class _HTTPClientConnection:
slots = ('reader', 'writer', 'http_parser', '_done', '_data')
def __init__(self):
self.reader = None
self.writer = None
self.http_parser = HttpResponseParser(self)
self.response = None
self._data = b''
self._done = False
async def connect(self, service, port, use_ssl):
for _ in range(3):
try:
self.reader, self.writer = await \
asyncio.open_connection(service, port, ssl=use_ssl)
return
except ConnectionRefusedError:
""" connection refused. Try again """
raise ConnectionRefusedError(
f'Connection refused to "{service}" on port {port}')
def send(self, method, path, headers, body):
self.writer.write(
f'{method.upper()} {path} HTTP/1.0\r\n'.encode('latin-1'))
for header, value in headers:
self.writer.write(f'{header}: {value}\r\n'.encode('latin-1'))
self.writer.write(b'\r\n')
if body:
self.writer.write(body)
async def get_response(self):
while True:
data = await self.reader.read(1064)
self.http_parser.feed_data(data)
if self._done:
return self.response
def close(self):
self.writer.close()
""" http parsing methods below """
def on_message_begin(self):
self.response = Response()
def on_header(self, name, value):
name = name.decode('latin-1')
value = value.decode()
if name == 'X-Correlation-ID':
self.response.correlation_id = value
elif name.lower() == 'content-type':
self.response.content_type = value
else:
self.response.headers[name] = value
def on_headers_complete(self):
self.response.status = HTTPStatus(self.http_parser.get_status_code())
def on_body(self, body):
self._data += body
def on_message_complete(self):
self.response.body = self._data
self._data = b''
self._done = True
class HTTPRequest:
"""
An HTTP request instantiated from a :class:`.Session`. HTTP requests are returned by the HTTP
session once they are sent and contain all information about the request and response.
"""
def __init__(self, connection):
self.connection = connection
async def send(self, method, host, path, headers=None, data=None):
"""
Send the request (usually called by the Session object).
"""
self.__keep_alive = None
self.__gzipped = None
self.headers_complete = False
self.contains_body = False
self.body_done = True
self.__text = b''
self.content = b''
self.__headers = {}
self.__header_dict = None
self.parser = HttpResponseParser(self)
self.method = method
self.request_headers = {
b"Host": host,
b"User-Agent": b"uvloop http client"
}
if data:
self.request_headers[b"Content-Length"] = str(len(data)).encode()
if headers:
self.request_headers.update(headers)
request = b"\r\n".join(
[ b" ".join([method, path, b"HTTP/1.1"]) ] +
[ b": ".join(header) for header in self.request_headers.items() ] +
[ b"\r\n" ]
)
if data:
request += data
await self.connection.send(request)
try:
await self.fetch()
except EOFError as e:
if self.headers[b'transfer-encoding'] \
or self.headers[b'content-encoding'] or self.headers[b'content-length']:
raise e
self.status_code = self.parser.get_status_code()
async def fetch(self):
# TODO: support streaming
while not self.headers_complete or not self.body_done:
data = await self.connection.read(65535)
if not data:
self.close()
raise EOFError()
self.parser.feed_data(data)
self.close()
def close(self):
"""
Closes the request, signalling that we're done with the request. The
connection is kept open and released back to the pool for re-use.
"""
if not self.keep_alive:
self.connection.close()
self.connection.release()
@property
def keep_alive(self):
if self.__keep_alive == None:
connection = self.headers[b'connection']
self.__keep_alive = not connection or connection != b'close'
return self.__keep_alive
@property
def gzipped(self):
"""
Return true if the response is gzipped.
"""
if self.__gzipped == None:
encoding = self.headers[b'content-encoding'] + self.headers[b'transfer-encoding']
self.__gzipped = b'gzip' in encoding or b'deflate' in encoding
return self.__gzipped
def json(self):
"""
Return the JSON decoded version of the body.
"""
# TODO: Possibly should use a better library.
return json.loads(self.text)
@property
def text(self):
"""
The string representation of the response body. It will be ungzipped
and encoded as a unicode string.
"""
if self.__text:
return self.__text
if self.gzipped:
self.__text = zlib.decompress(self.content, 16 + zlib.MAX_WBITS)
else:
self.__text = self.content
self.__text = self.__text.decode('utf-8')
return self.__text
@property
def headers(self):
"""
Return the headers from the request in a case-insensitive dictionary.
"""
if self.headers_complete and self.__header_dict:
return self.__header_dict
self.__header_dict = HeaderDict(self.__headers)
return self.__header_dict
def on_header(self, name, value):
self.__headers[name] = value
def on_body(self, body):
self.content += body
def on_headers_complete(self):
self.headers_complete = True
def on_chunk_complete(self):
self.body_done = True
def on_message_complete(self):
self.body_done = True
def on_message_begin(self):
if self.method != b"HEAD":
self.body_done = False