def _handle(self,
source,
dest,
to_backend,
on_between_handle,
data_sent=False):
buffer_size = self.option('buffer')
# Getting the HTTP query and sending it to the backend.
parser = HttpParser()
if not data_sent:
while not parser.is_message_complete():
data = self._get_data(source, buffer_size)
if not data:
return self._close_both(source, dest)
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
if self.option('overwrite_host_header'):
data = HOST_REPLACE.sub(
'\r\nHost: %s\r\n' % self.proxy.backend, data)
dest.sendall(data)
keep_alive_src = parser.should_keep_alive()
method = parser.get_method()
if on_between_handle():
# Getting the HTTP response and sending it back to the source.
parser = HttpParser()
while not (parser.is_message_complete() or
(method == 'HEAD' and parser.is_headers_complete())):
data = self._get_data(dest, buffer_size)
if not data:
return self._close_both(source, dest)
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
source.sendall(data)
keep_alive_dst = parser.should_keep_alive()
# do we close the client ?
if not keep_alive_src or not self.option('keep_alive'):
source.close()
source._closed = True
if (not keep_alive_dst or not self.option('reuse_socket')
or not self.option('keep_alive')):
dest.close()
dest._closed = True
else:
keep_alive_dst = False
return keep_alive_dst and self.option('keep_alive')
def __call__(self, addr):
'''Since an instance of HttpServer is passed to the Service
class (with appropriate request_handler established during
initialization), this __call__ method is what's actually
invoked by diesel.
'''
data = None
while True:
try:
h = HttpParser()
body = []
while True:
if data:
used = h.execute(data, len(data))
if h.is_headers_complete():
body.append(h.recv_body())
if h.is_message_complete():
data = data[used:]
break
data = receive()
env = h.get_wsgi_environ()
if 'HTTP_CONTENT_LENGTH' in env:
env['CONTENT_LENGTH'] = env.pop("HTTP_CONTENT_LENGTH")
if 'HTTP_CONTENT_TYPE' in env:
env['CONTENT_TYPE'] = env.pop("HTTP_CONTENT_TYPE")
env.update({
'wsgi.version': (1, 0),
'wsgi.url_scheme': 'http', # XXX incomplete
'wsgi.input': cStringIO.StringIO(''.join(body)),
'wsgi.errors': FileLikeErrorLogger(hlog),
'wsgi.multithread': False,
'wsgi.multiprocess': False,
'wsgi.run_once': False,
'REMOTE_ADDR': addr[0],
'SERVER_NAME': HOSTNAME,
'SERVER_PORT': str(self.port),
})
req = Request(env)
resp = self.request_handler(req)
if 'Server' not in resp.headers:
resp.headers.add('Server', SERVER_TAG)
if 'Date' not in resp.headers:
resp.headers.add(
'Date',
utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
assert resp, "HTTP request handler _must_ return a response"
self.send_response(resp, version=h.get_version())
if (not h.should_keep_alive()) or \
resp.headers.get('Connection', '').lower() == "close" or \
resp.headers.get('Content-Length') == None:
return
except ConnectionClosed:
break
def _handle(self, source, dest, to_backend, on_between_handle,
data_sent=False):
buffer_size = self.option('buffer')
# Getting the HTTP query and sending it to the backend.
parser = HttpParser()
if not data_sent:
while not parser.is_message_complete():
data = self._get_data(source, buffer_size)
if not data:
return self._close_both(source, dest)
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
if self.option('overwrite_host_header'):
data = HOST_REPLACE.sub('\r\nHost: %s\r\n'
% self.proxy.backend, data)
dest.sendall(data)
keep_alive_src = parser.should_keep_alive()
method = parser.get_method()
if on_between_handle():
# Getting the HTTP response and sending it back to the source.
parser = HttpParser()
while not (parser.is_message_complete() or
(method == 'HEAD' and parser.is_headers_complete())):
data = self._get_data(dest, buffer_size)
if not data:
return self._close_both(source, dest)
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
source.sendall(data)
keep_alive_dst = parser.should_keep_alive()
# do we close the client ?
if not keep_alive_src or not self.option('keep_alive'):
source.close()
source._closed = True
if (not keep_alive_dst or not self.option('reuse_socket') or not
self.option('keep_alive')):
dest.close()
dest._closed = True
else:
keep_alive_dst = False
return keep_alive_dst and self.option('keep_alive')
def __call__(self, addr):
'''Since an instance of HttpServer is passed to the Service
class (with appropriate request_handler established during
initialization), this __call__ method is what's actually
invoked by diesel.
'''
data = None
while True:
try:
h = HttpParser()
body = []
while True:
if data:
used = h.execute(data, len(data))
if h.is_headers_complete():
body.append(h.recv_body())
if h.is_message_complete():
data = data[used:]
break
data = receive()
env = h.get_wsgi_environ()
env.update({
'wsgi.version' : (1,0),
'wsgi.url_scheme' : 'http', # XXX incomplete
'wsgi.input' : cStringIO.StringIO(''.join(body)),
'wsgi.errors' : FileLikeErrorLogger(hlog),
'wsgi.multithread' : False,
'wsgi.multiprocess' : False,
'wsgi.run_once' : False,
})
req = Request(env)
resp = self.request_handler(req)
if 'Server' not in resp.headers:
resp.headers.add('Server', SERVER_TAG)
if 'Date' not in resp.headers:
resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
assert resp, "HTTP request handler _must_ return a response"
self.send_response(resp, version=h.get_version())
if (not h.should_keep_alive()) or \
resp.headers.get('Connection', '').lower() == "close" or \
resp.headers.get('Content-Length') == None:
return
except ConnectionClosed:
break
def _handle(self, source, dest, to_backend):
buffer_size = self.option('buffer')
# Getting the HTTP query and sending it to the backend.
parser = HttpParser()
while not parser.is_message_complete():
data = self._get_data(source, buffer_size)
if not data:
self._abort_handling(to_backend, dest)
return False
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
data = HOST_REPLACE.sub('\r\nHost: %s\r\n'
% self.proxy.backend, data)
dest.sendall(data)
# Getting the HTTP response and sending it back to the source.
parser = HttpParser()
while not parser.is_message_complete():
data = self._get_data(dest, buffer_size)
if not data:
self._abort_handling(to_backend, dest)
return False
nparsed = parser.execute(data, len(data))
assert nparsed == len(data)
source.sendall(data)
keep_alive = parser.should_keep_alive()
# do we close the client ?
if not keep_alive and not self.option('keep_alive'):
source.close()
source._closed = True
if not self.option('reuse_socket') and not self.option('keep_alive'):
dest.close()
dest._closed = True
# we're done
return keep_alive or self.option('keep_alive')
class HttpStream(object):
""" An HTTP parser providing higher-level access to a readable,
sequential io.RawIOBase object. You can use implementions of
http_parser.reader (IterReader, StringReader, SocketReader) or
create your own.
"""
def __init__(self, stream, kind=HTTP_BOTH, decompress=False):
""" constructor of HttpStream.
:attr stream: an io.RawIOBase object
:attr kind: Int, could be 0 to parseonly requests,
1 to parse only responses or 2 if we want to let
the parser detect the type.
"""
self.parser = HttpParser(kind=kind, decompress=decompress)
self.stream = stream
def _check_headers_complete(self):
if self.parser.is_headers_complete():
return
while True:
try:
next(self)
except StopIteration:
if self.parser.is_headers_complete():
return
raise NoMoreData("Can't parse headers")
if self.parser.is_headers_complete():
return
def _wait_status_line(self, cond):
if self.parser.is_headers_complete():
return True
data = []
if not cond():
while True:
try:
d = next(self)
data.append(d)
except StopIteration:
if self.parser.is_headers_complete():
return True
raise BadStatusLine(b"".join(data))
if cond():
return True
return True
def _wait_on_url(self):
return self._wait_status_line(self.parser.get_url)
def _wait_on_status(self):
return self._wait_status_line(self.parser.get_status_code)
def url(self):
""" get full url of the request """
self._wait_on_url()
return self.parser.get_url()
def path(self):
""" get path of the request (url without query string and
fragment """
self._wait_on_url()
return self.parser.get_path()
def query_string(self):
""" get query string of the url """
self._wait_on_url()
return self.parser.get_query_string()
def fragment(self):
""" get fragment of the url """
self._wait_on_url()
return self.parser.get_fragment()
def version(self):
self._wait_on_status()
return self.parser.get_version()
def status_code(self):
""" get status code of a response as integer """
self._wait_on_status()
return self.parser.get_status_code()
def status(self):
""" return complete status with reason """
status_code = self.status_code()
reason = status_reasons.get(int(status_code), 'unknown')
return "%s %s" % (status_code, reason)
def method(self):
""" get HTTP method as string"""
self._wait_on_status()
return self.parser.get_method()
def headers(self):
""" get request/response headers, headers are returned in a
OrderedDict that allows you to get value using insensitive
keys."""
self._check_headers_complete()
return self.parser.get_headers()
def should_keep_alive(self):
""" return True if the connection should be kept alive
"""
self._check_headers_complete()
return self.parser.should_keep_alive()
def is_chunked(self):
""" return True if Transfer-Encoding header value is chunked"""
self._check_headers_complete()
return self.parser.is_chunked()
def wsgi_environ(self, initial=None):
""" get WSGI environ based on the current request.
:attr initial: dict, initial values to fill in environ.
"""
self._check_headers_complete()
return self.parser.get_wsgi_environ()
def body_file(self, buffering=None, binary=True, encoding=None,
errors=None, newline=None):
""" return the body as a buffered stream object. If binary is
true an io.BufferedReader will be returned, else an
io.TextIOWrapper.
"""
self._check_headers_complete()
if buffering is None:
buffering = -1
if buffering < 0:
buffering = DEFAULT_BUFFER_SIZE
raw = HttpBodyReader(self)
buf = BufferedReader(raw, buffering)
if binary:
return buf
text = TextIOWrapper(buf, encoding, errors, newline)
return text
def body_string(self, binary=True, encoding=None, errors=None,
newline=None):
""" return body as string """
return self.body_file(binary=binary, encoding=encoding,
newline=newline).read()
def __iter__(self):
return self
def __next__(self):
if self.parser.is_message_complete():
raise StopIteration
# fetch data
b = bytearray(DEFAULT_BUFFER_SIZE)
recved = self.stream.readinto(b)
if recved is None:
raise NoMoreData("no more data")
del b[recved:]
to_parse = bytes(b)
# parse data
nparsed = self.parser.execute(to_parse, recved)
if nparsed != recved and not self.parser.is_message_complete():
raise ParserError("nparsed != recved (%s != %s) [%s]" % (nparsed,
recved, bytes_to_str(to_parse)))
if recved == 0:
raise StopIteration
return to_parse
next = __next__
def __call__(self, addr):
'''Since an instance of HttpServer is passed to the Service
class (with appropriate request_handler established during
initialization), this __call__ method is what's actually
invoked by diesel.
'''
data = None
while True:
try:
h = HttpParser()
body = []
while True:
if data:
used = h.execute(data, len(data))
if h.is_headers_complete():
body.append(h.recv_body())
if h.is_message_complete():
data = data[used:]
break
data = receive()
env = h.get_wsgi_environ()
if 'HTTP_CONTENT_LENGTH' in env:
env['CONTENT_LENGTH'] = env.pop("HTTP_CONTENT_LENGTH")
if 'HTTP_CONTENT_TYPE' in env:
env['CONTENT_TYPE'] = env.pop("HTTP_CONTENT_TYPE")
env.update({
'wsgi.version' : (1,0),
'wsgi.url_scheme' : 'http', # XXX incomplete
'wsgi.input' : cStringIO.StringIO(''.join(body)),
'wsgi.errors' : FileLikeErrorLogger(hlog),
'wsgi.multithread' : False,
'wsgi.multiprocess' : False,
'wsgi.run_once' : False,
'REMOTE_ADDR' : addr[0],
'SERVER_NAME' : HOSTNAME,
'SERVER_PORT': str(self.port),
})
req = Request(env)
if req.headers.get('Connection', '').lower() == 'upgrade':
req.data = data
resp = self.request_handler(req)
if 'Server' not in resp.headers:
resp.headers.add('Server', SERVER_TAG)
if 'Date' not in resp.headers:
resp.headers.add('Date', utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
assert resp, "HTTP request handler _must_ return a response"
self.send_response(resp, version=h.get_version())
if (not h.should_keep_alive()) or \
resp.headers.get('Connection', '').lower() == "close" or \
resp.headers.get('Content-Length') == None:
return
# Switching Protocols
if resp.status_code == 101 and hasattr(resp, 'new_protocol'):
resp.new_protocol(req)
break
except ConnectionClosed:
break
class HttpStream(object):
""" An HTTP parser providing higher-level access to a readable,
sequential io.RawIOBase object. You can use implementions of
http_parser.reader (IterReader, StringReader, SocketReader) or
create your own.
"""
def __init__(self, stream, kind=HTTP_BOTH, decompress=False):
""" constructor of HttpStream.
:attr stream: an io.RawIOBase object
:attr kind: Int, could be 0 to parseonly requests,
1 to parse only responses or 2 if we want to let
the parser detect the type.
"""
self.parser = HttpParser(kind=kind, decompress=decompress)
self.stream = stream
def _check_headers_complete(self):
if self.parser.is_headers_complete():
return
while True:
try:
data = self.next()
except StopIteration:
if self.parser.is_headers_complete():
return
raise NoMoreData()
if self.parser.is_headers_complete():
return
def url(self):
""" get full url of the request """
self._check_headers_complete()
return self.parser.get_url()
def path(self):
""" get path of the request (url without query string and
fragment """
self._check_headers_complete()
return self.parser.get_path()
def query_string(self):
""" get query string of the url """
self._check_headers_complete()
return self.parser.get_query_string()
def fragment(self):
""" get fragment of the url """
self._check_headers_complete()
return self.parser.get_fragment()
def version(self):
self._check_headers_complete()
return self.parser.get_version()
def status_code(self):
""" get status code of a response as integer """
self._check_headers_complete()
return self.parser.get_status_code()
def status(self):
""" return complete status with reason """
status_code = self.status_code()
reason = status_reasons.get(int(status_code), 'unknown')
return "%s %s" % (status_code, reason)
def method(self):
""" get HTTP method as string"""
self._check_headers_complete()
return self.parser.get_method()
def headers(self):
""" get request/response headers, headers are returned in a
OrderedDict that allows you to get value using insensitive
keys."""
self._check_headers_complete()
return self.parser.get_headers()
def should_keep_alive(self):
""" return True if the connection should be kept alive
"""
self._check_headers_complete()
return self.parser.should_keep_alive()
def is_chunked(self):
""" return True if Transfer-Encoding header value is chunked"""
self._check_headers_complete()
return self.parser.is_chunked()
def wsgi_environ(self, initial=None):
""" get WSGI environ based on the current request.
:attr initial: dict, initial values to fill in environ.
"""
self._check_headers_complete()
return self.parser.get_wsgi_environ()
def body_file(self,
buffering=None,
binary=True,
encoding=None,
errors=None,
newline=None):
""" return the body as a buffered stream object. If binary is
true an io.BufferedReader will be returned, else an
io.TextIOWrapper.
"""
self._check_headers_complete()
if buffering is None:
buffering = -1
if buffering < 0:
buffering = DEFAULT_BUFFER_SIZE
raw = HttpBodyReader(self)
buffer = BufferedReader(raw, buffering)
if binary:
return buffer
text = TextIOWrapper(buffer, encoding, errors, newline)
return text
def body_string(self,
binary=True,
encoding=None,
errors=None,
newline=None):
""" return body as string """
return self.body_file(binary=binary,
encoding=encoding,
newline=newline).read()
def __iter__(self):
return self
def next(self):
if self.parser.is_message_complete():
raise StopIteration
# fetch data
b = bytearray(DEFAULT_BUFFER_SIZE)
recved = self.stream.readinto(b)
if recved is None:
raise NoMoreData("no more data")
del b[recved:]
# parse data
nparsed = self.parser.execute(bytes(b), recved)
if nparsed != recved and not self.parser.is_message_complete():
raise ParserError("nparsed != recved")
if recved == 0:
raise StopIteration
return bytes(b)
def __call__(self, addr):
"""Since an instance of HttpServer is passed to the Service
class (with appropriate request_handler established during
initialization), this __call__ method is what's actually
invoked by diesel.
"""
data = None
while True:
try:
h = HttpParser()
body = []
while True:
if data:
used = h.execute(data, len(data))
if h.is_headers_complete():
body.append(h.recv_body())
if h.is_message_complete():
data = data[used:]
break
data = receive()
env = h.get_wsgi_environ()
if "HTTP_CONTENT_LENGTH" in env:
env["CONTENT_LENGTH"] = env.pop("HTTP_CONTENT_LENGTH")
if "HTTP_CONTENT_TYPE" in env:
env["CONTENT_TYPE"] = env.pop("HTTP_CONTENT_TYPE")
env.update(
{
"wsgi.version": (1, 0),
"wsgi.url_scheme": "http", # XXX incomplete
"wsgi.input": cStringIO.StringIO("".join(body)),
"wsgi.errors": FileLikeErrorLogger(hlog),
"wsgi.multithread": False,
"wsgi.multiprocess": False,
"wsgi.run_once": False,
"REMOTE_ADDR": addr[0],
"SERVER_NAME": HOSTNAME,
"SERVER_PORT": str(self.port),
}
)
req = Request(env)
if req.headers.get("Connection", "").lower() == "upgrade":
req.data = data
resp = self.request_handler(req)
if "Server" not in resp.headers:
resp.headers.add("Server", SERVER_TAG)
if "Date" not in resp.headers:
resp.headers.add("Date", utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
assert resp, "HTTP request handler _must_ return a response"
self.send_response(resp, version=h.get_version())
if (
(not h.should_keep_alive())
or resp.headers.get("Connection", "").lower() == "close"
or resp.headers.get("Content-Length") == None
):
return
except ConnectionClosed:
break
def handle_request(sock, parser, preread):
logging.debug("handle request")
if parser:
assert parser.is_headers_complete()
headers = parser.get_headers()
content_length = int(headers["Content-Length"]) if headers.has_key("Content-Length") else 0
assert content_length >= len(preread)
if content_length:
if preread:
nparsed = parser.execute(preread, len(preread))
assert nparsed == len(preread)
content_length -= len(preread)
while content_length:
data = sock.recv(content_length)
if not data:
logging.warn("client sock closed")
return False
recved = len(data)
content_length -= recved
nparsed = parser.execute(data, recved)
assert nparsed == recved
if parser.is_message_complete():
break
else:
parser = HttpParser()
while True:
logging.debug("recv........")
data = sock.recv(64 * 1024)
if not data:
logging.warn("client sock closed")
return False
recved = len(data)
nparsed = parser.execute(data, recved)
assert nparsed == recved
if parser.is_message_complete():
break
obj = None
if parser.get_path() == "/upload":
obj = handle_upload(sock, parser)
elif parser.get_path() == "/sync_upload":
obj = handle_sync_upload(sock, parser)
elif parser.get_path() == "/download":
obj = handle_download(sock, parser)
elif parser.get_path() == "/sync":
obj = handle_sync(sock, parser)
elif parser.get_path() == "/ping":
obj = handle_ping(sock, parser)
elif parser.get_path() == "/info":
obj = handle_info(sock, parser)
else:
logging.debug("unknown request path:%s", parser.get_path())
if obj is None:
sock.send("HTTP/1.1 404 Not Found\r\n")
sock.send("Content-Length: 0\r\n")
if keepalived:
sock.send("Connection: keep-alive\r\n")
else:
sock.send("Connection: close\r\n")
sock.send("\r\n")
return False
if not isinstance(obj, bool):
resp = json.dumps(obj)
keepalived = parser.should_keep_alive()
sock.send("HTTP/1.1 200 OK\r\n")
sock.send("Content-Type: application/json\r\n")
sock.send("Content-Length: %d\r\n" % len(resp))
if keepalived:
sock.send("Connection: keep-alive\r\n")
else:
sock.send("Connection: close\r\n")
sock.send("\r\n")
sock.send(resp)
return bool(keepalived)
else:
return obj
