def read_get(self):
reader = SocketReader(self.sock)
resp = HttpStream(reader)
status_code = resp.status_code()
headers = dict(resp.headers())
body = resp.body_string()
return {"status_code": status_code, "headers": headers, "body": body}
def create(server, client, config):
client.setblocking(1)
request = HttpStream(SocketReader(client))
body_stream = request.body_file()
environ = request.wsgi_environ()
'''
the environ produced by Python and C parser is not consistent
we have to set some missing values here
'''
host, port = server.getsockname()
environ.update({
'SERVER_NAME': host,
'SERVER_PORT': str(port),
'wsgi.version': (1, 0),
'wsgi.url_scheme': utils.guess_scheme(environ),
'wsgi.input': body_stream,
'wsgi.multithread': False,
'wsgi.multiprocess': config['workers'] > 1,
'wsgi.run_once': False,
'wsgi.file_wrapper': utils.FileWrapper,
})
response = Response(request, client)
return environ, response
def test_parse_from_real_socket():
# would fail on python2.6 before the recv_into hack
sock, sink = socket.socketpair()
sink.send(complete_request)
reader = SocketReader(sock)
stream = HttpStream(reader)
assert stream.headers()
def test_parse_from_real_socket():
# would fail on python2.6 before the recv_into hack
sock, sink = socket.socketpair()
sink.send(complete_request)
reader = SocketReader(sock)
stream = HttpStream(reader)
assert stream.headers()
def http_serve(config, stdin, stdout):
# Re-open the stdin and stdout file descriptors using raw unbuffered i/o.
stdin = io.FileIO(stdin.fileno())
stdout = io.FileIO(stdout.fileno(), 'w')
# Parse an HTTP request on stdin.
parser = HttpStream(stdin, kind=HTTP_REQUEST)
wsgi_env = parser.wsgi_environ()
request_method = wsgi_env['REQUEST_METHOD']
request_path = wsgi_env['PATH_INFO']
request_length = wsgi_env.get('HTTP_CONTENT_LENGTH', 0)
request_body = parser.body_file()
try:
success = False
if request_method != 'GET':
raise HttpError(405)
if request_path.endswith('/'):
http_respond(stdout,
301,
location=request_path.rstrip('/'),
body='')
else:
path = resolve_path(request_path)
mimetype = guess_type(path)[0]
try:
output = render(config, request_path)
http_respond(stdout, 200, mimetype=mimetype, body=output)
except TemplateNotFound as e:
# Retry as static file.
full_static_path = config.static_root + path
with file(full_static_path) as static_file:
http_respond(stdout,
200,
mimetype=mimetype,
body=static_file)
success = True
except InvalidRequestPath as e:
http_respond(stdout, 400)
except HttpError as e:
http_respond(stdout, e.status)
except IOError as e:
if e.errno == errno.ENOENT:
http_respond(stdout, 404)
elif e.errno == errno.EACCES:
http_respond(stdout, 403)
else:
http_respond(stdout, 500)
except:
http_respond(stdout, 500)
return success
def send_downstream(self, p, rhost, rport, method, url, upsock):
with self.connect_downstream(rhost, rport) as downsock:
request = '{method} {url} HTTP/{version[0]}.{version[1]}\r\n{headers}\r\n\r\n'.format(method=method.decode(), url=url, version=p.version(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in p.headers().items()]))
self.logger.debug('sending request {0!r}'.format(request))
downsock.send(request.encode())
downstream = HttpStream(SocketReader(downsock))
self.logger.debug('response: header={0}'.format(downstream.headers()))
upsock.send('HTTP/{version[0]}.{version[1]} {code} {status}\r\n{headers}\r\n\r\n'.format(version=downstream.version(), code=downstream.status_code(), status=downstream.status(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in downstream.headers().items()])).encode())
upsock.send(downstream.body_string())
def handle(self, sock, addr):
p = HttpStream(SocketReader(sock))
path = p.path()
data = "hello world"
sock.send("".join([
"HTTP/1.1 200 OK\r\n", "Content-Type: text/html\r\n",
"Content-Length:" + str(len(data)) + "\r\n",
"Connection: close\r\n\r\n", data
]))
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(('gunicorn.org', 80))
s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n")
p = HttpStream(SocketReader(s))
print(p.headers())
print(p.body_file().read())
finally:
s.close()
def handle(self, sock, addr):
p = HttpStream(SocketReader(sock))
path = p.path()
data = "welcome wold"
sock.send("".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: text/html\r\n",
"Content-Length:" + str(len(data)) + "\r\n",
"Connection: close\r\n\r\n",
data]))
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(('gunicorn.org', 80))
s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n")
p = HttpStream(SocketReader(s))
print p.headers()
print p.body_file().read()
finally:
s.close()
def recv(self):
try:
stream = HttpStream(self.reader, kind=HTTP_RESPONSE, parser_class=HttpParser, decompress=True)
status = stream.status_code()
version = stream.version()
fields = stream.headers()
content = stream.body_file()
self.version = 'HTTP/%s.%s' % version
return status, fields, content
except NoMoreData:
pass
def recPOSTheader(HOST, request_headerPOST):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
try:
s.sendall(request_headerPOST.encode('utf-8'))
except:
print("An error has occured while trying to send the POST request")
dataReceived = SocketReader(s)
parsed = HttpStream(dataReceived)
header = parsed.headers()
return header
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(('baike.baidu.com', 80))
s.send(b("GET /view/262.htm HTTP/1.1\r\nHost: baike.baidu.com\r\n\r\n"))
p = HttpStream(SocketReader(s), decompress=True)
print(p.headers())
print(p.body_file().read())
finally:
s.close()
def proxy_connection(self, request, req_addr, is_ssl):
""" do the proxy connection """
proxy_settings = os.environ.get('%s_proxy' % request.parsed_url.scheme)
if proxy_settings and proxy_settings is not None:
request.is_proxied = True
proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings)
addr = parse_netloc(urlparse.urlparse(proxy_settings))
if is_ssl:
if proxy_auth:
proxy_auth = 'Proxy-authorization: %s' % proxy_auth
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr
user_agent = request.headers.iget('user_agent')
if not user_agent:
user_agent = "User-Agent: restkit/%s\r\n" % __version__
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth,
user_agent)
conn = self._pool.get(host=addr[0],
port=addr[1],
pool=self._pool,
is_ssl=is_ssl,
extra_headers=[],
**self.ssl_args)
conn.send(proxy_pieces)
p = HttpStream(SocketReader(conn.socket()),
kind=1,
decompress=True)
if p.status_code != 200:
raise ProxyError("Tunnel connection failed: %d %s" %
(resp.status_int, body))
_ = p.body_string()
else:
headers = []
if proxy_auth:
headers = [('Proxy-authorization', proxy_auth)]
conn = self._pool.get(host=addr[0],
port=addr[1],
pool=self._pool,
is_ssl=False,
extra_headers=[],
**self.ssl_args)
return conn
return
def recGETbody(HOST, request_headerGET):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
try:
s.sendall(request_headerGET.encode('utf-8'))
except:
print("An error has occured while trying to send the GET request")
dataReceived = SocketReader(s)
parsed = HttpStream(dataReceived)
body = parsed.body_file().read().decode('utf-8')
return body
def main():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect(('baike.baidu.com', 80))
s.send(
b("GET /view/262.htm HTTP/1.1\r\nHost: baike.baidu.com\r\n\r\n"))
p = HttpStream(SocketReader(s), decompress=True)
print(p.headers())
print(p.body_file().read())
finally:
s.close()
def GETverbose(HOST, request_headerGET):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, 80))
try:
s.sendall(request_headerGET.encode('utf-8'))
except:
print("An error has occured while trying to send the GET request")
dataReceived = SocketReader(s)
parsed = HttpStream(dataReceived)
body = parsed.body_file().read().decode('utf-8')
header = parsed.headers()
return pprint.pformat(header) + body
def rewrite_response(resp):
try:
while True:
parser = HttpStream(resp)
# we aren't doing anything here
for data in parser:
resp.writeall(data)
if not parser.should_keep_alive():
# close the connection.
break
except (socket.error, NoMoreData):
pass
def handle(self, upsock, peer):
with closing(upsock) as upsock:
origaddr = upsock.getsockname()
self.logger.debug('intercepted connection from {0} to {1}'.format(peer, origaddr))
p = HttpStream(SocketReader(upsock))
self.logger.debug('request: method={0} url={1} version={2} headers={3}'.format(p.method(), p.url(), p.version(), p.headers()))
url = None
if p.method() == b'CONNECT':
self.handle_connect(p, upsock)
elif p.url()[0] == '/':
self.handle_transparent(p, upsock)
else:
self.handle_proxy(p, upsock)
def proxy_connection(self, request, req_addr, is_ssl):
""" do the proxy connection """
proxy_settings = os.environ.get('%s_proxy' %
request.parsed_url.scheme)
if proxy_settings and proxy_settings is not None:
request.is_proxied = True
proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings)
addr = parse_netloc(urlparse.urlparse(proxy_settings))
if is_ssl:
if proxy_auth:
proxy_auth = 'Proxy-authorization: %s' % proxy_auth
proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr
user_agent = request.headers.iget('user_agent')
if not user_agent:
user_agent = "User-Agent: restkit/%s\r\n" % __version__
proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth,
user_agent)
conn = self._session.get(host=addr[0], port=addr[1],
pool=self._session, is_ssl=is_ssl,
extra_headers=[], **self.ssl_args)
conn.send(proxy_pieces)
p = HttpStream(SocketReader(conn.socket()), kind=1,
decompress=True)
if p.status_code != 200:
raise ProxyError("Tunnel connection failed: %d %s" %
(resp.status_int, body))
_ = p.body_string()
else:
headers = []
if proxy_auth:
headers = [('Proxy-authorization', proxy_auth)]
conn = self._session.get(host=addr[0], port=addr[1],
pool=self._session, is_ssl=False,
extra_headers=[], **self.ssl_args)
return conn
return
def _handle_client(self, client, address):
try:
r = SocketReader(client)
p = HttpStream(r)
headers = p.headers()
body = ""
if "CONTENT-LENGTH" in headers and int(
headers["CONTENT-LENGTH"]) > 0:
body = p.body_string(binary=False)
# System requests
if p.path() == "/alive":
self._respond(client, "ok")
return
# User requests
req = IpcServer.Request(self._respond, client, p.path(),
p.method(), headers, body)
if p.path() == "/message":
req.respond()
self.requests.put(req)
except Exception as e:
client.close()
raise e
def recv(self, wsgi=False):
try:
stream = HttpStream(self.reader, kind=HTTP_REQUEST, parser_class=HttpParser, decompress=True)
if bool(wsgi):
environ = stream.wsgi_environ()
environ['wsgi.url_scheme'] = guess_scheme(environ)
environ['wsgi.input'] = stream.body_file()
environ['wsgi.socket'] = self.socket
return environ
# BUG:
# http-parser has an issue here, if we call 'method' before 'headers'
# and invalid method name is returned...
fields = stream.headers()
method = stream.method()
url = stream.url()
version = stream.version()
content = stream.body_file()
url = urlparse(url)
path = url.path
query = parse_qs(url.query, keep_blank_values=True)
fragment = url.fragment
for k, v in six.iteritems(dict(query)):
query[k] = parse_query_value(v)
self.version = 'HTTP/%s.%s' % version
return method, path, query, fragment, fields, content
except NoMoreData:
pass
def handle(self):
self.logger = logging.getLogger("proxy")
self.session = Session()
try:
request = HttpStream(SocketReader(self.request))
_ = request.status()
except http_parser.http.BadStatusLine:
return
self.logger.warning(request.path())
if 'CONNECT' == request.method():
self.https_proxy(request)
else:
self.http_proxy(request)
self.session.close()
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
new_headers = rewrite_headers(parser, dict({'Host': 'api.mailgun.net'}, **basic_auth_headers('api', MAILGUN_APIKEY)))
if new_headers is None:
break
req.send(new_headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
new_headers = rewrite_headers(parser, {'Host': 'gunicorn.org'})
if new_headers is None:
break
req.send(new_headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def processData(newsocketconn, fromaddr, context, ipaddr, q):
if useSSL:
connstreamout = context.wrap_socket(newsocketconn, server_side=True)
else:
connstreamout = newsocketconn
try:
try:
### Read the json response using Socket Reader and split header and body
r = SocketReader(connstreamout)
p = HttpStream(r)
headers = p.headers()
q.put(makeLogMessage("INFO", "idrac", "ProcessData"))
if p.method() == 'POST':
bodydata = p.body_file().read()
bodydata = bodydata.decode("utf-8", errors='ignore')
### Read the json response and print the output
# outdata = dict()
try:
with open(f"{ipaddr[3]}/output/{ipaddr[2]}", "a+") as fd:
fd.write(bodydata)
fd.write("\n")
except Exception as ex:
print("Exception Occured =", ex)
StatusCode = """HTTP/1.1 200 OK\r\n\r\n"""
connstreamout.send(bytes(StatusCode, 'UTF-8'))
# if p.method() == 'GET':
# res = "HTTP/1.1 200 OK\n" \
# "Content-Type: application/json\n" \
# "\n" + json.dumps(data_buffer)
# connstreamout.send(res.encode())
# data_buffer.clear()
except Exception as err:
outdata = connstreamout.read()
traceback.print_exc()
print("Data needs to read in normal Text format.")
finally:
connstreamout.shutdown(socket.SHUT_RDWR)
connstreamout.close()
def get_response(self, request, connection):
""" return final respons, it is only accessible via peform
method """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to parse response")
p = HttpStream(SocketReader(connection.socket()),
kind=1,
decompress=self.decompress)
if log.isEnabledFor(logging.DEBUG):
log.debug("Got response: %s %s" % (p.version(), p.status()))
log.debug("headers: [%s]" % p.headers())
location = p.headers().get('location')
if self.follow_redirect:
if p.status_code() in (
301,
302,
307,
):
connection.close()
if request.method in ('GET', 'HEAD',) or \
self.force_follow_redirect:
if hasattr(self.body, 'read'):
try:
self.body.seek(0)
except AttributeError:
raise RequestError(
"Can't redirect %s to %s "
"because body has already been read" %
(self.url, location))
return self.redirect(location, request)
elif p.status_code() == 303 and self.method == "POST":
connection.close()
request.method = "GET"
request.body = None
return self.redirect(location, request)
# create response object
resp = self.response_class(connection, request, p)
# apply response filters
for f in self.response_filters:
f.on_response(resp, request)
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
# return final response
return resp
def rewrite_response(self, resp, extra):
try:
if extra.get('rewrite_response', False):
parser = HttpStream(resp, decompress=True)
rw = RewriteResponse(parser, resp, extra)
rw.execute()
else:
parser = HttpStream(resp)
headers = parser.headers()
headers['connection'] = 'close'
new_headers = headers_lines(parser, headers)
resp.writeall("".join(new_headers) + "\r\n")
body = parser.body_file()
send_body(resp, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def handle(self):
h = SocketReader(self.request)
p = HttpStream(h)
#delagate=url[p.url()]
delagate = Server.urls[p.url()]
data = p.body_string()
if data:
data = data.decode('utf-8')
else:
data = ""
js = None
data = self.createData(data)
print(p.url(), data)
try:
res = delagate(self.request, data)
#todo change to:
# self.request.send(res)
except Exception as ex:
print(ex)
def rewrite_request(req):
log = logging.getLogger("rewrite_request")
peername = req._src.getpeername()
try:
while True:
request_id = str(uuid.uuid4())
parser = HttpStream(req)
new_headers = rewrite_headers(
parser, peername, {'x-nimbus-io-user-request-id': request_id})
if new_headers is None:
break
log.debug("rewriting request %s" % (request_id, ))
req.send(new_headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def rewrite_request(req):
log = logging.getLogger("rewrite_request")
peername = req._src.getpeername()
try:
while True:
request_id = str(uuid.uuid4())
parser = HttpStream(req)
new_headers = rewrite_headers(
parser, peername, {'x-nimbus-io-user-request-id': request_id})
if new_headers is None:
break
log.debug("rewriting request %s" % ( request_id, ))
req.send(new_headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def join_data(request: HttpStream, type_: str):
assert type_ in ('Request', 'Response')
# try:
major, minor = request.version()
# except ParserError:
# raise
if type_ == 'Request':
url = request.url()
start_line = f'{request.method()} {url} HTTP/{major}.{minor}'
else:
start_line = f'HTTP/{major}.{minor} {request.status()}'
body = request.body_string(binary=True)
headers = request.headers()
setcookies = headers['Set-Cookie'] if 'Set-Cookie' in headers else None
if "Content-Encoding" in headers:
assert headers["Content-Encoding"] == "br"
body = brotli.decompress(body)
for key in ['Transfer-Encoding', 'Content-Encoding', 'Content-Length', 'Set-Cookie']:
if key in headers:
del headers[key]
headers['Content-Length'] = len(body)
headers = [f'{key}: {value}' for key, value in headers.items()]
if setcookies is not None:
setcookies = setcookies.split(",")
ind = 0
while ind < len(setcookies):
if setcookies[ind].lower()[:-4].endswith('expires'):
headers.append(f"Set-Cookie: {setcookies[ind]}, {setcookies[ind + 1]}")
ind += 2
else:
headers.append(f"Set-Cookie: {setcookies[ind]}")
ind += 1
headers = END_ONE_HEADER.join(headers)
return (start_line + END_ONE_HEADER + headers + END_HEADERS).encode('ISO-8859-1') + body
def rewrite_request(self, req, extra):
try:
while True:
if extra.get('rewrite_location', True):
parser = HttpStream(req)
prefix = extra.get('prefix', '')
location = parser.url()
if prefix:
try:
location = location.split(prefix, 1)[1] or '/'
except IndexError:
pass
headers = rewrite_headers(parser, location,
[('host', extra.get('host'))])
if headers is None:
break
extra['path'] = parser.path()
req.writeall(headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
else:
while True:
data = req.read(io.DEFAULT_BUFFER_SIZE)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def rewrite_request(self, req, extra):
try:
while True:
if extra.get('rewrite_location', True):
parser = HttpStream(req)
prefix = extra.get('prefix', '')
location = parser.url()
if prefix:
try:
location = location.split(prefix, 1)[1] or '/'
except IndexError:
pass
headers = rewrite_headers(parser, location,
[('host', extra.get('host'))])
if headers is None:
break
extra['path'] = parser.path()
req.writeall(headers)
body = parser.body_file()
while True:
data = body.read(8192)
if not data:
break
req.writeall(data)
else:
while True:
data = req.read(io.DEFAULT_BUFFER_SIZE)
if not data:
break
req.writeall(data)
except (socket.error, NoMoreData):
pass
def repeat_request(req_id):
req = session.query(Request).filter(Request.id == req_id).limit(1).all()
if len(req) == 0:
return redirect(url_for('requests', error_message=f'Not found ID {req_id}'))
req = req[0]
sock_server = socket.create_connection((req.host, 443))
sock_server = ssl.create_default_context().wrap_socket(sock_server, server_hostname=req.host)
server_reader = SocketReader(sock_server)
sock_server.sendall(req.headers)
server_resp = HttpStream(server_reader, decompress=True)
req = req.headers.decode().split('\r\n')
answer = join_data(server_resp, 'Response')
answer = answer.decode(errors='ignore')
answer = answer.split('\r\n')
return render_template('repeat.html', req_id=req_id, req=req, answer=answer)
def get_response(self, request, connection):
""" return final respons, it is only accessible via peform
method """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to parse response")
p = HttpStream(SocketReader(connection.socket()), kind=1,
decompress=self.decompress)
if log.isEnabledFor(logging.DEBUG):
log.debug("Got response: %s %s" % (p.version(), p.status()))
log.debug("headers: [%s]" % p.headers())
location = p.headers().get('location')
if self.follow_redirect:
if p.status_code() in (301, 302, 307,):
connection.close()
if request.method in ('GET', 'HEAD',) or \
self.force_follow_redirect:
if hasattr(self.body, 'read'):
try:
self.body.seek(0)
except AttributeError:
raise RequestError("Can't redirect %s to %s "
"because body has already been read"
% (self.url, location))
return self.redirect(location, request)
elif p.status_code() == 303 and self.method == "POST":
connection.close()
request.method = "GET"
request.body = None
return self.redirect(location, request)
# create response object
resp = self.response_class(connection, request, p)
# apply response filters
for f in self.response_filters:
f.on_response(resp, request)
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
# return final response
return resp
def __init__(self, host, ip, port, path, response_code, response, original_header):
self.host = host
self.ip = ip
self.port = port
self.path = path
self.response_code = response_code
self.response = response
self.decompressed_response = self.decompress(response)
self.parsed_response = HttpStream(StringReader(self.decompressed_response))
self.original_header = original_header
self.arc1_record = None
try:
self.arc1_record = self.create_arc1_record()
except NoMoreData:
logging.warning(" HTTP parser failed on empty header '%s'", self.decompressed_response)
except ParserError:
logging.warning(" HTTP parser failed on '%s'", self.decompressed_response)
except KeyError, k:
logging.warning(" Couldn't find key %s in %s", k, self.parsed_response.headers().keys())
logging.warning(" " + self.decompressed_response)
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
headers = parser.headers()
parsed_url = urlparse.urlparse(parser.url())
is_ssl = parsed_url.scheme == "https"
host = get_host(parse_address(parsed_url.netloc, 80),
is_ssl=is_ssl)
headers['Host'] = host
headers['Connection'] = 'close'
if 'Proxy-Connection' in headers:
del headers['Proxy-Connection']
location = urlparse.urlunparse(
('', '', parsed_url.path, parsed_url.params, parsed_url.query,
parsed_url.fragment))
httpver = "HTTP/%s" % ".".join(map(str, parser.version()))
new_headers = [
"%s %s %s\r\n" % (parser.method(), location, httpver)
]
new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \
for hname, hvalue in headers.items()])
req.writeall(bytes("".join(new_headers) + "\r\n"))
body = parser.body_file()
send_body(req, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def rewrite_response(self, resp, extra):
try:
if extra.get('rewrite_response', False):
parser = HttpStream(resp, decompress=True)
rw = RewriteResponse(parser, resp, extra)
rw.execute()
else:
parser = HttpStream(resp)
headers = parser.headers()
headers['connection'] = 'close'
new_headers = headers_lines(parser, headers)
resp.writeall("".join(new_headers) + "\r\n")
body = parser.body_file()
send_body(resp, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def https_proxy(self, connectreq: HttpStream):
self.request.sendall(CON_ESTABLISH)
host, port = connectreq.path().split(':')
cert_path = self.generate_cert(host)
# request to localhost as server
try:
sock_client = ssl.wrap_socket(self.request, keyfile="cert.key", certfile=cert_path,
ssl_version=ssl.PROTOCOL_TLS, server_side=True)
sock_server = socket.create_connection((host, port))
sock_server = ssl.create_default_context().wrap_socket(sock_server, server_hostname=host)
except OSError:
return
client_reader = SocketReader(sock_client)
server_reader = SocketReader(sock_server)
while True:
# request to mail.ru as client
clientreq = HttpStream(client_reader, decompress=True)
try:
clientreq_tosend = join_data(clientreq, 'Request')
except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError):
return
sock_server.sendall(clientreq_tosend)
# request from mail.ru to localhost as client
server_resp = HttpStream(server_reader, decompress=True)
try:
server_resp_tosend = join_data(server_resp, 'Response')
except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError):
return
try:
sock_client.sendall(server_resp_tosend)
except BrokenPipeError:
return
req = Request(clientreq.method(), server_resp.status_code(), clientreq_tosend, host)
self.add_date_base(clientreq, req)
def send(self, protocol ,host, header):
h = Http()
h.build_header()
data = header
str_response_body=""
str_response_header=""
if protocol == "http":
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,80))
s.send(data.encode('utf-8'))
"""
response = b""
while True:
d = client.recv(1024)
response = response + d
if not d:
break
#print(response)
"""
p = HttpStream(SocketReader(s))
elif protocol == "https":
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, 443))
s = ssl.wrap_socket(s, keyfile=None, certfile=None, server_side=False, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_SSLv23)
s.sendall(data.encode('utf-8'))
"""
response = b""
while True:
d = s.recv(4096)
response = response + d
if not d:
break
"""
p = HttpStream(SocketReader(s))
#return html.escape(str(response, errors='replace'))
# Transfer-Encoding: chunked の対処をする。
str_response_header += p.status()+"\n"
for h in p.headers():
str_response_header += h+": "+p.headers()[h]+"\n"
str_response_body += str(p.body_file().read(), errors='replace')
return {"body":str_response_body, "header":str_response_header}
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
headers = parser.headers()
parsed_url = urlparse.urlparse(parser.url())
is_ssl = parsed_url.scheme == "https"
host = get_host(parse_address(parsed_url.netloc, 80),
is_ssl=is_ssl)
headers['Host'] = host
headers['Connection'] = 'close'
if 'Proxy-Connection' in headers:
del headers['Proxy-Connection']
location = urlparse.urlunparse(('', '', parsed_url.path,
parsed_url.params, parsed_url.query, parsed_url.fragment))
httpver = "HTTP/%s" % ".".join(map(str,
parser.version()))
new_headers = ["%s %s %s\r\n" % (parser.method(), location,
httpver)]
new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \
for hname, hvalue in headers.items()])
req.writeall(bytes("".join(new_headers) + "\r\n"))
body = parser.body_file()
send_body(req, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def http_proxy(self, connectreq: HttpStream):
reqheaders = connectreq.headers()
# if ":" in connectreq.path():
parsed_url = connectreq.url().split('//')[1].split(':')
if len(parsed_url) > 1:
host = parsed_url[0]
port = parsed_url[1].split('/')[0]
else:
host, port = reqheaders['host'], 80
if 'Proxy-Connection' in reqheaders:
del reqheaders['Proxy-Connection']
elif 'proxy-connection' in reqheaders:
del reqheaders['proxy-connection']
# request to mail.ru as client
sock_server = socket.create_connection((host, port))
try:
clientreq = join_data(connectreq, 'Request')
except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError):
return
sock_server.sendall(clientreq)
# request from mail.ru to localhost as client
if connectreq.method() == 'HEAD':
return
server_reader = SocketReader(sock_server)
server_resp = HttpStream(server_reader, decompress=True)
try:
server_resp_tosend = join_data(server_resp, 'Response')
except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError):
return
self.request.sendall(server_resp_tosend)
req = Request(connectreq.method(), server_resp.status_code(), clientreq, host)
self.add_date_base(connectreq, req)
def test_upgrade_header():
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: keep-alive, upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: hojIvDoHedBucveephosh8==\r\nSec-WebSocket-Version: 13\r\n\r\n')
hdr = HttpStream(stream)
hdr.headers()
assert hdr.parser.is_upgrade() == 1
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: hojIvDoHedBucveephosh8==\r\nSec-WebSocket-Version: 13\r\n\r\n')
hdr = HttpStream(stream)
hdr.headers()
assert hdr.parser.is_upgrade() == 1
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nConnection: shenanigans\r\n\r\n')
hdr = HttpStream(stream)
hdr.headers()
assert hdr.parser.is_upgrade() == 0
def test_repeated_header():
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\nX-Test: bar\r\n\r\n')
hdr = HttpStream(stream).headers()
assert hdr['X-Test'] == 'foo, bar'
def test_repeated_continuation_header_py():
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\n bar\r\nX-Test: baz\r\n qux\r\n\r\n')
hdr = HttpStream(stream, parser_class=PyHttpParser).headers()
assert hdr['X-Test'] == 'foo bar, baz qux'
def perform(self, request):
""" perform the request. If an error happen it will first try to
restart it """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to perform request: %s %s %s" %
(request.host, request.method, request.path))
tries = 0
while True:
conn = None
try:
# get or create a connection to the remote host
conn = self.get_connection(request)
# send headers
msg = self.make_headers_string(request, conn.extra_headers)
# send body
if request.body is not None:
chunked = request.is_chunked()
if request.headers.iget('content-length') is None and \
not chunked:
raise RequestError(
"Can't determine content length and " +
"Transfer-Encoding header is not chunked")
# handle 100-Continue status
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
hdr_expect = request.headers.iget("expect")
if hdr_expect is not None and \
hdr_expect.lower() == "100-continue":
conn.send(msg)
msg = None
p = HttpStream(SocketReader(conn.socket()),
kind=1,
decompress=True)
if p.status_code != 100:
self.reset_request()
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
return self.response_class(conn, request, p)
chunked = request.is_chunked()
if log.isEnabledFor(logging.DEBUG):
log.debug("send body (chunked: %s)" % chunked)
if isinstance(request.body, types.StringTypes):
if msg is not None:
conn.send(msg + to_bytestring(request.body),
chunked)
else:
conn.send(to_bytestring(request.body), chunked)
else:
if msg is not None:
conn.send(msg)
if hasattr(request.body, 'read'):
if hasattr(request.body, 'seek'):
request.body.seek(0)
conn.sendfile(request.body, chunked)
else:
conn.sendlines(request.body, chunked)
if chunked:
conn.send_chunk("")
else:
conn.send(msg)
return self.get_response(request, conn)
except socket.gaierror, e:
if conn is not None:
conn.release(True)
raise RequestError(str(e))
except socket.timeout, e:
if conn is not None:
conn.release(True)
raise RequestTimeout(str(e))
def handle(self, sock, addr):
p = HttpStream(SocketReader(sock))
path = p.path()
if not path or path == "/":
path = "index.html"
if path.startswith("/"):
path = path[1:]
real_path = os.path.join(CURDIR, "static", path)
if os.path.isdir(real_path):
lines = ["<ul>"]
for d in os.listdir(real_path):
fpath = os.path.join(real_path, d)
lines.append("<li><a href=" + d + ">" + d + "</a>")
data = "".join(lines)
resp = "".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: text/html\r\n",
"Content-Length:" + str(len(data)) + "\r\n",
"Connection: close\r\n\r\n",
data])
sock.sendall(resp)
elif not os.path.exists(real_path):
util.write_error(sock, 404, "Not found", real_path + " not found")
else:
ctype = mimetypes.guess_type(real_path)[0]
if ctype.startswith('text') or 'html' in ctype:
try:
f = open(real_path, 'rb')
data = f.read()
resp = "".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: " + ctype + "\r\n",
"Content-Length:" + str(len(data)) + "\r\n",
"Connection: close\r\n\r\n",
data])
sock.sendall(resp)
finally:
f.close()
else:
try:
f = open(real_path, 'r')
clen = int(os.fstat(f.fileno())[6])
# send headers
sock.send("".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: " + ctype + "\r\n",
"Content-Length:" + str(clen) + "\r\n",
"Connection: close\r\n\r\n"]))
if not sendfile:
while True:
data = f.read(4096)
if not data:
break
sock.send(data)
else:
fileno = f.fileno()
sockno = sock.fileno()
sent = 0
offset = 0
nbytes = clen
sent += sendfile(sockno, fileno, offset+sent, nbytes-sent)
while sent != nbytes:
sent += sendfile(sock.fileno(), fileno, offset+sent, nbytes-sent)
finally:
f.close()
def perform(self, request):
""" perform the request. If an error happen it will first try to
restart it """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to perform request: %s %s %s" %
(request.host, request.method, request.path))
tries = 0
while True:
conn = None
try:
# get or create a connection to the remote host
conn = self.get_connection(request)
# send headers
msg = self.make_headers_string(request, conn.extra_headers)
# send body
if request.body is not None:
chunked = request.is_chunked()
if request.headers.iget('content-length') is None and \
not chunked:
raise RequestError(
"Can't determine content length and " +
"Transfer-Encoding header is not chunked")
# handle 100-Continue status
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3
hdr_expect = request.headers.iget("expect")
if hdr_expect is not None and \
hdr_expect.lower() == "100-continue":
conn.send(msg)
msg = None
p = HttpStream(SocketReader(conn.socket()),
kind=1,
decompress=True)
if p.status_code != 100:
self.reset_request()
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
return self.response_class(conn, request, p)
chunked = request.is_chunked()
if log.isEnabledFor(logging.DEBUG):
log.debug("send body (chunked: %s)" % chunked)
if isinstance(request.body, (str, )):
if msg is not None:
conn.send(msg + to_bytestring(request.body),
chunked)
else:
conn.send(to_bytestring(request.body), chunked)
else:
if msg is not None:
conn.send(msg)
if hasattr(request.body, 'read'):
if hasattr(request.body, 'seek'):
request.body.seek(0)
conn.sendfile(request.body, chunked)
else:
conn.sendlines(request.body, chunked)
if chunked:
conn.send_chunk("")
else:
conn.send(msg)
return self.get_response(request, conn)
except socket.gaierror as e:
if conn is not None:
conn.release(True)
raise RequestError(str(e))
except socket.timeout as e:
if conn is not None:
conn.release(True)
raise RequestTimeout(str(e))
except socket.error as e:
if log.isEnabledFor(logging.DEBUG):
log.debug("socket error: %s" % str(e))
if conn is not None:
conn.close()
errors = (errno.EAGAIN, errno.EPIPE, errno.EBADF,
errno.ECONNRESET)
if e[0] not in errors or tries >= self.max_tries:
raise RequestError("socket.error: %s" % str(e))
# should raised an exception in other cases
request.maybe_rewind(msg=str(e))
except NoMoreData as e:
if conn is not None:
conn.release(True)
request.maybe_rewind(msg=str(e))
if tries >= self.max_tries:
raise
except BadStatusLine:
if conn is not None:
conn.release(True)
# should raised an exception in other cases
request.maybe_rewind(msg="bad status line")
if tries >= self.max_tries:
raise
except Exception:
# unkown error
log.debug("unhandled exception %s" % traceback.format_exc())
if conn is not None:
conn.release(True)
raise
tries += 1
self._pool.backend_mod.sleep(self.wait_tries)
def read_output_data(newsocketconn, fromaddr):
if useSSL:
connstreamout = context.wrap_socket(newsocketconn, server_side=True)
else:
connstreamout = newsocketconn
### Output File Name
outputfile = "Events_" + str(fromaddr[0]) + ".txt"
global event_count
outdata = headers = HostDetails = ""
try:
try:
### Read the json response using Socket Reader and split header and body
r = SocketReader(connstreamout)
p = HttpStream(r)
headers = p.headers()
print("headers: ", headers)
bodydata = p.body_file().read()
bodydata = bodydata.decode("utf-8")
print("bodydata: ", bodydata)
for eachHeader in headers.items():
if eachHeader[0] == 'Host' or eachHeader[0] == 'host':
HostDetails = eachHeader[1]
### Read the json response and print the output
print("\n")
print("Server IP Address is ", fromaddr[0])
print("Server PORT number is ", fromaddr[1])
print("Listener IP is ", HostDetails)
outdata = json.loads(bodydata)
event_array = outdata['Events']
for event in event_array:
print("EventType is ", event['EventType'])
print("MessageId is ", event['MessageId'])
if 'EventId' in event:
print("EventId is ", event['EventId'])
if 'EventTimestamp' in event:
print("EventTimestamp is ", event['EventTimestamp'])
if 'Severity' in event:
print("Severity is ", event['Severity'])
if 'Message' in event:
print("Message is ", event['Message'])
if 'MessageArgs' in event:
print("MessageArgs is ", event['MessageArgs'])
if 'Context' in outdata:
print("Context is ", outdata['Context'])
print("\n")
except Exception as err:
outdata = connstreamout.read()
print("Data needs to read in normal Text format.")
print(outdata)
### Check the context and send the status OK if context matches
if outdata.get('Context', None) != ContextDetail:
print("Context ({}) does not match with the server ({}).".format(
outdata.get('Context', None), ContextDetail))
StatusCode = """HTTP/1.1 200 OK\r\n\r\n"""
connstreamout.send(bytes(StatusCode, 'UTF-8'))
try:
if event_count.get(str(fromaddr[0])):
event_count[str(
fromaddr[0])] = event_count[str(fromaddr[0])] + 1
else:
event_count[str(fromaddr[0])] = 1
print("Event Counter for Host %s is %s" %
(str(fromaddr[0]), event_count[fromaddr[0]]))
print("\n")
fd = open(outputfile, "a")
fd.write("Time:%s Count:%s\nHost IP:%s\nEvent Details:%s\n" %
(time.ctime(), event_count[str(
fromaddr[0])], str(fromaddr), outdata))
fd.close()
except Exception as err:
print(traceback.print_exc())
finally:
connstreamout.shutdown(socket.SHUT_RDWR)
connstreamout.close()
def test_continuation_header():
stream = io.BytesIO(b'GET /test HTTP/1.1\r\nX-Test: foo\r\n bar\r\n\r\n')
hdr = HttpStream(stream).headers()
assert hdr['X-Test'] == 'foo bar'
def handle(self, sock, addr):
sock.setblocking(1)
try:
p = HttpStream(SocketReader(sock))
path = p.path()
if not path or path == "/":
path = "index.html"
if path.startswith("/"):
path = path[1:]
real_path = os.path.join(self.conf['path'], path)
if os.path.isdir(real_path):
lines = ["<ul>"]
for d in os.listdir(real_path):
fpath = os.path.join(real_path, d)
lines.append("<li><a href=" + d + ">" + d + "</a>")
data = "".join(lines)
resp = "".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: text/html\r\n",
"Content-Length:" + str(len(data)) + "\r\n",
"Connection: close\r\n\r\n",
data])
sock.sendall(resp)
elif not os.path.exists(real_path):
util.write_error(sock, 404, "Not found", real_path + " not found")
else:
ctype = mimetypes.guess_type(real_path)[0]
fno = None
try:
fno = os.open(real_path,
os.O_RDONLY|os.O_NONBLOCK)
clen = int(os.fstat(fno)[6])
# send headers
sock.send("".join(["HTTP/1.1 200 OK\r\n",
"Content-Type: " + ctype + "\r\n",
"Content-Length:" + str(clen) + "\r\n",
"Connection: close\r\n\r\n"]))
if not sendfile:
while True:
data = os.read(fno, 4096)
if not data:
break
sock.send(data)
else:
offset = 0
nbytes = clen
if nbytes > BLKSIZE:
# Send file in at most 1GB blocks as some operating
# systems can have problems with sending files in blocks
# over 2GB.
for m in range(0, nbytes, BLKSIZE):
sendfile_all(fno, sock.fileno(), offset,
min(nbytes, BLKSIZE))
offset += BLKSIZE
nbytes -= BLKSIZE
else:
sendfile_all(fno, sock.fileno(), offset,
nbytes)
finally:
if fno is not None:
os.close(fno)
finally:
# make sure we close the socket
try:
sock.close()
except:
pass
