class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application received
"""
SUPPORTED_METHODS = [
'GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'
]
def __new__(cls, application, request, **kwargs):
# http://stackoverflow.com/questions/3209233/how-to-replace-an-instance-in-init-with-a-different-object
# Based on upgrade header, websocket request handler must be used
try:
if request.headers['Upgrade'].lower() == 'websocket':
return CustomWebSocketHandler(application, request, **kwargs)
except KeyError:
pass
return tornado.web.RequestHandler.__new__(cls, application, request,
**kwargs)
def set_default_headers(self):
# This is automatically called by Tornado :P
# XD Using this to remove "Server" header set by tornado
del self._headers["Server"]
def set_status(self, status_code, reason=None):
"""
Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
# This function writes a new response & caches it
def finish_response(self, response):
self.set_status(response.code)
for header, value in response.headers.get_all():
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
self.finish()
# This function is a callback when a small chunk is received
def handle_data_chunk(self, data):
if data:
self.write(data)
self.request.response_buffer += data
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
# The flow starts here
self.request.local_timestamp = datetime.datetime.now()
self.request.response_buffer = ''
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.uri.startswith(self.request.protocol,
0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host
if self.request.uri != '/': # Add uri only if needed
self.request.url += self.request.uri
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(self.application.cache_dir,
self.request,
self.application.cookie_regex,
self.application.cookie_blacklist)
request_hash = yield tornado.gen.Task(
self.cache_handler.calculate_hash)
self.cached_response = self.cache_handler.load()
if self.cached_response:
if self.cached_response.body:
self.write(self.cached_response.body)
self.finish_response(self.cached_response)
else:
# Request header cleaning
for header in restricted_request_headers:
try:
del self.request.headers[header]
except:
continue
# HTTP auth if exists
http_auth_username = None
http_auth_password = None
http_auth_mode = None
if self.application.http_auth:
# HTTP AUTH settings
host = self.request.host
# If default ports are not provided, they are added
try:
test = self.request.host.index(':')
except ValueError:
default_ports = {'http': '80', 'https': '443'}
try:
host = self.request.host + ':' + default_ports[
self.request.protocol]
except KeyError:
pass
# Check if auth is provided for that host
try:
index = self.application.http_auth_hosts.index(host)
http_auth_username = self.application.http_auth_usernames[
index]
http_auth_password = self.application.http_auth_passwords[
index]
http_auth_mode = self.application.http_auth_modes[index]
except ValueError:
pass
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
# httprequest object is created and then passed to async client with a callback
success_response = False # is used to check the response in the botnet mode
while not success_response:
#Proxy Switching (botnet_mode) code
if self.application.proxy_manager:
proxy = self.application.proxy_manager.get_next_available_proxy(
)
#print proxy
self.application.outbound_ip = proxy["proxy"][0]
self.application.outbound_port = int(proxy["proxy"][1])
# httprequest object is created and then passed to async client with a callback
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body if self.request.body else None,
headers=self.request.headers,
auth_username=http_auth_username,
auth_password=http_auth_password,
auth_mode=http_auth_mode,
follow_redirects=False,
use_gzip=True,
streaming_callback=self.handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
allow_nonstandard_methods=True,
prepare_curl_callback=prepare_curl_callback if self.application.outbound_proxy_type == "socks"\
else None, # socks callback function
validate_cert=False)
try:
response = yield tornado.gen.Task(async_client.fetch,
request)
except Exception:
response = None
pass
# Request retries
for i in range(0, 3):
if (response is None) or response.code in [408, 599]:
self.request.response_buffer = ''
response = yield tornado.gen.Task(
async_client.fetch, request)
else:
success_response = True
break
#botnet mode code (proxy switching)
#checking the status of the proxy (asynchronous)
if self.application.proxy_manager and not success_response:
proxy_check_req = tornado.httpclient.HTTPRequest(
url=self.application.proxy_manager.testing_url, #testing url is google.com
use_gzip=True,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
prepare_curl_callback=prepare_curl_callback if self.application.outbound_proxy_type == "socks"\
else None, # socks callback function
validate_cert=False)
try:
proxy_check_resp = yield tornado.gen.Task(
async_client.fetch, proxy_check_req)
except Exception:
pass
if proxy_check_resp.code != 200:
#self.application.proxy_manager.remove_proxy(proxy)
self.application.proxy_manager.remove_proxy(
proxy["index"])
else:
success_response = True
else:
success_response = True
self.finish_response(response)
# Cache the response after finishing the response, so caching time is not included in response time
self.cache_handler.dump(response)
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def trace(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is received.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(
b"HTTP/1.1 200 Connection established\r\n\r\n")
wrap_socket(self.request.connection.stream.socket,
host,
self.application.ca_cert,
self.application.ca_key,
self.application.ca_key_pass,
self.application.certs_folder,
success=ssl_success)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip)
# Tiny Hack to satisfy proxychains CONNECT request to HTTP port.
# HTTPS fail check has to be improvised
def ssl_fail():
try:
self.request.connection.stream.write(
b"HTTP/1.1 200 Connection established\r\n\r\n")
except tornado.iostream.StreamClosedError:
pass
server.handle_stream(self.request.connection.stream,
self.application.inbound_ip)
# Hacking to be done here, so as to check for ssl using proxy and auth
try:
s = ssl.wrap_socket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
#start_tunnel()
upstream.set_close_callback(ssl_fail)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.finish()
class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application received
"""
SUPPORTED_METHODS = ['GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']
def __new__(cls, application, request, **kwargs):
# http://stackoverflow.com/questions/3209233/how-to-replace-an-instance-in-init-with-a-different-object
# Based on upgrade header, websocket request handler must be used
try:
if request.headers['Upgrade'].lower() == 'websocket':
return CustomWebSocketHandler(application, request, **kwargs)
except KeyError:
pass
return tornado.web.RequestHandler.__new__(cls, application, request, **kwargs)
def set_default_headers(self):
# This is automatically called by Tornado :P
# XD Using this to remove "Server" header set by tornado
del self._headers["Server"]
def set_status(self, status_code, reason=None):
"""
Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
# This function writes a new response & caches it
def finish_response(self, response):
self.set_status(response.code)
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
self.finish()
# This function is a callback when a small chunk is received
def handle_data_chunk(self, data):
if data:
self.write(data)
self.request.response_buffer += data
@tornado.web.asynchronous
@tornado.gen.coroutine
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
# The flow starts here
self.request.response_buffer = ''
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.uri.startswith(self.request.protocol,0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(
self.application.cache_dir,
self.request,
self.application.cookie_regex,
self.application.cookie_blacklist
)
request_hash = yield tornado.gen.Task(self.cache_handler.calculate_hash)
self.cached_response = self.cache_handler.load()
if self.cached_response:
if self.cached_response.body:
self.write(self.cached_response.body)
self.finish_response(self.cached_response)
else:
# Request header cleaning
for header in restricted_request_headers:
try:
del self.request.headers[header]
except:
continue
# HTTP auth if exists
http_auth_username = None
http_auth_password = None
http_auth_mode = None
if self.application.http_auth:
# HTTP AUTH settings
host = self.request.host
# If default ports are not provided, they are added
try:
test = self.request.host.index(':')
except ValueError:
default_ports = {'http':'80', 'https':'443'}
try:
host = self.request.host + ':' + default_ports[self.request.protocol]
except KeyError:
pass
# Check if auth is provided for that host
try:
index = self.application.http_auth_hosts.index(host)
http_auth_username = self.application.http_auth_usernames[index]
http_auth_password = self.application.http_auth_passwords[index]
http_auth_mode = self.application.http_auth_modes[index]
except ValueError:
pass
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
# httprequest object is created and then passed to async client with a callback
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body if self.request.body else None,
headers=self.request.headers,
auth_username=http_auth_username,
auth_password=http_auth_password,
auth_mode=http_auth_mode,
follow_redirects=False,
use_gzip=True,
streaming_callback=self.handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
allow_nonstandard_methods=True,
prepare_curl_callback=prepare_curl_callback if self.application.outbound_proxy_type == "socks"\
else None, # socks callback function
validate_cert=False)
try:
response = yield tornado.gen.Task(async_client.fetch, request)
except Exception:
pass
# Request retries
for i in range(0,3):
if response.code in [408, 599]:
self.request.response_buffer = ''
response = yield tornado.gen.Task(async_client.fetch, request)
else:
break
self.finish_response(response)
# Cache the response after finishing the response, so caching time is not included in response time
self.cache_handler.dump(response)
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def trace(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is received.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
wrap_socket(
self.request.connection.stream.socket,
host,
self.application.ca_cert,
self.application.ca_key,
self.application.certs_folder,
success=ssl_success
)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip)
# Tiny Hack to satisfy proxychains CONNECT request to HTTP port.
# HTTPS fail check has to be improvised
#def ssl_fail():
# self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
# server.handle_stream(self.request.connection.stream, self.application.inbound_ip)
# Hacking to be done here, so as to check for ssl using proxy and auth
try:
s = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
#start_tunnel()
#upstream.set_close_callback(ssl_fail)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.finish()
class CustomWebSocketHandler(tornado.websocket.WebSocketHandler):
"""
* See docs XD
* This class is used for handling websocket traffic.
* Object of this class replaces the main request handler for a request with
header => "Upgrade: websocket"
* wss:// - CONNECT request is handled by main handler
"""
def upstream_connect(self, io_loop=None, callback=None):
"""
Implemented as a custom alternative to tornado.websocket.websocket_connect
"""
# io_loop is needed, how else will it work with tornado :P
if io_loop is None:
io_loop = tornado.ioloop.IOLoop.current()
# During secure communication, we get relative URI, so make them absolute
if self.request.uri.startswith(self.request.protocol,
0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
# WebSocketClientConnection expects ws:// & wss://
self.request.url = self.request.url.replace("http", "ws", 1)
# Have to add cookies and stuff
request_headers = tornado.httputil.HTTPHeaders()
for name, value in self.request.headers.iteritems():
if name not in restricted_request_headers:
request_headers.add(name, value)
# Build a custom request
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
headers=request_headers,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password)
self.upstream_connection = CustomWebSocketClientConnection(
io_loop, request)
if callback is not None:
io_loop.add_future(self.upstream_connection.connect_future,
callback)
return self.upstream_connection.connect_future # This returns a future
def _execute(self, transforms, *args, **kwargs):
"""
Overriding of a method of WebSocketHandler
"""
def start_tunnel(future):
"""
A callback which is called when connection to url is successful
"""
self.upstream = future.result(
) # We need upstream to write further messages
self.handshake_request = self.upstream_connection.request # HTTPRequest needed for caching :P
self.handshake_request.response_buffer = "" # Needed for websocket data & compliance with cache_handler stuff
self.handshake_request.version = "HTTP/1.1" # Tiny hack to protect caching (But according to websocket standards)
self.handshake_request.body = self.handshake_request.body or "" # I dont know why a None is coming :P
tornado.websocket.WebSocketHandler._execute(
self, transforms, *args,
**kwargs) # The regular procedures are to be done
# We try to connect to provided URL & then we proceed with connection on client side.
self.upstream = self.upstream_connect(callback=start_tunnel)
def store_upstream_data(self, message):
"""
Save websocket data sent from client to server, i.e add it to HTTPRequest.response_buffer with direction (>>)
"""
try: # Cannot write binary content as a string, so catch it
self.handshake_request.response_buffer += (">>> %s\r\n" %
(message))
except TypeError:
self.handshake_request.response_buffer += (">>> May be binary\r\n")
def store_downstream_data(self, message):
"""
Save websocket data sent from client to server, i.e add it to HTTPRequest.response_buffer with direction (<<)
"""
try: # Cannot write binary content as a string, so catch it
self.handshake_request.response_buffer += ("<<< %s\r\n" %
(message))
except TypeError:
self.handshake_request.response_buffer += ("<<< May be binary\r\n")
def on_message(self, message):
"""
Everytime a message is received from client side, this instance method is called
"""
self.upstream.write_message(
message) # The obtained message is written to upstream
self.store_upstream_data(message)
# The following check ensures that if a callback is added for reading message from upstream, another one is not added
if not self.upstream.read_future:
self.upstream.read_message(
callback=self.on_response
) # A callback is added to read the data when upstream responds
def on_response(self, message):
"""
A callback when a message is recieved from upstream
*** Here message is a future
"""
# The following check ensures that if a callback is added for reading message from upstream, another one is not added
if not self.upstream.read_future:
self.upstream.read_message(callback=self.on_response)
if self.ws_connection: # Check if connection still exists
if message.result(
): # Check if it is not NULL ( Indirect checking of upstream connection )
self.write_message(
message.result()) # Write obtained message to client
self.store_downstream_data(message.result())
else:
self.close()
def on_close(self):
"""
Called when websocket is closed. So handshake request-response pair along with websocket data as response body is saved
"""
# Required for cache_handler
self.handshake_response = tornado.httpclient.HTTPResponse(
self.handshake_request,
self.upstream_connection.code,
headers=self.upstream_connection.headers,
request_time=0)
# Procedure for dumping a tornado request-response
self.cache_handler = CacheHandler(self.application.cache_dir,
self.handshake_request,
self.application.cookie_regex,
self.application.cookie_blacklist)
self.cached_response = self.cache_handler.load()
self.cache_handler.dump(self.handshake_response)
class CustomWebSocketHandler(tornado.websocket.WebSocketHandler):
"""
* See docs XD
* This class is used for handling websocket traffic.
* Object of this class replaces the main request handler for a request with
header => "Upgrade: websocket"
* wss:// - CONNECT request is handled by main handler
"""
def upstream_connect(self, io_loop=None, callback=None):
"""
Implemented as a custom alternative to tornado.websocket.websocket_connect
"""
# io_loop is needed, how else will it work with tornado :P
if io_loop is None:
io_loop = tornado.ioloop.IOLoop.current()
# During secure communication, we get relative URI, so make them absolute
if self.request.uri.startswith(self.request.protocol,0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
# WebSocketClientConnection expects ws:// & wss://
self.request.url = self.request.url.replace("http", "ws", 1)
# Have to add cookies and stuff
request_headers = tornado.httputil.HTTPHeaders()
for name, value in self.request.headers.iteritems():
if name not in restricted_request_headers:
request_headers.add(name, value)
# Build a custom request
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
headers=request_headers,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password
)
self.upstream_connection = CustomWebSocketClientConnection(io_loop, request)
if callback is not None:
io_loop.add_future(self.upstream_connection.connect_future, callback)
return self.upstream_connection.connect_future # This returns a future
def _execute(self, transforms, *args, **kwargs):
"""
Overriding of a method of WebSocketHandler
"""
def start_tunnel(future):
"""
A callback which is called when connection to url is successful
"""
self.upstream = future.result() # We need upstream to write further messages
self.handshake_request = self.upstream_connection.request # HTTPRequest needed for caching :P
self.handshake_request.response_buffer = "" # Needed for websocket data & compliance with cache_handler stuff
self.handshake_request.version = "HTTP/1.1" # Tiny hack to protect caching (But according to websocket standards)
self.handshake_request.body = self.handshake_request.body or "" # I dont know why a None is coming :P
tornado.websocket.WebSocketHandler._execute(self, transforms, *args, **kwargs) # The regular procedures are to be done
# We try to connect to provided URL & then we proceed with connection on client side.
self.upstream = self.upstream_connect(callback=start_tunnel)
def store_upstream_data(self, message):
"""
Save websocket data sent from client to server, i.e add it to HTTPRequest.response_buffer with direction (>>)
"""
try: # Cannot write binary content as a string, so catch it
self.handshake_request.response_buffer += (">>> %s\r\n"%(message))
except TypeError:
self.handshake_request.response_buffer += (">>> May be binary\r\n")
def store_downstream_data(self, message):
"""
Save websocket data sent from client to server, i.e add it to HTTPRequest.response_buffer with direction (<<)
"""
try: # Cannot write binary content as a string, so catch it
self.handshake_request.response_buffer += ("<<< %s\r\n"%(message))
except TypeError:
self.handshake_request.response_buffer += ("<<< May be binary\r\n")
def on_message(self, message):
"""
Everytime a message is received from client side, this instance method is called
"""
self.upstream.write_message(message) # The obtained message is written to upstream
self.store_upstream_data(message)
# The following check ensures that if a callback is added for reading message from upstream, another one is not added
if not self.upstream.read_future:
self.upstream.read_message(callback=self.on_response) # A callback is added to read the data when upstream responds
def on_response(self, message):
"""
A callback when a message is recieved from upstream
*** Here message is a future
"""
# The following check ensures that if a callback is added for reading message from upstream, another one is not added
if not self.upstream.read_future:
self.upstream.read_message(callback=self.on_response)
if self.ws_connection: # Check if connection still exists
if message.result(): # Check if it is not NULL ( Indirect checking of upstream connection )
self.write_message(message.result()) # Write obtained message to client
self.store_downstream_data(message.result())
else:
self.close()
def on_close(self):
"""
Called when websocket is closed. So handshake request-response pair along with websocket data as response body is saved
"""
# Required for cache_handler
self.handshake_response = tornado.httpclient.HTTPResponse(
self.handshake_request,
self.upstream_connection.code,
headers=self.upstream_connection.headers,
request_time=0
)
# Procedure for dumping a tornado request-response
self.cache_handler = CacheHandler(
self.application.cache_dir,
self.handshake_request,
self.application.cookie_regex,
self.application.cookie_blacklist
)
self.cached_response = self.cache_handler.load()
self.cache_handler.dump(self.handshake_response)
class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application received
"""
SUPPORTED_METHODS = [
'GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'
]
def __new__(cls, application, request, **kwargs):
# http://stackoverflow.com/questions/3209233/how-to-replace-an-instance-in-init-with-a-different-object
# Based on upgrade header, websocket request handler must be used
try:
if request.headers['Upgrade'].lower() == 'websocket':
return CustomWebSocketHandler(application, request, **kwargs)
except KeyError:
pass
return tornado.web.RequestHandler.__new__(cls, application, request,
**kwargs)
def set_status(self, status_code, reason=None):
"""
Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
def calculate_delay(self, response):
self.application.throttle_variables["hosts"][
self.request.host]["request_times"].append(response.request_time)
if len(self.application.throttle_variables["hosts"][
self.request.host]) > 20:
self.application.throttle_variables["hosts"][
self.request.host]["request_times"].pop(0)
response_times = self.application.throttle_variables["hosts"][
self.request.host]["request_times"]
last_ten = sum(
response_times[:int(len(response_times) / 2)]) / int(
len(response_times) / 2)
second_last_ten = sum(
response_times[int(len(response_times) / 2):]) / (
len(response_times) - int(len(response_times) / 2))
if round(last_ten - second_last_ten,
3) > self.application.throttle_variables["threshold"]:
self.application.throttle_variables["hosts"][
self.request.host]["delay"] = round(
last_ten - second_last_ten, 3)
else:
self.application.throttle_variables["hosts"][
self.request.host]["delay"] = 0
# This function is a callback after the async client gets the full response
# This method will be improvised with more headers from original responses
def handle_response(self, response):
if self.application.throttle_variables:
self.calculate_delay(response)
if response.code in [408, 599]:
try:
old_count = self.request.retries
self.request.retries = old_count + 1
except AttributeError:
self.request.retries = 1
finally:
if self.request.retries < 3:
self.request.response_buffer = ''
self.clear()
self.process_request()
else:
self.write_response(response)
else:
self.write_response(response)
# This function writes a new response & caches it
def write_response(self, response):
self.set_status(response.code)
del self._headers['Server']
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
if self.request.response_buffer:
self.cache_handler.dump(response)
self.finish()
# This function handles a dummy response object which is created from cache
def write_cached_response(self, response):
self.set_status(response.code)
for header, value in response.headers.items():
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
self.write(response.body)
self.finish()
# This function is a callback when a small chunk is received
def handle_data_chunk(self, data):
if data:
self.write(data)
self.request.response_buffer += data
# This function creates and makes the request to upstream server
def process_request(self):
if self.cached_response:
self.write_cached_response(self.cached_response)
else:
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
# httprequest object is created and then passed to async client with a callback
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
use_gzip=True,
streaming_callback=self.handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
allow_nonstandard_methods=True,
validate_cert=False)
try:
async_client.fetch(request, callback=self.handle_response)
except Exception:
pass
def cache_check(self):
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(self.application.cache_dir,
self.request,
self.application.cookie_regex,
self.application.cookie_blacklist)
self.cached_response = self.cache_handler.load()
self.process_request()
@tornado.web.asynchronous
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
# The flow starts here
self.request.response_buffer = ''
# Request header cleaning
for header in restricted_request_headers:
try:
del self.request.headers[header]
except:
continue
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.uri.startswith(self.request.protocol,
0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
if self.application.throttle_variables:
try:
throttle_delay = self.application.throttle_variables["hosts"][
self.request.host]["delay"]
except KeyError:
self.application.throttle_variables["hosts"][
self.request.host] = {
"request_times": [],
"delay": 0
}
throttle_delay = 0
finally:
if throttle_delay == 0:
self.cache_check()
else:
tornado.ioloop.IOLoop.instance().add_timeout(
datetime.timedelta(seconds=throttle_delay),
self.cache_check)
else:
self.cache_check()
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def trace(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is received.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(
b"HTTP/1.1 200 Connection established\r\n\r\n")
wrap_socket(self.request.connection.stream.socket,
host,
self.application.ca_cert,
self.application.ca_key,
self.application.certs_folder,
success=ssl_success)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip)
# Tiny Hack to satisfy proxychains CONNECT request to HTTP port.
# HTTPS fail check has to be improvised
#def ssl_fail():
# self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
# server.handle_stream(self.request.connection.stream, self.application.inbound_ip)
# Hacking to be done here, so as to check for ssl using proxy and auth
try:
s = ssl.wrap_socket(
socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
#start_tunnel()
#upstream.set_close_callback(ssl_fail)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.finish()
class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application recieves
"""
SUPPORTED_METHODS = ['GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS']
def set_status(self, status_code, reason=None):
"""Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
@tornado.web.asynchronous
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
# Data for handling headers through a streaming callback
self.request.response_buffer = ''
restricted_headers = ['Content-Length',
'Content-Encoding',
'Etag',
'Transfer-Encoding',
'Connection',
'Vary',
'Accept-Ranges',
'Pragma']
# This function is a callback after the async client gets the full response
# This method will be improvised with more headers from original responses
def handle_response(response):
self.set_status(response.code)
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_headers:
self.set_header(header, value)
if self.request.response_buffer:
self.cache_handler.dump(response)
else:
self.write(response.body)
self.finish()
# This function is a callback when a small chunk is recieved
def handle_data_chunk(data):
if data:
self.write(data)
self.request.response_buffer += data
# More headers are to be removed
for header in ('Connection', 'Pragma', 'Cache-Control'):
try:
del self.request.headers[header]
except:
continue
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.host in self.request.uri.split('/'): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(self.application.cache_dir, self.request)
cached_response = self.cache_handler.load()
#if cached_response:
#handle_response(cached_response)
# httprequest object is created and then passed to async client with a callback
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
use_gzip=True,
streaming_callback=handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
allow_nonstandard_methods=True,
validate_cert=False)
try:
async_client.fetch(request, callback=handle_response)
except Exception:
pass
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is recieved.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(b"HTTP/1.1 200 OK CONNECTION ESTABLISHED\r\n\r\n")
wrap_socket(self.request.connection.stream.socket, host, success=ssl_success)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip) # lint:ok
try:
s = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.write(b"Server Not Found")
self.finish()
class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application received
"""
SUPPORTED_METHODS = ['GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']
def __new__(cls, application, request, **kwargs):
# http://stackoverflow.com/questions/3209233/how-to-replace-an-instance-in-init-with-a-different-object
# Based on upgrade header, websocket request handler must be used
try:
if request.headers['Upgrade'].lower() == 'websocket':
return CustomWebSocketHandler(application, request, **kwargs)
except KeyError:
pass
return tornado.web.RequestHandler.__new__(cls, application, request, **kwargs)
def set_status(self, status_code, reason=None):
"""
Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
def calculate_delay(self, response):
self.application.throttle_variables["hosts"][self.request.host]["request_times"].append(response.request_time)
if len(self.application.throttle_variables["hosts"][self.request.host]) > 20:
self.application.throttle_variables["hosts"][self.request.host]["request_times"].pop(0)
response_times = self.application.throttle_variables["hosts"][self.request.host]["request_times"]
last_ten = sum(response_times[:int(len(response_times)/2)])/int(len(response_times)/2)
second_last_ten = sum(response_times[int(len(response_times)/2):])/(len(response_times)-int(len(response_times)/2))
if round(last_ten - second_last_ten, 3) > self.application.throttle_variables["threshold"]:
self.application.throttle_variables["hosts"][self.request.host]["delay"] = round(last_ten - second_last_ten, 3)
else:
self.application.throttle_variables["hosts"][self.request.host]["delay"] = 0
# This function is a callback after the async client gets the full response
# This method will be improvised with more headers from original responses
def handle_response(self, response):
if self.application.throttle_variables:
self.calculate_delay(response)
if response.code in [408, 599]:
try:
old_count = self.request.retries
self.request.retries = old_count + 1
except AttributeError:
self.request.retries = 1
finally:
if self.request.retries < 3:
self.request.response_buffer = ''
self.clear()
self.process_request()
else:
self.write_response(response)
else:
self.write_response(response)
# This function writes a new response & caches it
def write_response(self, response):
self.set_status(response.code)
del self._headers['Server']
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
if self.request.response_buffer:
self.cache_handler.dump(response)
self.finish()
# This function handles a dummy response object which is created from cache
def write_cached_response(self, response):
self.set_status(response.code)
for header, value in response.headers.items():
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
self.write(response.body)
self.finish()
# This function is a callback when a small chunk is received
def handle_data_chunk(self, data):
if data:
self.write(data)
self.request.response_buffer += data
# This function creates and makes the request to upstream server
def process_request(self):
if self.cached_response:
self.write_cached_response(self.cached_response)
else:
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
# httprequest object is created and then passed to async client with a callback
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
use_gzip=True,
streaming_callback=self.handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
allow_nonstandard_methods=True,
validate_cert=False)
try:
async_client.fetch(request, callback=self.handle_response)
except Exception:
pass
def cache_check(self):
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(
self.application.cache_dir,
self.request,
self.application.cookie_regex,
self.application.cookie_blacklist
)
self.cached_response = self.cache_handler.load()
self.process_request()
@tornado.web.asynchronous
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
# The flow starts here
self.request.response_buffer = ''
# Request header cleaning
for header in restricted_request_headers:
try:
del self.request.headers[header]
except:
continue
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.uri.startswith(self.request.protocol,0): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
if self.application.throttle_variables:
try:
throttle_delay = self.application.throttle_variables["hosts"][self.request.host]["delay"]
except KeyError:
self.application.throttle_variables["hosts"][self.request.host] = {"request_times":[], "delay":0}
throttle_delay = 0
finally:
if throttle_delay == 0:
self.cache_check()
else:
tornado.ioloop.IOLoop.instance().add_timeout(datetime.timedelta(seconds=throttle_delay), self.cache_check)
else:
self.cache_check()
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def trace(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is received.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
wrap_socket(
self.request.connection.stream.socket,
host,
self.application.ca_cert,
self.application.ca_key,
self.application.certs_folder,
success=ssl_success
)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip)
# Tiny Hack to satisfy proxychains CONNECT request to HTTP port.
# HTTPS fail check has to be improvised
#def ssl_fail():
# self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
# server.handle_stream(self.request.connection.stream, self.application.inbound_ip)
# Hacking to be done here, so as to check for ssl using proxy and auth
try:
s = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
#start_tunnel()
#upstream.set_close_callback(ssl_fail)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.finish()
class ProxyHandler(tornado.web.RequestHandler):
"""
This RequestHandler processes all the requests that the application received
"""
SUPPORTED_METHODS = ['GET', 'POST', 'CONNECT', 'HEAD', 'PUT', 'DELETE', 'OPTIONS']
def set_status(self, status_code, reason=None):
"""Sets the status code for our response.
Overriding is done so as to handle unknown
response codes gracefully.
"""
self._status_code = status_code
if reason is not None:
self._reason = tornado.escape.native_str(reason)
else:
try:
self._reason = tornado.httputil.responses[status_code]
except KeyError:
self._reason = tornado.escape.native_str("Server Not Found")
@tornado.web.asynchronous
def get(self):
"""
* This function handles all requests except the connect request.
* Once ssl stream is formed between browser and proxy, the requests are
then processed by this function
"""
self.request.response_buffer = ''
# Data for handling headers through a streaming callback
# Need to work around for something
restricted_response_headers = ['Content-Length',
'Content-Encoding',
'Etag',
'Transfer-Encoding',
'Connection',
'Vary',
'Accept-Ranges',
'Pragma']
# This function is a callback after the async client gets the full response
# This method will be improvised with more headers from original responses
def handle_response(response):
self.set_status(response.code)
del self._headers['Server']
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
if self.request.response_buffer:
self.cache_handler.dump(response)
self.finish()
def handle_cached_response(response):
self.set_status(response.code)
for header, value in list(response.headers.items()):
if header == "Set-Cookie":
self.add_header(header, value)
else:
if header not in restricted_response_headers:
self.set_header(header, value)
self.write(response.body)
self.finish()
# This function is a callback when a small chunk is received
def handle_data_chunk(data):
if data:
self.write(data)
self.request.response_buffer += data
# More headers are to be removed
for header in ('Connection', 'Pragma', 'Cache-Control', 'If-Modified-Since'):
try:
del self.request.headers[header]
except:
continue
# The requests that come through ssl streams are relative requests, so transparent
# proxying is required. The following snippet decides the url that should be passed
# to the async client
if self.request.host in self.request.uri.split('/'): # Normal Proxy Request
self.request.url = self.request.uri
else: # Transparent Proxy Request
self.request.url = self.request.protocol + "://" + self.request.host + self.request.uri
# This block here checks for already cached response and if present returns one
self.cache_handler = CacheHandler(
self.application.cache_dir,
self.request,
self.application.cookie_regex,
self.application.cookie_blacklist
)
cached_response = self.cache_handler.load()
if cached_response:
handle_cached_response(cached_response)
else:
# httprequest object is created and then passed to async client with a callback
# pycurl is needed for curl client
async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
url=self.request.url,
method=self.request.method,
body=self.request.body,
headers=self.request.headers,
follow_redirects=False,
use_gzip=True,
streaming_callback=handle_data_chunk,
header_callback=None,
proxy_host=self.application.outbound_ip,
proxy_port=self.application.outbound_port,
proxy_username=self.application.outbound_username,
proxy_password=self.application.outbound_password,
allow_nonstandard_methods=True,
validate_cert=False)
try:
async_client.fetch(request, callback=handle_response)
except Exception:
pass
# The following 5 methods can be handled through the above implementation
@tornado.web.asynchronous
def post(self):
return self.get()
@tornado.web.asynchronous
def head(self):
return self.get()
@tornado.web.asynchronous
def put(self):
return self.get()
@tornado.web.asynchronous
def delete(self):
return self.get()
@tornado.web.asynchronous
def options(self):
return self.get()
@tornado.web.asynchronous
def connect(self):
"""
This function gets called when a connect request is received.
* The host and port are obtained from the request uri
* A socket is created, wrapped in ssl and then added to SSLIOStream
* This stream is used to connect to speak to the remote host on given port
* If the server speaks ssl on that port, callback start_tunnel is called
* An OK response is written back to client
* The client side socket is wrapped in ssl
* If the wrapping is successful, a new SSLIOStream is made using that socket
* The stream is added back to the server for monitoring
"""
host, port = self.request.uri.split(':')
def start_tunnel():
try:
self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
wrap_socket(
self.request.connection.stream.socket,
host,
self.application.ca_cert,
self.application.ca_key,
self.application.certs_folder,
success=ssl_success
)
except tornado.iostream.StreamClosedError:
pass
def ssl_success(client_socket):
client = tornado.iostream.SSLIOStream(client_socket)
server.handle_stream(client, self.application.inbound_ip)
# Tiny Hack to satisfy proxychains CONNECT request to HTTP port.
# HTTPS fail check has to be improvised
#def ssl_fail():
# self.request.connection.stream.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
# server.handle_stream(self.request.connection.stream, self.application.inbound_ip)
# Hacking to be done here, so as to check for ssl using proxy and auth
try:
s = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0))
upstream = tornado.iostream.SSLIOStream(s)
#start_tunnel()
#upstream.set_close_callback(ssl_fail)
upstream.connect((host, int(port)), start_tunnel)
except Exception:
self.finish()
