async def process_https(client_reader, client_writer, request_method, uri,
ident, loop):
response_code = 200
error_message = None
host, port = get_host_and_port(uri)
logger = get_logger()
try:
req_reader, req_writer = await asyncio.open_connection(host,
port,
ssl=False,
loop=loop)
client_writer.write(b'HTTP/1.1 200 Connection established\r\n')
client_writer.write(b'\r\n')
# HTTPS need to log here, as the connection may keep alive for long.
logger.info(('[{id}][{client}]: %s %d %s' %
(request_method, response_code, uri)).format(**ident))
tasks = [
asyncio.ensure_future(relay_stream(client_reader, req_writer,
ident),
loop=loop),
asyncio.ensure_future(relay_stream(req_reader, client_writer,
ident),
loop=loop),
]
await asyncio.wait(tasks, loop=loop)
except Exception as ex:
response_code = 502
error_message = '%s: %s' % (ex.__class__.__name__, ' '.join(
[str(arg) for arg in ex.args]))
if error_message:
logger.error(
('[{id}][{client}]: %s %d %s (%s)' %
(request_method, response_code, uri, error_message)).format(
**ident))
def debug_wormhole_semaphore(client_reader, client_writer):
global wormhole_semaphore
ident = get_ident(client_reader, client_writer)
available = wormhole_semaphore._value
logger = get_logger()
logger.debug((
'[{id}][{client}]: Resource available: %.2f%% (%d/%d)' % (
100 * float(available) / MAX_TASKS,
available,
MAX_TASKS
)).format(**ident))
async def process_wormhole(client_reader, client_writer, cloaking, auth, loop):
logger = get_logger()
ident = get_ident(client_reader, client_writer)
request_line, headers, payload = await process_request(
client_reader, MAX_RETRY, ident, loop
)
if not request_line:
logger.debug((
'[{id}][{client}]: !!! Task reject (empty request)'
).format(**ident))
return
request_fields = request_line.split(' ')
if len(request_fields) == 2:
request_method, uri = request_fields
http_version = 'HTTP/1.0'
elif len(request_fields) == 3:
request_method, uri, http_version = request_fields
else:
logger.debug((
'[{id}][{client}]: !!! Task reject (invalid request)'
).format(**ident))
return
if auth:
user_ident = await verify(client_reader, client_writer, headers, auth)
if user_ident is None:
logger.info((
'[{id}][{client}]: %s 407 %s' % (request_method, uri)
).format(**ident))
return
ident = user_ident
if request_method == 'CONNECT':
async with get_wormhole_semaphore(loop=loop):
debug_wormhole_semaphore(client_reader, client_writer)
return await process_https(
client_reader, client_writer, request_method, uri,
ident, loop
)
else:
async with get_wormhole_semaphore(loop=loop):
debug_wormhole_semaphore(client_reader, client_writer)
return await process_http(
client_writer, request_method, uri, http_version,
headers, payload, cloaking,
ident, loop
)
async def start_wormhole_server(host, port, cloaking, auth, loop):
logger = get_logger()
try:
accept = functools.partial(
accept_client, cloaking=cloaking, auth=auth, loop=loop
)
server = await asyncio.start_server(accept, host, port, loop=loop)
except OSError as ex:
logger.critical(
'[000000][%s]: !!! Failed to bind server at [%s:%d]: %s' % (
host, host, port, ex.args[1]
)
)
raise
else:
logger.info(
'[000000][%s]: wormhole bound at %s:%d' % (host, host, port)
)
return server
async def process_request(client_reader, max_retry, ident, loop):
logger = get_logger()
request_line = ''
headers = []
header = ''
payload = b''
try:
retry = 0
while True:
line = await client_reader.readline()
if not line:
if len(header) == 0 and retry < max_retry:
# handle the case when the client make connection
# but sending data is delayed for some reasons
retry += 1
await asyncio.sleep(0.1, loop=loop)
continue
else:
break
if line == b'\r\n':
break
if line != b'':
header += line.decode()
content_length = get_content_length(header)
while len(payload) < content_length:
payload += await client_reader.read(1024)
except Exception as ex:
logger.debug((
'[{id}][{client}]: !!! Task reject (%s: %s)' % (
ex.__class__.__name__,
' '.join([str(arg) for arg in ex.args])
)
).format(**ident))
if header:
header_lines = header.split('\r\n')
if len(header_lines) > 1:
request_line = header_lines[0]
if len(header_lines) > 2:
headers = header_lines[1:-1]
return request_line, headers, payload
async def process_request(client_reader, max_retry, ident, loop):
logger = get_logger()
request_line = ''
headers = []
header = ''
payload = b''
try:
retry = 0
while True:
line = await client_reader.readline()
if not line:
if len(header) == 0 and retry < max_retry:
# handle the case when the client make connection
# but sending data is delayed for some reasons
retry += 1
await asyncio.sleep(0.1, loop=loop)
continue
else:
break
if line == b'\r\n':
break
if line != b'':
header += line.decode()
content_length = get_content_length(header)
while len(payload) < content_length:
payload += await client_reader.read(1024)
except Exception as ex:
logger.debug(
('[{id}][{client}]: !!! Task reject (%s: %s)' %
(ex.__class__.__name__, ' '.join([str(arg)
for arg in ex.args]))).format(
**ident))
if header:
header_lines = header.split('\r\n')
if len(header_lines) > 1:
request_line = header_lines[0]
if len(header_lines) > 2:
headers = header_lines[1:-1]
return request_line, headers, payload
async def relay_stream(stream_reader,
stream_writer,
ident,
return_first_line=False):
logger = get_logger()
first_line = None
while True:
try:
line = await stream_reader.read(1024)
if len(line) == 0:
break
stream_writer.write(line)
except Exception as ex:
error_message = '%s: %s' % (ex.__class__.__name__, ' '.join(
[str(arg) for arg in ex.args]))
logger.debug(
('[{id}][{client}]: %s' % error_message).format(**ident))
break
else:
if return_first_line and first_line is None:
first_line = line[:line.find(b'\r\n')]
return first_line
async def process_https(client_reader, client_writer, request_method, uri,
ident, loop):
response_code = 200
error_message = None
host, port = get_host_and_port(uri)
logger = get_logger()
try:
req_reader, req_writer = await asyncio.open_connection(
host, port, ssl=False, loop=loop
)
client_writer.write(b'HTTP/1.1 200 Connection established\r\n')
client_writer.write(b'\r\n')
# HTTPS need to log here, as the connection may keep alive for long.
logger.info((
'[{id}][{client}]: %s %d %s' % (
request_method, response_code, uri
)
).format(**ident))
tasks = [
asyncio.ensure_future(
relay_stream(client_reader, req_writer, ident), loop=loop),
asyncio.ensure_future(
relay_stream(req_reader, client_writer, ident), loop=loop),
]
await asyncio.wait(tasks, loop=loop)
except Exception as ex:
response_code = 502
error_message = '%s: %s' % (
ex.__class__.__name__,
' '.join([str(arg) for arg in ex.args])
)
if error_message:
logger.error((
'[{id}][{client}]: %s %d %s (%s)' % (
request_method, response_code, uri, error_message
)
).format(**ident))
async def relay_stream(stream_reader, stream_writer,
ident, return_first_line=False):
logger = get_logger()
first_line = None
while True:
try:
line = await stream_reader.read(1024)
if len(line) == 0:
break
stream_writer.write(line)
except Exception as ex:
error_message = '%s: %s' % (
ex.__class__.__name__,
' '.join([str(arg) for arg in ex.args])
)
logger.debug((
'[{id}][{client}]: %s' % error_message
).format(**ident))
break
else:
if return_first_line and first_line is None:
first_line = line[:line.find(b'\r\n')]
return first_line
def accept_client(client_reader, client_writer, cloaking, auth, loop):
logger = get_logger()
ident = get_ident(client_reader, client_writer)
task = asyncio.ensure_future(
limit_wormhole(client_reader, client_writer, cloaking, auth, loop),
loop=loop
)
global clients
clients[task] = (client_reader, client_writer)
started_time = time()
def client_done(task):
del clients[task]
client_writer.close()
logger.debug((
'[{id}][{client}]: Connection closed (%.5f seconds)' % (
time() - started_time
)
).format(**ident))
logger.debug((
'[{id}][{client}]: Connection started'
).format(**ident))
task.add_done_callback(client_done)
def main():
"""CLI frontend function. It takes command line options e.g. host,
port and provides `--help` message.
"""
parser = ArgumentParser(
description='Wormhole(%s): Asynchronous IO HTTP and HTTPS Proxy' %
VERSION
)
parser.add_argument(
'-H', '--host', default='0.0.0.0',
help='Host to listen [default: %(default)s]'
)
parser.add_argument(
'-p', '--port', type=int, default=8800,
help='Port to listen [default: %(default)d]'
)
parser.add_argument(
'-a', '--authentication', default='',
help=('File contains username and password list '
'for proxy authentication [default: no authentication]')
)
parser.add_argument(
'-c', '--cloaking', action='store_true', default=False,
help='Add random string to header [default: %(default)s]'
)
parser.add_argument(
'-S', '--syslog-host', default='DISABLED',
help='Syslog Host [default: %(default)s]'
)
parser.add_argument(
'-P', '--syslog-port', type=int, default=514,
help='Syslog Port to listen [default: %(default)d]'
)
parser.add_argument(
'-l', '--license', action='store_true', default=False,
help='Print LICENSE and exit'
)
parser.add_argument(
'-v', '--verbose', action='count', default=0,
help='Print verbose'
)
args = parser.parse_args()
if args.license:
print(parser.description)
print(LICENSE)
exit()
if not (1 <= args.port <= 65535):
parser.error('port must be 1-65535')
logger = get_logger(args.syslog_host, args.syslog_port, args.verbose)
try:
import uvloop
except ImportError:
pass
else:
logger.debug(
'[000000][%s]: Using event loop from uvloop.' % args.host
)
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(
start_wormhole_server(
args.host, args.port,
args.cloaking, args.authentication,
loop
)
)
loop.run_forever()
except OSError:
pass
except KeyboardInterrupt:
print('bye')
finally:
loop.close()
async def process_http(client_writer, request_method, uri, http_version,
headers, payload, cloaking, ident, loop):
response_status = None
response_code = None
error_message = None
hostname = '127.0.0.1' # hostname (with optional port) e.g. example.com:80
request_headers = []
request_headers_end_index = 0
has_connection_header = False
for header in headers:
name_and_value = header.split(': ', 1)
if len(name_and_value) == 2:
name, value = name_and_value
else:
name, value = name_and_value[0], None
if name.lower() == "host":
if value is not None:
hostname = value
elif name.lower() == "connection":
has_connection_header = True
if value.lower() in ('keep-alive', 'persist'):
# current version of this program does not support
# the HTTP keep-alive feature
request_headers.append("Connection: close")
else:
request_headers.append(header)
elif name.lower() != 'proxy-connection':
request_headers.append(header)
if len(header) == 0 and request_headers_end_index == 0:
request_headers_end_index = len(request_headers) - 1
if request_headers_end_index == 0:
request_headers_end_index = len(request_headers)
if not has_connection_header:
request_headers.insert(request_headers_end_index, "Connection: close")
path = uri[len(hostname) + 7:] # 7 is len('http://')
new_head = ' '.join([request_method, path, http_version])
host, port = get_host_and_port(hostname, 80)
try:
req_reader, req_writer = await asyncio.open_connection(
host, port, flags=TCP_NODELAY, loop=loop)
req_writer.write(('%s\r\n' % new_head).encode())
await req_writer.drain()
if cloaking:
await cloak(req_writer, hostname, loop)
else:
req_writer.write(b'Host: ' + hostname.encode())
req_writer.write(b'\r\n')
[
req_writer.write((header + '\r\n').encode())
for header in request_headers
]
req_writer.write(b'\r\n')
if payload != b'':
req_writer.write(payload)
req_writer.write(b'\r\n')
await req_writer.drain()
response_status = await relay_stream(req_reader, client_writer, ident,
True)
except Exception as ex:
response_code = 502
error_message = '%s: %s' % (ex.__class__.__name__, ' '.join(
[str(arg) for arg in ex.args]))
if response_code is None:
response_code = int(response_status.decode('ascii').split(' ')[1])
logger = get_logger()
if error_message is None:
logger.info(('[{id}][{client}]: %s %d %s' %
(request_method, response_code, uri)).format(**ident))
else:
logger.error(
('[{id}][{client}]: %s %d %s (%s)' %
(request_method, response_code, uri, error_message)).format(
**ident))
async def process_http(client_writer, request_method, uri, http_version,
headers, payload, cloaking, ident, loop):
response_status = None
response_code = None
error_message = None
hostname = '127.0.0.1' # hostname (with optional port) e.g. example.com:80
request_headers = []
request_headers_end_index = 0
has_connection_header = False
for header in headers:
name_and_value = header.split(': ', 1)
if len(name_and_value) == 2:
name, value = name_and_value
else:
name, value = name_and_value[0], None
if name.lower() == "host":
if value is not None:
hostname = value
elif name.lower() == "connection":
has_connection_header = True
if value.lower() in ('keep-alive', 'persist'):
# current version of this program does not support
# the HTTP keep-alive feature
request_headers.append("Connection: close")
else:
request_headers.append(header)
elif name.lower() != 'proxy-connection':
request_headers.append(header)
if len(header) == 0 and request_headers_end_index == 0:
request_headers_end_index = len(request_headers) - 1
if request_headers_end_index == 0:
request_headers_end_index = len(request_headers)
if not has_connection_header:
request_headers.insert(request_headers_end_index, "Connection: close")
path = uri[len(hostname) + 7:] # 7 is len('http://')
new_head = ' '.join([request_method, path, http_version])
host, port = get_host_and_port(hostname, 80)
try:
req_reader, req_writer = await asyncio.open_connection(
host, port, flags=TCP_NODELAY, loop=loop
)
req_writer.write(('%s\r\n' % new_head).encode())
await req_writer.drain()
if cloaking:
await cloak(req_writer, hostname, loop)
else:
req_writer.write(b'Host: ' + hostname.encode())
req_writer.write(b'\r\n')
[req_writer.write((header + '\r\n').encode())
for header in request_headers]
req_writer.write(b'\r\n')
if payload != b'':
req_writer.write(payload)
req_writer.write(b'\r\n')
await req_writer.drain()
response_status = await relay_stream(
req_reader, client_writer, ident, True
)
except Exception as ex:
response_code = 502
error_message = '%s: %s' % (
ex.__class__.__name__,
' '.join([str(arg) for arg in ex.args])
)
if response_code is None:
response_code = int(response_status.decode('ascii').split(' ')[1])
logger = get_logger()
if error_message is None:
logger.info((
'[{id}][{client}]: %s %d %s' % (
request_method, response_code, uri
)
).format(**ident))
else:
logger.error((
'[{id}][{client}]: %s %d %s (%s)' % (
request_method, response_code, uri, error_message
)
).format(**ident))
def main():
"""CLI frontend function. It takes command line options e.g. host,
port and provides `--help` message.
"""
parser = ArgumentParser(
description='Wormhole(%s): Asynchronous IO HTTP and HTTPS Proxy' %
VERSION)
parser.add_argument('-H',
'--host',
default='0.0.0.0',
help='Host to listen [default: %(default)s]')
parser.add_argument('-p',
'--port',
type=int,
default=8800,
help='Port to listen [default: %(default)d]')
parser.add_argument(
'-a',
'--authentication',
default='',
help=('File contains username and password list '
'for proxy authentication [default: no authentication]'))
parser.add_argument(
'-c',
'--cloaking',
action='store_true',
default=False,
help='Add random string to header [default: %(default)s]')
parser.add_argument('-S',
'--syslog-host',
default='DISABLED',
help='Syslog Host [default: %(default)s]')
parser.add_argument('-P',
'--syslog-port',
type=int,
default=514,
help='Syslog Port to listen [default: %(default)d]')
parser.add_argument('-l',
'--license',
action='store_true',
default=False,
help='Print LICENSE and exit')
parser.add_argument('-v',
'--verbose',
action='count',
default=0,
help='Print verbose')
args = parser.parse_args()
if args.license:
print(parser.description)
print(LICENSE)
exit()
if not (1 <= args.port <= 65535):
parser.error('port must be 1-65535')
logger = get_logger(args.syslog_host, args.syslog_port, args.verbose)
try:
import uvloop
except ImportError:
pass
else:
logger.debug('[000000][%s]: Using event loop from uvloop.' % args.host)
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(
start_wormhole_server(args.host, args.port, args.cloaking,
args.authentication, loop))
loop.run_forever()
except OSError:
pass
except KeyboardInterrupt:
print('bye')
finally:
loop.close()