def test_headers(self):
'''test_add_http ensures that http is added to a url
'''
print("Testing utils header functions...")
from utils import parse_headers
from utils import basic_auth_header
# If we don't give headers, and no default, should get {}
print("Case 1: Don't give headers, return empty dictionary")
empty_dict = parse_headers(default_header=False)
self.assertEqual(empty_dict,dict())
# If we ask for default, should get something back
print("Case 2: ask for default headers...")
headers = parse_headers(default_header=True)
for field in ["Accept","Content-Type"]:
self.assertTrue(field in headers)
# Can we add a header?
print("Case 3: add a custom header")
new_header = {"cookies":"nom"}
headers = parse_headers(default_header=True,
headers=new_header)
for field in ["Accept","Content-Type","cookies"]:
self.assertTrue(field in headers)
# Basic auth header
print("Case 4: basic_auth_header - ask for custom authentication header")
auth = basic_auth_header(username='******',
password='******')
self.assertEqual(auth['Authorization'],
'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def test_headers(self):
'''test_add_http ensures that http is added to a url
'''
print("Testing utils header functions...")
from utils import parse_headers
from utils import basic_auth_header
# If we don't give headers, and no default, should get {}
print("Case 1: Don't give headers, return empty dictionary")
empty_dict = parse_headers(default_header=False)
self.assertEqual(empty_dict, dict())
# If we ask for default, should get something back
print("Case 2: ask for default headers...")
headers = parse_headers(default_header=True)
for field in ["Accept", "Content-Type"]:
self.assertTrue(field in headers)
# Can we add a header?
print("Case 3: add a custom header")
new_header = {"cookies": "nom"}
headers = parse_headers(default_header=True, headers=new_header)
for field in ["Accept", "Content-Type", "cookies"]:
self.assertTrue(field in headers)
# Basic auth header
print(
"Case 4: basic_auth_header - ask for custom authentication header")
auth = basic_auth_header(username='******', password='******')
self.assertEqual(auth['Authorization'],
'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def __process(self, data):
"""
Process data to fill properties of Burp object.
"""
self.host = data.get('host', None)
self.ip_address = data.get('ip_address', None)
self.time = data.get('time', None)
self.request.update({
'method':
data['request'].get('method'),
'path':
data['request'].get('path'),
'version':
data['request'].get('version'),
'headers':
parse_headers(data['request'].get('headers')),
'body':
data['request'].get('body', ""),
})
self.response.update({
'version':
data['response'].get('version'),
'status':
int(data['response'].get('status', 0)),
'reason':
data['response'].get('reason'),
'headers':
parse_headers(data['response'].get('headers')),
'body':
data['response'].get('body', ""),
})
self.url = urlparse(urljoin(self.host, self.request.get('path')))
self.parameters = parse_parameters(self)
# During parsing, we may parse an extra CRLF or two. So to account
# for that, we'll just grab the actual content-length from the
# HTTP header and slice the request/response body appropriately.
if self.get_response_header('Content-Length'):
content_length = int(self.get_response_header('Content-Length'))
if len(self) != content_length:
#LOGGER.debug("Response content-length differs by %d" % (len(self) - content_length))
self.response['body'] = self.response['body'][:content_length]
if self.get_request_header('Content-Length'):
content_length = int(self.get_request_header('Content-Length'))
if len(self.get_request_body()) != content_length and 'amf' not in \
self.get_request_header('Content-Type'):
#LOGGER.debug("Request content-length differs by %d" % (len(self.get_request_body()) - content_length))
self.request['body'] = self.request['body'][:content_length]
def __process(self, data):
"""
Process data to fill properties of Burp object.
"""
self.host = data.get('host', None)
self.ip_address = data.get('ip_address', None)
self.time = data.get('time', None)
self.request.update({
'method':
data['request'].get('method'),
'path':
data['request'].get('path'),
'version':
data['request'].get('version'),
'headers':
parse_headers(data['request'].get('headers')),
'body':
data['request'].get('body', ""),
})
self.response.update({
'version':
data['response'].get('version'),
'status':
int(data['response'].get('status', 0)),
'reason':
data['response'].get('reason'),
'headers':
parse_headers(data['response'].get('headers')),
'body':
data['response'].get('body', ""),
})
self.url = urlparse(urljoin(self.host, self.request.get('path')))
self.parameters = parse_parameters(self)
# During parsing, we may parse an extra CRLF or two. So to account
# for that, we'll just grab the actual content-length from the
# HTTP header and slice the request/response body appropriately.
if self.get_response_header('Content-Length'):
content_length = int(self.get_response_header('Content-Length'))
if len(self) != content_length:
#LOGGER.debug("Response content-length differs by %d" % (len(self) - content_length))
self.response['body'] = self.response['body'][:content_length]
if self.get_request_header('Content-Length'):
content_length = int(self.get_request_header('Content-Length'))
if len(self.get_request_body()) != content_length and 'amf' not in \
self.get_request_header('Content-Type'):
#LOGGER.debug("Request content-length differs by %d" % (len(self.get_request_body()) - content_length))
self.request['body'] = self.request['body'][:content_length]
def proxy(data: bytes, sock: socket, from_client: bool) -> bytes:
"""
Function that handles the requests that are passed to the proxy and changes the data / header on-the-fly.
:param data: raw data e.g. "GET /index.html HTTP/1.1\r\n..."
:param sock: the socket object for this connection
:param from_client: True if traffic coming from client
:return: raw (possible changed) data
"""
try:
request_key = sock.getpeername()[1]
if from_client:
data_split = data.split(b'\r\n\r\n')
raw_head = data_split[0]
headers = parse_headers(raw_head)
request_to_response[request_key] = [None, None, None]
if b'user-agent' in headers:
request_to_response[request_key][0] = headers[b'user-agent']
if b'host' in headers:
requested = b'http://' + headers[b'host'] + raw_head.split(
b'\r\n')[0].split()[1]
request_to_response[request_key][1] = requested
if b'referer' in headers:
request_to_response[request_key][2] = headers[b'referer']
return data
else:
if not data.startswith(b'HTTP'):
return data
data_split = data.split(b'\r\n\r\n')
raw_head = data_split[0]
raw_body = b'\r\n\r\n'.join(data_split[1:])
headers = parse_headers(raw_head)
if b'content-type' not in headers:
return data
if not headers[b'content-type'].lower().startswith(b'text/html'):
return data
new_headers = retrofit_headers(headers, request_key)
raw_response_head = [raw_head.split(b'\r\n')[0]]
for name, value in new_headers.items():
raw_response_head.append(name + b': ' + value)
raw_response_head = b'\r\n'.join(raw_response_head)
return raw_response_head + b'\r\n\r\n' + raw_body
except Exception as e:
print('Unexpected Exception', e)
traceback.print_exc()
return data
def proxy(data: bytes, input_url: str) -> FrameCheckResponse:
frame_response = FrameCheckResponse()
raw_response_head = list()
raw_response_head = []
try:
if not data.startswith(b'HTTP'):
return data
data_split = data.split(b'\r\n\r\n')
raw_head = data_split[0]
raw_body = b'\r\n\r\n'.join(data_split[1:])
headers = parse_headers(raw_head)
frame_response = retrofit_headers(headers, input_url)
new_headers = frame_response.new_header_bytes
if frame_response.flag_code['no_policy'] == 0:
for name, value in new_headers.items():
raw_response_head.append(name + b': ' + value)
raw_response_head = b'\r\n'.join(raw_response_head)
frame_response.new_header_bytes = raw_response_head + b'\r\n\r\n' + raw_body
return frame_response
except Exception as e:
print('Unexpected Exception', e)
traceback.print_exc()
frame_response.new_header_bytes = data
return frame_response
async def download(request):
"""
:param path: Filepath
"""
host, port, username, password = parse_headers(request,
default_user=None,
default_port=22)
path = request.query.get('path', '')
if not path:
raise MissingMandatoryQueryParameter('path')
try:
async with asyncssh.connect(host,
port=port,
username=username,
password=password,
known_hosts=None) as conn:
async with conn.start_sftp_client() as sftp:
async with sftp.open(path, 'rb') as fp:
response = web.StreamResponse()
response.content_type = 'application/octet-stream'
await response.prepare(request)
chunk = await fp.read(CHUNK_SIZE)
while chunk:
await response.write(chunk)
chunk = await fp.read(CHUNK_SIZE)
return response
except asyncssh.misc.Error as exc:
raise AsyncsshError(exc)
except OSError:
raise ServerUnreachable
async def ls(request):
"""
:param path: (optional) Path to list
:param extension: (optional) Filter by extension
"""
host, port, username, password = parse_headers(request,
default_user=None,
default_port=22)
path = request.query.get('path', '')
path = path.rstrip('/') + '/'
extension = request.query.get('extension', '')
try:
async with asyncssh.connect(host,
port=port,
username=username,
password=password,
known_hosts=None) as conn:
async with conn.start_sftp_client() as sftp:
files = [
f'{path}{f}' for f in await sftp.listdir(path)
if f not in ('.', '..') and (
not extension or f.endswith(extension))
]
return web.json_response(files)
except asyncssh.misc.Error as exc:
raise AsyncsshError(exc)
except OSError:
raise ServerUnreachable
async def _parse_request(self, reader, length=65536):
request = await reader.read(length)
headers = parse_headers(request)
if headers['Method'] == 'POST' and request.endswith(b'\r\n\r\n'):
# For aiohttp. POST data returns on second reading
request += await reader.read(length)
return request, headers
async def download(request):
host, port, login, password = parse_headers(request)
path = request.query.get('path')
if not path:
raise MissingMandatoryQueryParameter('path')
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
ftp_stream = await client.download_stream(path)
response = web.StreamResponse()
response.content_type = 'application/octet-stream'
await response.prepare(request)
async for chunk in ftp_stream.iter_by_block():
await response.write(chunk)
return response
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
async def ls(request):
"""ftp LS command
Optional query params:
path: directory to list (defaults to "/")
recursive: recurse down folders (defaults to "false")
"""
host, port, login, password = parse_headers(request)
root_path = request.query.get('path', '/')
recursive = request.query.get('recursive', 'false') == 'true'
extension = request.query.get('extension')
files = []
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
async for path, info in client.list(root_path,
recursive=recursive):
if extension is None or path.suffix == extension:
files.append(str(path))
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
return web.json_response(files)
def gen_user(current_server_session):
""" Returns the current user on the page as our Amplitude User Object"""
data = utils.parse_headers(current_server_session.ws.request)
user_data = {"has_precise_ip": False}
if "user_public_data" in data["Cookie"].keys():
for key in data["Cookie"]["user_public_data"].keys():
user_data[key] = data["Cookie"]["user_public_data"][key]
user_data["has_precise_ip"] = True
else:
user_data["ip"] = data["Remote_ip"]
if "user_unique_id" in data["Cookie"].keys():
user_data["user_id"] = data["Cookie"]["user_unique_id"]
else:
user_data["user_id"] = "unknown_user_placeholder"
for inkey in data.keys():
if inkey[:3] == "ua_":
user_data[inkey] = data[inkey]
return Amplitude_user(os.getenv("AMPLITUDE_KEY"), user_data)
def send_backend_request(addr, port, request):
body = request['body']
env = request['env']
path = request['path']
headers = request['headers']
conn = httplib.HTTPConnection('%s:%s' % (addr, port))
conn.request(env['REQUEST_METHOD'],
path, body, headers)
res = conn.getresponse()
headers_out = parse_headers(res.msg)
status = '%s %s' % (res.status, res.reason)
length = res.getheader('content-length')
# TODO: This shouldn't really read in all the content at once
if length is not None:
body = res.read(int(length))
else:
body = res.read()
conn.close()
return (status, headers_out, body)
async def ping(request):
"""test FTP connection by sending a minimal LS command
returns "pong" on success
"""
host, port, login, password = parse_headers(request)
try:
async with aioftp.ClientSession(host,
port,
login,
password,
socket_timeout=FTP_TIMEOUT,
path_timeout=FTP_TIMEOUT) as client:
async for _ in client.list('/'): # noqa
# Iterate once if any result, only list command matters not the results
break
return web.json_response({'success': True})
except (OSError, asyncio.TimeoutError, TimeoutError):
raise ServerUnreachable
except aioftp.errors.StatusCodeError as ftp_error:
raise AioftpError(ftp_error)
async def ping(request):
"""test SFTP connection by sending a minimal LS command
returns "pong" on success
"""
host, port, username, password = parse_headers(request,
default_user=None,
default_port=22)
try:
# known_hosts explicitly disabled
async with asyncssh.connect(host,
port=port,
username=username,
password=password,
known_hosts=None) as conn:
async with conn.start_sftp_client():
return web.json_response({'success': True})
except asyncssh.misc.Error as exc:
raise AsyncsshError(exc)
except OSError:
raise ServerUnreachable
import asyncio
import math
from typing import BinaryIO
import aiohttp
from utils import parse_headers
DEFAULT_HEADERS = parse_headers('''
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
sec-ch-ua: "Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"
sec-ch-ua-mobile: ?0
sec-fetch-dest: document
sec-fetch-mode: navigate
sec-fetch-site: same-origin
sec-fetch-user: ?1
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36
''')
class Downloader:
def __init__(self,
url: str,
fp: BinaryIO,
headers: dict = DEFAULT_HEADERS,
task_count=10) -> None:
self.session = aiohttp.ClientSession(headers=headers)
self.url = url
self.filesize = 0
self.task_count = task_count
self.tasks = []
self.fp = fp
