def test_headers(self): '''test_add_http ensures that http is added to a url ''' print("Testing utils header functions...") from utils import parse_headers from utils import basic_auth_header # If we don't give headers, and no default, should get {} print("Case 1: Don't give headers, return empty dictionary") empty_dict = parse_headers(default_header=False) self.assertEqual(empty_dict,dict()) # If we ask for default, should get something back print("Case 2: ask for default headers...") headers = parse_headers(default_header=True) for field in ["Accept","Content-Type"]: self.assertTrue(field in headers) # Can we add a header? print("Case 3: add a custom header") new_header = {"cookies":"nom"} headers = parse_headers(default_header=True, headers=new_header) for field in ["Accept","Content-Type","cookies"]: self.assertTrue(field in headers) # Basic auth header print("Case 4: basic_auth_header - ask for custom authentication header") auth = basic_auth_header(username='******', password='******') self.assertEqual(auth['Authorization'], 'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def test_headers(self): '''test_add_http ensures that http is added to a url ''' print("Testing utils header functions...") from utils import parse_headers from utils import basic_auth_header # If we don't give headers, and no default, should get {} print("Case 1: Don't give headers, return empty dictionary") empty_dict = parse_headers(default_header=False) self.assertEqual(empty_dict, dict()) # If we ask for default, should get something back print("Case 2: ask for default headers...") headers = parse_headers(default_header=True) for field in ["Accept", "Content-Type"]: self.assertTrue(field in headers) # Can we add a header? print("Case 3: add a custom header") new_header = {"cookies": "nom"} headers = parse_headers(default_header=True, headers=new_header) for field in ["Accept", "Content-Type", "cookies"]: self.assertTrue(field in headers) # Basic auth header print( "Case 4: basic_auth_header - ask for custom authentication header") auth = basic_auth_header(username='******', password='******') self.assertEqual(auth['Authorization'], 'Basic dmFuZXNzYTpwYW5jYWtlcw==')
def __process(self, data): """ Process data to fill properties of Burp object. """ self.host = data.get('host', None) self.ip_address = data.get('ip_address', None) self.time = data.get('time', None) self.request.update({ 'method': data['request'].get('method'), 'path': data['request'].get('path'), 'version': data['request'].get('version'), 'headers': parse_headers(data['request'].get('headers')), 'body': data['request'].get('body', ""), }) self.response.update({ 'version': data['response'].get('version'), 'status': int(data['response'].get('status', 0)), 'reason': data['response'].get('reason'), 'headers': parse_headers(data['response'].get('headers')), 'body': data['response'].get('body', ""), }) self.url = urlparse(urljoin(self.host, self.request.get('path'))) self.parameters = parse_parameters(self) # During parsing, we may parse an extra CRLF or two. So to account # for that, we'll just grab the actual content-length from the # HTTP header and slice the request/response body appropriately. if self.get_response_header('Content-Length'): content_length = int(self.get_response_header('Content-Length')) if len(self) != content_length: #LOGGER.debug("Response content-length differs by %d" % (len(self) - content_length)) self.response['body'] = self.response['body'][:content_length] if self.get_request_header('Content-Length'): content_length = int(self.get_request_header('Content-Length')) if len(self.get_request_body()) != content_length and 'amf' not in \ self.get_request_header('Content-Type'): #LOGGER.debug("Request content-length differs by %d" % (len(self.get_request_body()) - content_length)) self.request['body'] = self.request['body'][:content_length]
def proxy(data: bytes, sock: socket, from_client: bool) -> bytes: """ Function that handles the requests that are passed to the proxy and changes the data / header on-the-fly. :param data: raw data e.g. "GET /index.html HTTP/1.1\r\n..." :param sock: the socket object for this connection :param from_client: True if traffic coming from client :return: raw (possible changed) data """ try: request_key = sock.getpeername()[1] if from_client: data_split = data.split(b'\r\n\r\n') raw_head = data_split[0] headers = parse_headers(raw_head) request_to_response[request_key] = [None, None, None] if b'user-agent' in headers: request_to_response[request_key][0] = headers[b'user-agent'] if b'host' in headers: requested = b'http://' + headers[b'host'] + raw_head.split( b'\r\n')[0].split()[1] request_to_response[request_key][1] = requested if b'referer' in headers: request_to_response[request_key][2] = headers[b'referer'] return data else: if not data.startswith(b'HTTP'): return data data_split = data.split(b'\r\n\r\n') raw_head = data_split[0] raw_body = b'\r\n\r\n'.join(data_split[1:]) headers = parse_headers(raw_head) if b'content-type' not in headers: return data if not headers[b'content-type'].lower().startswith(b'text/html'): return data new_headers = retrofit_headers(headers, request_key) raw_response_head = [raw_head.split(b'\r\n')[0]] for name, value in new_headers.items(): raw_response_head.append(name + b': ' + value) raw_response_head = b'\r\n'.join(raw_response_head) return raw_response_head + b'\r\n\r\n' + raw_body except Exception as e: print('Unexpected Exception', e) traceback.print_exc() return data
def proxy(data: bytes, input_url: str) -> FrameCheckResponse: frame_response = FrameCheckResponse() raw_response_head = list() raw_response_head = [] try: if not data.startswith(b'HTTP'): return data data_split = data.split(b'\r\n\r\n') raw_head = data_split[0] raw_body = b'\r\n\r\n'.join(data_split[1:]) headers = parse_headers(raw_head) frame_response = retrofit_headers(headers, input_url) new_headers = frame_response.new_header_bytes if frame_response.flag_code['no_policy'] == 0: for name, value in new_headers.items(): raw_response_head.append(name + b': ' + value) raw_response_head = b'\r\n'.join(raw_response_head) frame_response.new_header_bytes = raw_response_head + b'\r\n\r\n' + raw_body return frame_response except Exception as e: print('Unexpected Exception', e) traceback.print_exc() frame_response.new_header_bytes = data return frame_response
async def download(request): """ :param path: Filepath """ host, port, username, password = parse_headers(request, default_user=None, default_port=22) path = request.query.get('path', '') if not path: raise MissingMandatoryQueryParameter('path') try: async with asyncssh.connect(host, port=port, username=username, password=password, known_hosts=None) as conn: async with conn.start_sftp_client() as sftp: async with sftp.open(path, 'rb') as fp: response = web.StreamResponse() response.content_type = 'application/octet-stream' await response.prepare(request) chunk = await fp.read(CHUNK_SIZE) while chunk: await response.write(chunk) chunk = await fp.read(CHUNK_SIZE) return response except asyncssh.misc.Error as exc: raise AsyncsshError(exc) except OSError: raise ServerUnreachable
async def ls(request): """ :param path: (optional) Path to list :param extension: (optional) Filter by extension """ host, port, username, password = parse_headers(request, default_user=None, default_port=22) path = request.query.get('path', '') path = path.rstrip('/') + '/' extension = request.query.get('extension', '') try: async with asyncssh.connect(host, port=port, username=username, password=password, known_hosts=None) as conn: async with conn.start_sftp_client() as sftp: files = [ f'{path}{f}' for f in await sftp.listdir(path) if f not in ('.', '..') and ( not extension or f.endswith(extension)) ] return web.json_response(files) except asyncssh.misc.Error as exc: raise AsyncsshError(exc) except OSError: raise ServerUnreachable
async def _parse_request(self, reader, length=65536): request = await reader.read(length) headers = parse_headers(request) if headers['Method'] == 'POST' and request.endswith(b'\r\n\r\n'): # For aiohttp. POST data returns on second reading request += await reader.read(length) return request, headers
async def download(request): host, port, login, password = parse_headers(request) path = request.query.get('path') if not path: raise MissingMandatoryQueryParameter('path') try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: ftp_stream = await client.download_stream(path) response = web.StreamResponse() response.content_type = 'application/octet-stream' await response.prepare(request) async for chunk in ftp_stream.iter_by_block(): await response.write(chunk) return response except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error)
async def ls(request): """ftp LS command Optional query params: path: directory to list (defaults to "/") recursive: recurse down folders (defaults to "false") """ host, port, login, password = parse_headers(request) root_path = request.query.get('path', '/') recursive = request.query.get('recursive', 'false') == 'true' extension = request.query.get('extension') files = [] try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: async for path, info in client.list(root_path, recursive=recursive): if extension is None or path.suffix == extension: files.append(str(path)) except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error) return web.json_response(files)
def gen_user(current_server_session): """ Returns the current user on the page as our Amplitude User Object""" data = utils.parse_headers(current_server_session.ws.request) user_data = {"has_precise_ip": False} if "user_public_data" in data["Cookie"].keys(): for key in data["Cookie"]["user_public_data"].keys(): user_data[key] = data["Cookie"]["user_public_data"][key] user_data["has_precise_ip"] = True else: user_data["ip"] = data["Remote_ip"] if "user_unique_id" in data["Cookie"].keys(): user_data["user_id"] = data["Cookie"]["user_unique_id"] else: user_data["user_id"] = "unknown_user_placeholder" for inkey in data.keys(): if inkey[:3] == "ua_": user_data[inkey] = data[inkey] return Amplitude_user(os.getenv("AMPLITUDE_KEY"), user_data)
def send_backend_request(addr, port, request): body = request['body'] env = request['env'] path = request['path'] headers = request['headers'] conn = httplib.HTTPConnection('%s:%s' % (addr, port)) conn.request(env['REQUEST_METHOD'], path, body, headers) res = conn.getresponse() headers_out = parse_headers(res.msg) status = '%s %s' % (res.status, res.reason) length = res.getheader('content-length') # TODO: This shouldn't really read in all the content at once if length is not None: body = res.read(int(length)) else: body = res.read() conn.close() return (status, headers_out, body)
async def ping(request): """test FTP connection by sending a minimal LS command returns "pong" on success """ host, port, login, password = parse_headers(request) try: async with aioftp.ClientSession(host, port, login, password, socket_timeout=FTP_TIMEOUT, path_timeout=FTP_TIMEOUT) as client: async for _ in client.list('/'): # noqa # Iterate once if any result, only list command matters not the results break return web.json_response({'success': True}) except (OSError, asyncio.TimeoutError, TimeoutError): raise ServerUnreachable except aioftp.errors.StatusCodeError as ftp_error: raise AioftpError(ftp_error)
async def ping(request): """test SFTP connection by sending a minimal LS command returns "pong" on success """ host, port, username, password = parse_headers(request, default_user=None, default_port=22) try: # known_hosts explicitly disabled async with asyncssh.connect(host, port=port, username=username, password=password, known_hosts=None) as conn: async with conn.start_sftp_client(): return web.json_response({'success': True}) except asyncssh.misc.Error as exc: raise AsyncsshError(exc) except OSError: raise ServerUnreachable
import asyncio import math from typing import BinaryIO import aiohttp from utils import parse_headers DEFAULT_HEADERS = parse_headers(''' accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 sec-ch-ua: "Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99" sec-ch-ua-mobile: ?0 sec-fetch-dest: document sec-fetch-mode: navigate sec-fetch-site: same-origin sec-fetch-user: ?1 user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36 ''') class Downloader: def __init__(self, url: str, fp: BinaryIO, headers: dict = DEFAULT_HEADERS, task_count=10) -> None: self.session = aiohttp.ClientSession(headers=headers) self.url = url self.filesize = 0 self.task_count = task_count self.tasks = [] self.fp = fp