def setUp(self): self.engine = FakeEngine() self.rule = FollowRedirects(max_redirect=10) self.rule.set_engine(self.engine) self.response = Response(code=302, headers={"location": "https://www.example.com/"}) self.response.set_content(b"", at_eof=True) self.entry = Entry.create("http://example.com", response=self.response)
def config_hammertime(self): global_heuristics = [ DynamicTimeout(0.05, 2), RetryOnErrors(range(500, 503)), DeadHostDetection(threshold=200), ContentHashSampling(), ContentSampling(), ContentSimhashSampling() ] soft_404 = DetectSoft404() follow_redirects = FollowRedirects() reject_error_code = RejectStatusCode(range(400, 600)) heuristics = [ reject_error_code, RejectWebApplicationFirewall(), RejectCatchAllRedirect(), follow_redirects, soft_404, HashResponse(), SetExpectedMimeType(), RejectUnexpectedResponse() ] self.hammertime.heuristics.add_multiple(global_heuristics) self.hammertime.heuristics.add_multiple(heuristics) soft_404.child_heuristics.add_multiple(global_heuristics) follow_redirects.child_heuristics.add(reject_error_code) follow_redirects.child_heuristics.add_multiple(global_heuristics)
def setup_hammertime_heuristics(hammertime, *, user_agent=default_user_agent, vhost=None): # TODO Make sure rejecting 404 does not conflict with tomcat fake 404 detection. global heuristics_with_child dead_host_detection = DeadHostDetection(threshold=200) detect_soft_404 = DetectSoft404(distance_threshold=6) follow_redirects = FollowRedirects() heuristics_with_child = [ RejectCatchAllRedirect(), follow_redirects, RejectIgnoredQuery() ] hosts = (vhost, conf.target_host) if vhost is not None else conf.target_host global_heuristics = [ RejectStatusCode({404, 406, 502, 503}), DynamicTimeout(1.0, 5), RedirectLimiter(), FilterRequestFromURL(allowed_urls=hosts), IgnoreLargeBody(initial_limit=initial_limit) ] heuristics = [ StripTag('input'), StripTag('script'), detect_soft_404, RejectSoft404(), MatchString(), DetectBehaviorChange(buffer_size=100), LogBehaviorChange() ] # Dead host detection must be first to make sure there is no skipped after_headers hammertime.heuristics.add(dead_host_detection) hammertime.heuristics.add_multiple(global_heuristics) # Make sure follow redirect comes in before soft404 hammertime.heuristics.add_multiple(heuristics_with_child) hammertime.heuristics.add_multiple(heuristics) for heuristic in heuristics_with_child: heuristic.child_heuristics.add_multiple(global_heuristics) detect_soft_404.child_heuristics.add(StripTag('input')) detect_soft_404.child_heuristics.add(StripTag('script')) detect_soft_404.child_heuristics.add(dead_host_detection) detect_soft_404.child_heuristics.add(follow_redirects) add_http_header(hammertime, "User-Agent", user_agent) add_http_header(hammertime, "Host", vhost if vhost is not None else conf.target_host)
async def test_add_http_headers(self, loop): hammertime = HammerTime(loop=loop) config.heuristics_with_child = [ RejectCatchAllRedirect(), FollowRedirects() ] hammertime.heuristics.add_multiple(config.heuristics_with_child) hammertime.heuristics.add = MagicMock() for heuristic in config.heuristics_with_child: heuristic.child_heuristics.add = MagicMock() config.add_http_header(hammertime, "header", "value") set_header = hammertime.heuristics.add.call_args[0][0] self.assertEqual(set_header.name, "header") self.assertEqual(set_header.value, "value") for heuristic_with_child in config.heuristics_with_child: set_header = heuristic_with_child.child_heuristics.add.call_args[ 0][0] self.assertEqual(set_header.name, "header") self.assertEqual(set_header.value, "value")
class TestFollowRedirects(TestCase): def setUp(self): self.engine = FakeEngine() self.rule = FollowRedirects(max_redirect=10) self.rule.set_engine(self.engine) self.rule.child_heuristics = MagicMock() self.response = Response( code=302, headers={"location": "https://www.example.com/"}) self.response.set_content(b"", at_eof=True) self.entry = Entry.create("http://example.com", response=self.response) @async_test() async def test_on_request_successful_ignore_response_if_not_redirect(self): response = MagicMock(code=200, headers={}) entry = Entry.create("http://example.com", response=response) await self.rule.on_request_successful(entry) self.engine.mock.assert_not_called() @async_test() async def test_on_request_successful_perform_new_request_for_redirect( self): final_response = MagicMock(code=200, headers={}) self.engine.response = final_response await self.rule.on_request_successful(self.entry) self.engine.mock.assert_called_once_with( Entry.create("https://www.example.com/", response=final_response), self.rule.child_heuristics) @async_test() async def test_on_request_successful_keep_initial_request(self): initial_request = self.entry.request final_response = MagicMock(code=200, headers={}) self.engine.response = final_response await self.rule.on_request_successful(self.entry) self.assertEqual(self.entry.request, initial_request) @async_test() async def test_on_request_successful_set_final_response_as_entry_response( self): final_response = Response(code=200, headers={}) final_response.set_content(b"data", at_eof=True) self.engine.response = final_response await self.rule.on_request_successful(self.entry) self.assertEqual(self.entry.response, final_response) @async_test() async def test_on_request_successful_store_intermediate_entry_in_result( self): response = copy(self.response) final_response = Response(code=200, headers={}) final_response.set_content(b"response content", at_eof=True) self.engine.response = final_response await self.rule.on_request_successful(self.entry) expected = [ Entry.create(self.entry.request.url, method=self.entry.request.method, headers=self.entry.request.headers, response=response), Entry.create("https://www.example.com/", method='GET', headers={}, response=final_response) ] self.assertEqual(self.entry.result.redirects, expected) @async_test() async def test_on_request_successful_raise_reject_request_if_max_redirect_limit_reached( self): self.engine.response = Response( code=302, headers={"location": "http://example.com/"}) self.engine.response.set_content(b"data", at_eof=True) with self.assertRaises(RejectRequest): await self.rule.on_request_successful(self.entry) self.assertEqual(self.engine.mock.call_count, self.rule.max_redirect) self.assertEqual(len(self.entry.result.redirects), self.rule.max_redirect + 1) @async_test() async def test_on_request_successful_increment_stats_for_each_redirect( self): final_response = Response(code=200, headers={}) final_response.set_content(b"response content", at_eof=True) self.engine.response = final_response await self.rule.on_request_successful(self.entry) self.assertEqual(self.engine.stats.requested, 2) self.assertEqual(self.engine.stats.completed, 1) @async_test() async def test_on_request_successful_reject_request_if_no_location_in_response_header( self): self.engine.response = Response(code=302, headers={}) self.engine.response.set_content(b"data", at_eof=True) with self.assertRaises(RejectRequest): await self.rule.on_request_successful(self.entry) @async_test() async def test_relative_path_in_location(self): self.engine.mock.side_effect = [ Response(code=302, headers={"location": "splash/index.html"}), Response(code=201, headers={}), ] await self.rule.on_request_successful(self.entry) self.assertEqual(201, self.entry.response.code) self.assertEqual("https://www.example.com/splash/index.html", self.entry.result.redirects[-1].request.url) @async_test() async def test_relative_path_with_mutliple_redirects(self): self.engine.mock.side_effect = [ Response(code=302, headers={"location": "splash/index.html"}), Response(code=302, headers={"location": "index.php"}), Response(code=201, headers={}), ] await self.rule.on_request_successful(self.entry) self.assertEqual(201, self.entry.response.code) self.assertEqual("https://www.example.com/splash/index.php", self.entry.result.redirects[-1].request.url) @async_test() async def test_path_absolute(self): self.engine.mock.side_effect = [ Response(code=302, headers={"location": "splash/index.html"}), Response(code=302, headers={"location": "/index.php"}), Response(code=201, headers={}), ] await self.rule.on_request_successful(self.entry) self.assertEqual(201, self.entry.response.code) self.assertEqual("https://www.example.com/index.php", self.entry.result.redirects[-1].request.url) @async_test() async def test_full_different_domain(self): self.engine.mock.side_effect = [ Response( code=302, headers={"location": "http://example.org/splash/index.html"}), Response(code=302, headers={"location": "test"}), Response(code=201, headers={}), ] await self.rule.on_request_successful(self.entry) self.assertEqual(201, self.entry.response.code) self.assertEqual("http://example.org/splash/test", self.entry.result.redirects[-1].request.url) @async_test() async def test_on_request_successful_raise_exception_if_redirect_fail( self): engine = MagicMock() engine.perform = make_mocked_coro(raise_exception=StopRequest()) self.rule.set_engine(engine) with self.assertRaises(StopRequest): await self.rule.on_request_successful(self.entry)
def setup_hammertime_heuristics(hammertime, *, user_agent=default_user_agent, vhost=None, confirmation_factor=1, har_output_dir=None): global heuristics_with_child dead_host_detection = DeadHostDetection(threshold=200) detect_soft_404 = DetectSoft404(distance_threshold=6, confirmation_factor=confirmation_factor) follow_redirects = FollowRedirects() heuristics_with_child = [ RejectCatchAllRedirect(), follow_redirects, RejectIgnoredQuery() ] hosts = (vhost, conf.target_host) if vhost is not None else conf.target_host init_heuristics = [ SetHeader("User-Agent", user_agent), SetHeader("Host", vhost if vhost is not None else conf.target_host), ContentHashSampling(), ContentSampling(), ContentSimhashSampling(), dead_host_detection, RejectStatusCode({503, 508}, exception_class=StopRequest), StripTag('input'), StripTag('script') ] global_heuristics = [ RejectStatusCode({404, 406, 502}), RejectWebApplicationFirewall(), DynamicTimeout(1.0, 5), RedirectLimiter(), FilterRequestFromURL(allowed_urls=hosts), IgnoreLargeBody(initial_limit=initial_limit) ] # Dead host detection must be first to make sure there is no skipped after_headers hammertime.heuristics.add_multiple(init_heuristics) # General hammertime.heuristics.add_multiple(global_heuristics) hammertime.heuristics.add_multiple(heuristics_with_child) hammertime.heuristics.add_multiple([ detect_soft_404, MatchString(), ValidateEntry(), DetectBehaviorChange(buffer_size=100), LogBehaviorChange(), ValidateEntry(), ]) detect_soft_404.child_heuristics.add_multiple(init_heuristics) detect_soft_404.child_heuristics.add_multiple(heuristics_with_child) for heuristic in heuristics_with_child: heuristic.child_heuristics.add_multiple(init_heuristics) heuristic.child_heuristics.add_multiple(global_heuristics) if har_output_dir is not None: from tachyon.har import StoreHAR, FileWriter hammertime.heuristics.add(StoreHAR(writer=FileWriter(har_output_dir)))