def setUp(self):
self.engine = FakeEngine()
self.rule = FollowRedirects(max_redirect=10)
self.rule.set_engine(self.engine)
self.response = Response(code=302, headers={"location": "https://www.example.com/"})
self.response.set_content(b"", at_eof=True)
self.entry = Entry.create("http://example.com", response=self.response)
def config_hammertime(self):
global_heuristics = [
DynamicTimeout(0.05, 2),
RetryOnErrors(range(500, 503)),
DeadHostDetection(threshold=200),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling()
]
soft_404 = DetectSoft404()
follow_redirects = FollowRedirects()
reject_error_code = RejectStatusCode(range(400, 600))
heuristics = [
reject_error_code,
RejectWebApplicationFirewall(),
RejectCatchAllRedirect(), follow_redirects, soft_404,
HashResponse(),
SetExpectedMimeType(),
RejectUnexpectedResponse()
]
self.hammertime.heuristics.add_multiple(global_heuristics)
self.hammertime.heuristics.add_multiple(heuristics)
soft_404.child_heuristics.add_multiple(global_heuristics)
follow_redirects.child_heuristics.add(reject_error_code)
follow_redirects.child_heuristics.add_multiple(global_heuristics)
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None):
# TODO Make sure rejecting 404 does not conflict with tomcat fake 404 detection.
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
global_heuristics = [
RejectStatusCode({404, 406, 502, 503}),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
heuristics = [
StripTag('input'),
StripTag('script'), detect_soft_404,
RejectSoft404(),
MatchString(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange()
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add(dead_host_detection)
hammertime.heuristics.add_multiple(global_heuristics)
# Make sure follow redirect comes in before soft404
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple(heuristics)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(global_heuristics)
detect_soft_404.child_heuristics.add(StripTag('input'))
detect_soft_404.child_heuristics.add(StripTag('script'))
detect_soft_404.child_heuristics.add(dead_host_detection)
detect_soft_404.child_heuristics.add(follow_redirects)
add_http_header(hammertime, "User-Agent", user_agent)
add_http_header(hammertime, "Host",
vhost if vhost is not None else conf.target_host)
async def test_add_http_headers(self, loop):
hammertime = HammerTime(loop=loop)
config.heuristics_with_child = [
RejectCatchAllRedirect(),
FollowRedirects()
]
hammertime.heuristics.add_multiple(config.heuristics_with_child)
hammertime.heuristics.add = MagicMock()
for heuristic in config.heuristics_with_child:
heuristic.child_heuristics.add = MagicMock()
config.add_http_header(hammertime, "header", "value")
set_header = hammertime.heuristics.add.call_args[0][0]
self.assertEqual(set_header.name, "header")
self.assertEqual(set_header.value, "value")
for heuristic_with_child in config.heuristics_with_child:
set_header = heuristic_with_child.child_heuristics.add.call_args[
0][0]
self.assertEqual(set_header.name, "header")
self.assertEqual(set_header.value, "value")
class TestFollowRedirects(TestCase):
def setUp(self):
self.engine = FakeEngine()
self.rule = FollowRedirects(max_redirect=10)
self.rule.set_engine(self.engine)
self.rule.child_heuristics = MagicMock()
self.response = Response(
code=302, headers={"location": "https://www.example.com/"})
self.response.set_content(b"", at_eof=True)
self.entry = Entry.create("http://example.com", response=self.response)
@async_test()
async def test_on_request_successful_ignore_response_if_not_redirect(self):
response = MagicMock(code=200, headers={})
entry = Entry.create("http://example.com", response=response)
await self.rule.on_request_successful(entry)
self.engine.mock.assert_not_called()
@async_test()
async def test_on_request_successful_perform_new_request_for_redirect(
self):
final_response = MagicMock(code=200, headers={})
self.engine.response = final_response
await self.rule.on_request_successful(self.entry)
self.engine.mock.assert_called_once_with(
Entry.create("https://www.example.com/", response=final_response),
self.rule.child_heuristics)
@async_test()
async def test_on_request_successful_keep_initial_request(self):
initial_request = self.entry.request
final_response = MagicMock(code=200, headers={})
self.engine.response = final_response
await self.rule.on_request_successful(self.entry)
self.assertEqual(self.entry.request, initial_request)
@async_test()
async def test_on_request_successful_set_final_response_as_entry_response(
self):
final_response = Response(code=200, headers={})
final_response.set_content(b"data", at_eof=True)
self.engine.response = final_response
await self.rule.on_request_successful(self.entry)
self.assertEqual(self.entry.response, final_response)
@async_test()
async def test_on_request_successful_store_intermediate_entry_in_result(
self):
response = copy(self.response)
final_response = Response(code=200, headers={})
final_response.set_content(b"response content", at_eof=True)
self.engine.response = final_response
await self.rule.on_request_successful(self.entry)
expected = [
Entry.create(self.entry.request.url,
method=self.entry.request.method,
headers=self.entry.request.headers,
response=response),
Entry.create("https://www.example.com/",
method='GET',
headers={},
response=final_response)
]
self.assertEqual(self.entry.result.redirects, expected)
@async_test()
async def test_on_request_successful_raise_reject_request_if_max_redirect_limit_reached(
self):
self.engine.response = Response(
code=302, headers={"location": "http://example.com/"})
self.engine.response.set_content(b"data", at_eof=True)
with self.assertRaises(RejectRequest):
await self.rule.on_request_successful(self.entry)
self.assertEqual(self.engine.mock.call_count,
self.rule.max_redirect)
self.assertEqual(len(self.entry.result.redirects),
self.rule.max_redirect + 1)
@async_test()
async def test_on_request_successful_increment_stats_for_each_redirect(
self):
final_response = Response(code=200, headers={})
final_response.set_content(b"response content", at_eof=True)
self.engine.response = final_response
await self.rule.on_request_successful(self.entry)
self.assertEqual(self.engine.stats.requested, 2)
self.assertEqual(self.engine.stats.completed, 1)
@async_test()
async def test_on_request_successful_reject_request_if_no_location_in_response_header(
self):
self.engine.response = Response(code=302, headers={})
self.engine.response.set_content(b"data", at_eof=True)
with self.assertRaises(RejectRequest):
await self.rule.on_request_successful(self.entry)
@async_test()
async def test_relative_path_in_location(self):
self.engine.mock.side_effect = [
Response(code=302, headers={"location": "splash/index.html"}),
Response(code=201, headers={}),
]
await self.rule.on_request_successful(self.entry)
self.assertEqual(201, self.entry.response.code)
self.assertEqual("https://www.example.com/splash/index.html",
self.entry.result.redirects[-1].request.url)
@async_test()
async def test_relative_path_with_mutliple_redirects(self):
self.engine.mock.side_effect = [
Response(code=302, headers={"location": "splash/index.html"}),
Response(code=302, headers={"location": "index.php"}),
Response(code=201, headers={}),
]
await self.rule.on_request_successful(self.entry)
self.assertEqual(201, self.entry.response.code)
self.assertEqual("https://www.example.com/splash/index.php",
self.entry.result.redirects[-1].request.url)
@async_test()
async def test_path_absolute(self):
self.engine.mock.side_effect = [
Response(code=302, headers={"location": "splash/index.html"}),
Response(code=302, headers={"location": "/index.php"}),
Response(code=201, headers={}),
]
await self.rule.on_request_successful(self.entry)
self.assertEqual(201, self.entry.response.code)
self.assertEqual("https://www.example.com/index.php",
self.entry.result.redirects[-1].request.url)
@async_test()
async def test_full_different_domain(self):
self.engine.mock.side_effect = [
Response(
code=302,
headers={"location": "http://example.org/splash/index.html"}),
Response(code=302, headers={"location": "test"}),
Response(code=201, headers={}),
]
await self.rule.on_request_successful(self.entry)
self.assertEqual(201, self.entry.response.code)
self.assertEqual("http://example.org/splash/test",
self.entry.result.redirects[-1].request.url)
@async_test()
async def test_on_request_successful_raise_exception_if_redirect_fail(
self):
engine = MagicMock()
engine.perform = make_mocked_coro(raise_exception=StopRequest())
self.rule.set_engine(engine)
with self.assertRaises(StopRequest):
await self.rule.on_request_successful(self.entry)
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None,
confirmation_factor=1,
har_output_dir=None):
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6,
confirmation_factor=confirmation_factor)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
init_heuristics = [
SetHeader("User-Agent", user_agent),
SetHeader("Host", vhost if vhost is not None else conf.target_host),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling(), dead_host_detection,
RejectStatusCode({503, 508}, exception_class=StopRequest),
StripTag('input'),
StripTag('script')
]
global_heuristics = [
RejectStatusCode({404, 406, 502}),
RejectWebApplicationFirewall(),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add_multiple(init_heuristics)
# General
hammertime.heuristics.add_multiple(global_heuristics)
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple([
detect_soft_404,
MatchString(),
ValidateEntry(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange(),
ValidateEntry(),
])
detect_soft_404.child_heuristics.add_multiple(init_heuristics)
detect_soft_404.child_heuristics.add_multiple(heuristics_with_child)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(init_heuristics)
heuristic.child_heuristics.add_multiple(global_heuristics)
if har_output_dir is not None:
from tachyon.har import StoreHAR, FileWriter
hammertime.heuristics.add(StoreHAR(writer=FileWriter(har_output_dir)))