def config_hammertime(self):
global_heuristics = [
DynamicTimeout(0.05, 2),
RetryOnErrors(range(500, 503)),
DeadHostDetection(threshold=200),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling()
]
soft_404 = DetectSoft404()
follow_redirects = FollowRedirects()
reject_error_code = RejectStatusCode(range(400, 600))
heuristics = [
reject_error_code,
RejectWebApplicationFirewall(),
RejectCatchAllRedirect(), follow_redirects, soft_404,
HashResponse(),
SetExpectedMimeType(),
RejectUnexpectedResponse()
]
self.hammertime.heuristics.add_multiple(global_heuristics)
self.hammertime.heuristics.add_multiple(heuristics)
soft_404.child_heuristics.add_multiple(global_heuristics)
follow_redirects.child_heuristics.add(reject_error_code)
follow_redirects.child_heuristics.add_multiple(global_heuristics)
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None):
# TODO Make sure rejecting 404 does not conflict with tomcat fake 404 detection.
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
global_heuristics = [
RejectStatusCode({404, 406, 502, 503}),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
heuristics = [
StripTag('input'),
StripTag('script'), detect_soft_404,
RejectSoft404(),
MatchString(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange()
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add(dead_host_detection)
hammertime.heuristics.add_multiple(global_heuristics)
# Make sure follow redirect comes in before soft404
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple(heuristics)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(global_heuristics)
detect_soft_404.child_heuristics.add(StripTag('input'))
detect_soft_404.child_heuristics.add(StripTag('script'))
detect_soft_404.child_heuristics.add(dead_host_detection)
detect_soft_404.child_heuristics.add(follow_redirects)
add_http_header(hammertime, "User-Agent", user_agent)
add_http_header(hammertime, "Host",
vhost if vhost is not None else conf.target_host)
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None,
confirmation_factor=1,
har_output_dir=None):
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6,
confirmation_factor=confirmation_factor)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
init_heuristics = [
SetHeader("User-Agent", user_agent),
SetHeader("Host", vhost if vhost is not None else conf.target_host),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling(), dead_host_detection,
RejectStatusCode({503, 508}, exception_class=StopRequest),
StripTag('input'),
StripTag('script')
]
global_heuristics = [
RejectStatusCode({404, 406, 502}),
RejectWebApplicationFirewall(),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add_multiple(init_heuristics)
# General
hammertime.heuristics.add_multiple(global_heuristics)
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple([
detect_soft_404,
MatchString(),
ValidateEntry(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange(),
ValidateEntry(),
])
detect_soft_404.child_heuristics.add_multiple(init_heuristics)
detect_soft_404.child_heuristics.add_multiple(heuristics_with_child)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(init_heuristics)
heuristic.child_heuristics.add_multiple(global_heuristics)
if har_output_dir is not None:
from tachyon.har import StoreHAR, FileWriter
hammertime.heuristics.add(StoreHAR(writer=FileWriter(har_output_dir)))
def setUp(self):
self.dead_host_detection = DeadHostDetection()
self.kb = KnowledgeBase()
self.dead_host_detection.set_kb(self.kb)
class TestDeadHostDetection(TestCase):
def setUp(self):
self.dead_host_detection = DeadHostDetection()
self.kb = KnowledgeBase()
self.dead_host_detection.set_kb(self.kb)
@async_test()
async def test_before_request_init_host_state(self):
entry = Entry.create("http://example.com/test")
await self.dead_host_detection.before_request(entry)
self.assertEqual(
self.dead_host_detection.hosts["example.com"]["timeout_requests"],
0)
@async_test()
async def test_before_request_raise_offline_host_exception_if_host_is_offline(
self):
entry = Entry.create("http://example.com/")
await self.dead_host_detection.before_request(entry)
self.dead_host_detection.dead_hosts = ["example.com"]
with self.assertRaises(OfflineHostException):
await self.dead_host_detection.before_request(entry)
@async_test()
async def test_after_headers_reset_timeout_requests(self):
self.dead_host_detection.hosts["example.com"] = {
"timeout_requests": 10
}
await self.dead_host_detection.after_headers(
Entry.create("http://example.com/"))
self.assertEqual(
self.dead_host_detection.hosts["example.com"]["timeout_requests"],
0)
@async_test()
async def test_on_timeout_increment_timeout_requests_for_host(self):
await self.dead_host_detection.before_request(
Entry.create("http://example.com/"))
await self.dead_host_detection.before_request(
Entry.create("http://www.test.com/"))
await self.dead_host_detection.before_request(
Entry.create("http://10.10.10.10:8080/"))
for i in range(2):
await self.dead_host_detection.on_timeout(
Entry.create("http://example.com/"))
await self.dead_host_detection.on_timeout(
Entry.create("http://www.test.com/"))
for i in range(3):
await self.dead_host_detection.on_timeout(
Entry.create("http://10.10.10.10:8080/"))
self.assertEqual(
self.dead_host_detection.hosts["example.com"]["timeout_requests"],
2)
self.assertEqual(
self.dead_host_detection.hosts["www.test.com"]["timeout_requests"],
1)
self.assertEqual(
self.dead_host_detection.hosts["10.10.10.10:8080"]
["timeout_requests"], 3)
@async_test()
async def test_on_timeout_raise_offline_host_exception_if_timeout_requests_exceed_threshold(
self):
self.dead_host_detection.threshold = 2
entry = Entry.create("http://example.com/")
await self.dead_host_detection.before_request(entry)
await self.dead_host_detection.on_timeout(
entry
) # only 1 request have timed out, no exception should be raised
with self.assertRaises(OfflineHostException):
await self.dead_host_detection.on_timeout(entry)
@async_test()
async def test_on_timeout_add_dead_host_to_dead_host_list(self):
entry = Entry.create("http://example.com/")
self.dead_host_detection.hosts["example.com"] = {
"timeout_requests": self.dead_host_detection.threshold
}
try:
await self.dead_host_detection.on_timeout(entry)
except OfflineHostException:
pass
self.assertIn("example.com", self.kb.dead_hosts)
@async_test()
async def test_on_host_unreachable_increment_timeout_requests_count(self):
entry = Entry.create("http://example.com/")
self.dead_host_detection.hosts["example.com"] = {"timeout_requests": 0}
await self.dead_host_detection.on_host_unreachable(entry)
self.assertEqual(
self.dead_host_detection.hosts["example.com"]["timeout_requests"],
1)
@async_test()
async def test_on_host_unreachable_raise_offline_host_exception_if_timeout_requests_count_exceed_threshold(
self):
entry = Entry.create("http://example.com/")
self.dead_host_detection.hosts["example.com"] = {
"timeout_requests": 49
}
with self.assertRaises(OfflineHostException):
await self.dead_host_detection.on_host_unreachable(entry)
self.assertIn("example.com", self.kb.dead_hosts)