def config_hammertime(self):
global_heuristics = [
DynamicTimeout(0.05, 2),
RetryOnErrors(range(500, 503)),
DeadHostDetection(threshold=200),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling()
]
soft_404 = DetectSoft404()
follow_redirects = FollowRedirects()
reject_error_code = RejectStatusCode(range(400, 600))
heuristics = [
reject_error_code,
RejectWebApplicationFirewall(),
RejectCatchAllRedirect(), follow_redirects, soft_404,
HashResponse(),
SetExpectedMimeType(),
RejectUnexpectedResponse()
]
self.hammertime.heuristics.add_multiple(global_heuristics)
self.hammertime.heuristics.add_multiple(heuristics)
soft_404.child_heuristics.add_multiple(global_heuristics)
follow_redirects.child_heuristics.add(reject_error_code)
follow_redirects.child_heuristics.add_multiple(global_heuristics)
async def test_reject_nothing(self):
r = RejectStatusCode()
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(200, {})))
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(302, {})))
async def test_explicit_abandon_obtained_when_requested(self, loop):
h = HammerTime(loop=loop, request_engine=FakeEngine(), retry_count=2)
h.heuristics.add(RejectStatusCode(range(0, 600))) # Everything
future = h.request("http://example.com/1")
with self.assertRaises(RejectRequest):
await future
self.assertEqual(0, h.stats.retries)
async def test_file_exists(hammertime):
""" Test for file existence using http codes and computed 404 """
fetcher = FileFetcher(conf.base_url, hammertime)
generator = FileGenerator()
database.valid_paths = generator.generate_files()
textutils.output_info('Probing ' + str(len(database.valid_paths)) +
' files')
if len(database.valid_paths) > 0:
hammertime.heuristics.add(RejectStatusCode({401, 403}))
await fetcher.fetch_files(database.valid_paths)
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None):
# TODO Make sure rejecting 404 does not conflict with tomcat fake 404 detection.
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
global_heuristics = [
RejectStatusCode({404, 406, 502, 503}),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
heuristics = [
StripTag('input'),
StripTag('script'), detect_soft_404,
RejectSoft404(),
MatchString(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange()
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add(dead_host_detection)
hammertime.heuristics.add_multiple(global_heuristics)
# Make sure follow redirect comes in before soft404
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple(heuristics)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(global_heuristics)
detect_soft_404.child_heuristics.add(StripTag('input'))
detect_soft_404.child_heuristics.add(StripTag('script'))
detect_soft_404.child_heuristics.add(dead_host_detection)
detect_soft_404.child_heuristics.add(follow_redirects)
add_http_header(hammertime, "User-Agent", user_agent)
add_http_header(hammertime, "Host",
vhost if vhost is not None else conf.target_host)
async def test_do_not_reject_outside_specified_ranges(self):
r = RejectStatusCode(range(400, 410), range(500, 700))
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(200, {})))
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(302, {})))
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(410, {})))
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(460, {})))
async def test_file_exists(hammertime, accumulator, skip_root=False):
""" Test for file existence using http codes and computed 404 """
check_closed(hammertime)
fetcher = FileFetcher(conf.base_url, hammertime, accumulator=accumulator)
generator = FileGenerator()
files_to_fetch = generator.generate_files(skip_root=skip_root)
count = len(files_to_fetch)
textutils.output_info('Probing %d files' % count)
if len(database.valid_paths) > 0:
hammertime.heuristics.add(RejectStatusCode({401, 403}))
await fetcher.fetch_files(files_to_fetch)
async def test_do_not_reject_reject_within_the_specified_ranges(self):
r = RejectStatusCode(range(400, 410), range(500, 700))
with self.assertRaises(RejectRequest):
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(400, {})))
with self.assertRaises(RejectRequest):
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(409, {})))
with self.assertRaises(RejectRequest):
await r.after_headers(
Entry.create("http://example.om/test",
response=StaticResponse(503, {})))
def setup_hammertime_heuristics(hammertime,
*,
user_agent=default_user_agent,
vhost=None,
confirmation_factor=1,
har_output_dir=None):
global heuristics_with_child
dead_host_detection = DeadHostDetection(threshold=200)
detect_soft_404 = DetectSoft404(distance_threshold=6,
confirmation_factor=confirmation_factor)
follow_redirects = FollowRedirects()
heuristics_with_child = [
RejectCatchAllRedirect(), follow_redirects,
RejectIgnoredQuery()
]
hosts = (vhost,
conf.target_host) if vhost is not None else conf.target_host
init_heuristics = [
SetHeader("User-Agent", user_agent),
SetHeader("Host", vhost if vhost is not None else conf.target_host),
ContentHashSampling(),
ContentSampling(),
ContentSimhashSampling(), dead_host_detection,
RejectStatusCode({503, 508}, exception_class=StopRequest),
StripTag('input'),
StripTag('script')
]
global_heuristics = [
RejectStatusCode({404, 406, 502}),
RejectWebApplicationFirewall(),
DynamicTimeout(1.0, 5),
RedirectLimiter(),
FilterRequestFromURL(allowed_urls=hosts),
IgnoreLargeBody(initial_limit=initial_limit)
]
# Dead host detection must be first to make sure there is no skipped after_headers
hammertime.heuristics.add_multiple(init_heuristics)
# General
hammertime.heuristics.add_multiple(global_heuristics)
hammertime.heuristics.add_multiple(heuristics_with_child)
hammertime.heuristics.add_multiple([
detect_soft_404,
MatchString(),
ValidateEntry(),
DetectBehaviorChange(buffer_size=100),
LogBehaviorChange(),
ValidateEntry(),
])
detect_soft_404.child_heuristics.add_multiple(init_heuristics)
detect_soft_404.child_heuristics.add_multiple(heuristics_with_child)
for heuristic in heuristics_with_child:
heuristic.child_heuristics.add_multiple(init_heuristics)
heuristic.child_heuristics.add_multiple(global_heuristics)
if har_output_dir is not None:
from tachyon.har import StoreHAR, FileWriter
hammertime.heuristics.add(StoreHAR(writer=FileWriter(har_output_dir)))