def sample_root_404():
""" Get the root 404, this has to be done as soon as possible since plugins could use this information. """
manager = ThreadManager()
textutils.output_info('Benchmarking root 404')
for ext in conf.crafted_404_extensions:
random_file = str(uuid.uuid4())
path = conf.path_template.copy()
if path['url'] != '/':
path['url'] = '/' + random_file + ext
else:
path['url'] = random_file + ext
# Were not using the fetch cache for 404 sampling
database.fetch_queue.put(path)
# Forced bogus path check
random_file = str(uuid.uuid4())
path = conf.path_template.copy()
path['url'] = '/' + random_file + '/'
# Were not using the fetch cache for 404 sampling
database.fetch_queue.put(path)
workers = manager.spawn_workers(len(conf.crafted_404_extensions), FetchCrafted404Worker)
manager.wait_for_idle(workers, database.fetch_queue)
def test_file_exists():
""" Test for file existence using http codes and computed 404 """
manager = ThreadManager()
# Fill work queue with fetch list
for item in database.valid_paths:
dbutils.add_file_to_fetch_queue(item)
# Wait for initial valid path lookup
workers = manager.spawn_workers(conf.thread_count, TestFileExistsWorker)
manager.wait_for_idle(workers, database.fetch_queue)
def test_paths_exists():
"""
Test for path existence using http codes and computed 404
Spawn workers and turn off output for now, it would be irrelevant at this point.
"""
manager = ThreadManager()
# Fill work queue with fetch list
for path in database.paths:
dbutils.add_path_to_fetch_queue(path)
# Consider some file target as potential path
for file in database.files:
if not file.get('no_suffix'):
file_as_path = file.copy()
file_as_path['url'] = '/' + file_as_path['url']
dbutils.add_path_to_fetch_queue(file_as_path)
done_paths = []
recursion_depth = 0
textutils.output_debug('Cached: ' + str(database.path_cache))
while database.fetch_queue.qsize() > 0:
textutils.output_info('Probing ' + str(database.fetch_queue.qsize()) + ' paths')
# Wait for initial valid path lookup
workers = manager.spawn_workers(conf.thread_count, TestPathExistsWorker)
manager.wait_for_idle(workers, database.fetch_queue)
recursion_depth += 1
if not conf.recursive:
break
if recursion_depth >= conf.recursive_depth_limit:
break
for validpath in database.valid_paths:
if validpath['url'] == '/' or validpath['url'] in done_paths:
continue
done_paths.append(validpath['url'])
for path in database.paths:
if path['url'] in ('/', ''):
continue
path = path.copy()
path['url'] = validpath['url'] + path['url']
dbutils.add_path_to_fetch_queue(path)
textutils.output_info('Found ' + str(len(database.valid_paths)) + ' valid paths')
def sample_404_from_found_path():
""" For all existing path, compute the 404 CRC so we don't get trapped in a tarpit """
manager = ThreadManager()
for path in database.valid_paths:
textutils.output_debug("Path in valid path table: " + str(path))
for ext in conf.crafted_404_extensions:
path_clone = dict(path)
random_file = str(uuid.uuid4())
# We don't benchmark / since we do it first before path discovery
if path_clone['url'] != '/':
path_clone['url'] = path_clone['url'] + '/' + random_file + ext
database.fetch_queue.put(path_clone)
workers = manager.spawn_workers(conf.thread_count, FetchCrafted404Worker)
manager.wait_for_idle(workers, database.fetch_queue)
def sample_404_from_found_path():
""" For all existing path, compute the 404 CRC so we don't get trapped in a tarpit """
manager = ThreadManager()
for path in database.valid_paths:
textutils.output_debug("Path in valid path table: " + str(path))
for ext in conf.crafted_404_extensions:
path_clone = path.copy()
random_file = str(uuid.uuid4())
# We don't benchmark / since we do it first before path discovery
if path_clone['url'] != '/':
path_clone['url'] = path_clone['url'] + '/' + random_file + ext
# Were not using the fetch cache for 404 sampling
database.fetch_queue.put(path_clone)
workers = manager.spawn_workers(conf.thread_count, FetchCrafted404Worker)
manager.wait_for_idle(workers, database.fetch_queue)
def sample_root_404():
""" Get the root 404, this has to be done as soon as possible since plugins could use this information. """
manager = ThreadManager()
textutils.output_info('Benchmarking root 404')
for ext in conf.crafted_404_extensions:
random_file = str(uuid.uuid4())
path = dict(conf.path_template)
if path['url'] != '/':
path['url'] = '/' + random_file + ext
else:
path['url'] = random_file + ext
database.fetch_queue.put(path)
# Forced bogus path check
random_file = str(uuid.uuid4())
path = dict(conf.path_template)
path['url'] = '/' + random_file + '/'
database.fetch_queue.put(path)
workers = manager.spawn_workers(len(conf.crafted_404_extensions), FetchCrafted404Worker)
manager.wait_for_idle(workers, database.fetch_queue)
