def sample_root_404(): """ Get the root 404, this has to be done as soon as possible since plugins could use this information. """ manager = ThreadManager() textutils.output_info('Benchmarking root 404') for ext in conf.crafted_404_extensions: random_file = str(uuid.uuid4()) path = conf.path_template.copy() if path['url'] != '/': path['url'] = '/' + random_file + ext else: path['url'] = random_file + ext # Were not using the fetch cache for 404 sampling database.fetch_queue.put(path) # Forced bogus path check random_file = str(uuid.uuid4()) path = conf.path_template.copy() path['url'] = '/' + random_file + '/' # Were not using the fetch cache for 404 sampling database.fetch_queue.put(path) workers = manager.spawn_workers(len(conf.crafted_404_extensions), FetchCrafted404Worker) manager.wait_for_idle(workers, database.fetch_queue)
def test_file_exists(): """ Test for file existence using http codes and computed 404 """ manager = ThreadManager() # Fill work queue with fetch list for item in database.valid_paths: dbutils.add_file_to_fetch_queue(item) # Wait for initial valid path lookup workers = manager.spawn_workers(conf.thread_count, TestFileExistsWorker) manager.wait_for_idle(workers, database.fetch_queue)
def test_paths_exists(): """ Test for path existence using http codes and computed 404 Spawn workers and turn off output for now, it would be irrelevant at this point. """ manager = ThreadManager() # Fill work queue with fetch list for path in database.paths: dbutils.add_path_to_fetch_queue(path) # Consider some file target as potential path for file in database.files: if not file.get('no_suffix'): file_as_path = file.copy() file_as_path['url'] = '/' + file_as_path['url'] dbutils.add_path_to_fetch_queue(file_as_path) done_paths = [] recursion_depth = 0 textutils.output_debug('Cached: ' + str(database.path_cache)) while database.fetch_queue.qsize() > 0: textutils.output_info('Probing ' + str(database.fetch_queue.qsize()) + ' paths') # Wait for initial valid path lookup workers = manager.spawn_workers(conf.thread_count, TestPathExistsWorker) manager.wait_for_idle(workers, database.fetch_queue) recursion_depth += 1 if not conf.recursive: break if recursion_depth >= conf.recursive_depth_limit: break for validpath in database.valid_paths: if validpath['url'] == '/' or validpath['url'] in done_paths: continue done_paths.append(validpath['url']) for path in database.paths: if path['url'] in ('/', ''): continue path = path.copy() path['url'] = validpath['url'] + path['url'] dbutils.add_path_to_fetch_queue(path) textutils.output_info('Found ' + str(len(database.valid_paths)) + ' valid paths')
def sample_404_from_found_path(): """ For all existing path, compute the 404 CRC so we don't get trapped in a tarpit """ manager = ThreadManager() for path in database.valid_paths: textutils.output_debug("Path in valid path table: " + str(path)) for ext in conf.crafted_404_extensions: path_clone = dict(path) random_file = str(uuid.uuid4()) # We don't benchmark / since we do it first before path discovery if path_clone['url'] != '/': path_clone['url'] = path_clone['url'] + '/' + random_file + ext database.fetch_queue.put(path_clone) workers = manager.spawn_workers(conf.thread_count, FetchCrafted404Worker) manager.wait_for_idle(workers, database.fetch_queue)
def sample_404_from_found_path(): """ For all existing path, compute the 404 CRC so we don't get trapped in a tarpit """ manager = ThreadManager() for path in database.valid_paths: textutils.output_debug("Path in valid path table: " + str(path)) for ext in conf.crafted_404_extensions: path_clone = path.copy() random_file = str(uuid.uuid4()) # We don't benchmark / since we do it first before path discovery if path_clone['url'] != '/': path_clone['url'] = path_clone['url'] + '/' + random_file + ext # Were not using the fetch cache for 404 sampling database.fetch_queue.put(path_clone) workers = manager.spawn_workers(conf.thread_count, FetchCrafted404Worker) manager.wait_for_idle(workers, database.fetch_queue)
def sample_root_404(): """ Get the root 404, this has to be done as soon as possible since plugins could use this information. """ manager = ThreadManager() textutils.output_info('Benchmarking root 404') for ext in conf.crafted_404_extensions: random_file = str(uuid.uuid4()) path = dict(conf.path_template) if path['url'] != '/': path['url'] = '/' + random_file + ext else: path['url'] = random_file + ext database.fetch_queue.put(path) # Forced bogus path check random_file = str(uuid.uuid4()) path = dict(conf.path_template) path['url'] = '/' + random_file + '/' database.fetch_queue.put(path) workers = manager.spawn_workers(len(conf.crafted_404_extensions), FetchCrafted404Worker) manager.wait_for_idle(workers, database.fetch_queue)