def wait_for_idle(workers, queue): """ Wait until fetch queue is empty and handle user interrupt """ while len(workers) > 0: try: if queue.empty(): # Wait for all threads to return their state queue.join() for worker in workers: worker.kill_received = True workers = [] except KeyboardInterrupt: utils.output_raw('') utils.output_info('Keyboard Interrupt Received, cleaning up threads') # Kill remaining workers for worker in workers: worker.kill_received = True if worker is not None and worker.isAlive(): worker.join(1) sys.exit()
def execute(): """ Fetch /robots.txt and add the disallowed paths as target """ worker_template = {'url': '', 'expected_response': [200, 302], 'timeout_count': 0, 'description': 'Robots.txt entry'} target_url = urljoin(conf.target_host, "/robots.txt") fetcher = Fetcher() response_code, content, headers = fetcher.fetch_url(target_url, 'GET', conf.user_agent, True, conf.fetch_timeout_secs) if response_code is 200 or response_code is 302 and content: if conf.debug: utils.output_debug(content) match = re.findall(r'Disallow:\s*[a-zA-Z0-9-/.]*', content) added = 0 for match_obj in match: if '?' not in match_obj and '.' not in match_obj: splitted = match_obj.split(':') if splitted[1]: path = splitted[1].strip() if path != '/' or path != '': new_path = urljoin(conf.target_host, path) current_template = dict(worker_template) current_template['url'] = new_path database.paths.append(current_template) if conf.debug: utils.output_debug(str(current_template)) added += 1 if added > 0: utils.output_info('Robots plugin: added ' + str(added) + ' base paths using /robots.txt') else : utils.output_info('Robots plugin: no usable entries in /robots.txt') else: utils.output_info('Robots plugin: /robots.txt not found on target site')
def main(): """ Main app logic """ # Ensure the host is of the right format utils.sanitize_config() # Load target paths utils.output_info('Loading target paths') database.paths = loaders.load_targets('data/path.lst') # Import and run host plugins utils.output_info('Executing ' + str(len(host.__all__)) + ' host plugins') for plugin_name in host.__all__: plugin = __import__ ("plugins.host." + plugin_name, fromlist=[plugin_name]) if hasattr(plugin , 'execute'): plugin.execute() # Spawn workers if conf.search_files or conf.debug: workers = spawn_workers(conf.thread_count, output=False) else: workers = spawn_workers(conf.thread_count) # Fill work queue with fetch list utils.output_info('Probing ' + str(len(database.paths)) + ' paths') for item in database.paths: database.fetch_queue.put(item) # Wait for initial valid path lookup wait_for_idle(workers, database.fetch_queue) utils.output_info('Found ' + str(len(database.valid_paths)) + ' valid paths') if conf.debug: for item in database.valid_paths: utils.output_debug(str(item)) if conf.search_files: # Load target files utils.output_info('Loading target files') database.files = loaders.load_targets('data/file.lst') if conf.debug: for item in database.files: utils.output_debug('Target file added: ' + str(item)) # Combine files with '/' and all valid paths tmp_list = list() for file in database.files: file_copy = dict(file) file_copy['url'] = urljoin(conf.target_host, file_copy['url']) if conf.debug: utils.output('Adding base target: ' + str(file_copy)) tmp_list.append(file_copy) for valid_url in database.valid_paths: file_copy = dict(file) file_copy['url'] = valid_url['url'] + file['url'] if conf.debug: utils.output('Adding combined target: ' + str(file_copy)) tmp_list.append(file_copy) # Fill Valid path with generated urls for item in tmp_list: database.valid_paths.append(item) if conf.debug: for item in database.valid_paths: utils.output_debug('Path to test: ' + str(item)) # Add to valid paths # Import and run file plugins utils.output_info('Executing ' + str(len(path.__all__)) + ' file plugins') for plugin_name in path.__all__: plugin = __import__ ("plugins.path." + plugin_name, fromlist=[plugin_name]) if hasattr(plugin , 'execute'): plugin.execute() # Spawn workers workers = spawn_workers(conf.thread_count) # Fill work queue with fetch list utils.output_info('Probing ' + str(len(database.valid_paths)) + ' items...') for item in database.valid_paths: database.fetch_queue.put(item) # Wait for file lookup wait_for_idle(workers, database.fetch_queue)
# Spawn synchronized print output worker print_worker = PrintWorker() print_worker.daemon = True print_worker.start() if conf.debug: utils.output_debug('Version: ' + str(conf.version)) utils.output_debug('Use GET instead of HEAD: ' + str(conf.use_get)) utils.output_debug('Fetch timeout: ' + str(conf.fetch_timeout_secs)) utils.output_debug('Max timeouts per url: ' + str(conf.max_timeout_count)) utils.output_debug('Worker threads: ' + str(conf.thread_count)) utils.output_debug('Target Host: ' + str(conf.target_host)) utils.output_debug('Using Tor: ' + str(conf.use_tor)) utils.output_debug('Content-type Blacklisting: ' + str(conf.content_type_blacklist)) utils.output_debug('Using User-Agent: ' + str(conf.user_agent)) utils.output_info('Starting Discovery on ' + conf.target_host) # Handle keyboard exit before multi-thread operations try: # Launch main loop main() # Print all remaining messages utils.output_info('Done.\n') database.output_queue.join() except KeyboardInterrupt: utils.output_raw('') utils.output_info('Keyboard Interrupt Received') database.output_queue.join() sys.exit(0)
def run(self): while not self.kill_received: # don't wait for any items if empty if not database.fetch_queue.empty(): queued = database.fetch_queue.get() url = urljoin(conf.target_host, queued.get('url')) expected = queued.get('expected_response') description = queued.get('description') content_type_blacklist = queued.get('blacklist_content_types') if not content_type_blacklist: content_type_blacklist = [] if conf.use_get: method = 'GET' else: method = 'HEAD' response_code, content, headers = self.fetcher.fetch_url(url, method, conf.user_agent, False, conf.fetch_timeout_secs) if conf.debug: utils.output_info("Thread #" + str(self.thread_id) + ": " + str(queued)) if response_code is 0: # timeout if queued.get('timeout_count') < conf.max_timeout_count: new_timeout_count = queued.get('timeout_count') + 1 queued['timeout_count'] = new_timeout_count if conf.debug: utils.output_info('Thread #' + str(self.thread_id) + ': re-queuing ' + str(queued)) # Add back the timed-out item database.fetch_queue.put(queued) else: utils.output_timeout(url) elif response_code in expected: # Response content type content_type = headers['content-type'] if not content_type: content_type = '' # Fuse with current url. (/test become url.dom/test) queued['url'] = urljoin(conf.target_host, queued['url']) # If we don't blacklist, just show the result if not conf.content_type_blacklist: if self.output: if response_code == 401: utils.output_found('*Password Protected* ' + description + ' at: ' + url) else: utils.output_found(description + ' at: ' + url) # Add to valid path database.valid_paths.append(queued) # if we DO blacklist but content is not blacklisted, show the result elif content_type not in content_type_blacklist: if self.output: if response_code == 401: utils.output_found('*Password Protected* ' + description + ' at: ' + url) else: utils.output_found(description + ' at: ' + url) # Add to valid path database.valid_paths.append(queued) # Mark item as processed database.fetch_queue.task_done()