def requests_headers_names_test(self, module, payloads, outputs): """Test if the request headers are vulnerable""" print(process_icon, 'Analyzing the requests headers.') requests_headers_names = module['entry_points']['requests_headers_names'] if module['entry_points'][ 'requests_headers_names'] else self.project['headers'].keys() for link in self.project['links']: if module['entry_points']['requests_headers_names'] and isinstance(requests_headers_names[0], list): for headers_names_list in requests_headers_names: for payload_index in range(len(payloads[0])): headers = self.project['headers'].copy() inp = [] for n, name in enumerate(headers_names_list): inp.append(payloads[n][payload_index]) headers[name] = payloads[n][payload_index] resp, _ = Get(link, headers=headers, timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, inp, {link: headers}, outputs) if vulnerable or vulnerable is None: break else: for name in requests_headers_names: for inp in payloads[0]: if name == 'Pentest': continue headers = self.project['headers'].copy() headers[name] = inp resp, _ = Get(link, headers=headers, timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, inp, {link: headers}, outputs) if vulnerable or vulnerable is None: break
def forms_inp_names_test(self, module, payloads, outputs): """Test if the form inputs are vulnerable""" print(process_icon, 'Analyzing the Forms Inputs.') for form in self.project['forms']: forms_inputs_names = module['entry_points'].get('forms_inputs_names') data = {} for form_input in form['inputs']: if form_input.get('name'): data[form_input.get('name')] = form_input.get('value', None) if module['entry_points']['forms_inputs_names'] and isinstance(forms_inputs_names[0], list): for form_input_list in forms_inputs_names: for payload_index in range(len(payloads[0])): mod_data = data.copy() passed = True inp = [] for n, form_input in enumerate(form_input_list): inp.append(payloads[n][payload_index]) if form_input not in mod_data.keys(): passed = False break mod_data[form_input] = payloads[n][payload_index] if passed: if form['method'] == 'post': resp, _ = Post(form['action'], data=data, headers=self.project['headers']) else: resp, _ = Get(form['action'], params=data, timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, inp, {form['action']: mod_data}, outputs) if vulnerable or vulnerable is None: break else: for form_input in form['inputs']: if form_input['type'] == 'hidden' and not module['test_hidden_form_inputs']: continue for payload in payloads[0]: mod_data = data.copy() mod_data[form_input['name']] = payload if form['method'] == 'post': resp, _ = Post(form['action'], data=data, headers=self.project['headers']) else: url = list(urlparse(form['action'])) url[4] = urlencode(mod_data) resp, _ = Get(urlunparse(url), timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, payload, {form['action']: mod_data, 'method': form['method']}, outputs) if vulnerable or vulnerable is None: break
def crawl_robots_txt(): links = [] robots_dict = parse_robots_txt(Spider.project['base_url'], Spider.project['base_url']) if robots_dict: for xml_link in robots_dict['Sitemap']: if xml_link.endswith('.xml'): resp, err = Get(xml_link, headers=Spider.project['headers'], cookies=Spider.project['cookies']) if err: continue if resp.content != '': try: tree = html.fromstring(resp.content) links = links + tree.xpath('.//loc/text()') except Exception as e: continue links = [ url_check(str(link), Spider.project['base_url']) for link in links if link not in Spider.project['links'].keys() ] links = filter(None, links) Spider.project['queue'].update(set(links)) Spider.project['queue'].update(robots_dict['Allow']) Spider.project['queue'].update(robots_dict['Disallow']) Spider.project['queue'].update(robots_dict['Noindex'])
def paths_test(self, module, payloads, outputs): """Send get request to the path and analyze the output points input parameter must be a in this form {0: ['path1','path2']}""" print(process_icon, 'Analyzing the paths.') for payload in payloads[0]: url = url_check(payload, self.project['base_url']) resp, _ = Get(url, headers=self.project['headers'], timeout=self.req_timout) if self.analyze_the_output(module, resp, payload, {url: payload}, outputs) is None: return
def scan_target(name, modules): # Start a new project scan_path = f'scans/{name}.yml' msg, project = load_scan(scan_path) if not project: logging.error('Scan config not found.') return resp, err = Get(project['base_url'], headers=project['headers'], cookies=project['cookies']) if err: logging.error('[!] Host can not be reached') return project['queue'].add(project['base_url']) try: msg, project = load_project(project) logging.info(msg) if project.get('done', False): return project['done'] = False if project['enable_crawler']: logging.info('Parsing robots.txt file.') Spider(project).crawl_robots_txt() logging.info('Start Crawling.') logging.info('The spiders are Just Doing Their Best.') Spider(project).run() # Delete the duplicated forms forms_hashes = set() new_forms = [] for form in project['forms']: form_hash = hashlib.sha1(str(form).encode()).hexdigest() if form_hash in forms_hashes: continue else: forms_hashes.add(form_hash) new_forms.append(form) project['forms'] = new_forms del forms_hashes logging.info('Crawling DONE!') if project['subdomains_research']: logging.info('Subdomain research started.') project['subdomains'] = get_subdomains(project['base_url']) logging.info('Subdomain research done!') save_project(project) logging.info('Analysis Engine started.') AnalysisEngine(project, modules).start() project['done'] = True save_project(project) except KeyboardInterrupt as e: logging.error(f'Terminated by user, {e}') save_project(project) except KeyError as e: logging.error( f'Invalid project please check the template and try again, {e}') except Exception as e: logging.error(e) save_project(project)
def queries_names_test(self, module, payloads, outputs): """Test if the queries are vulnerable""" print(process_icon, 'Analyzing the URL\'s queries.') for url in self.project['queries']: parsed_url = urlparse(url) url_parts = list(parsed_url) query = dict(parse_qsl(parsed_url.query)) query_names = module['entry_points']['queries_names'] if module['entry_points'][ 'queries_names'] else query.keys() if module['entry_points']['queries_names'] and isinstance(query_names[0], list): mod_url = '' for query_names_list in query_names: if len(set(query_names_list).intersection(set(query.keys()))) == len(query_names_list): for inp_index in range(len(payloads[0])): mod_queries = query.copy() inp = [] for n, query_name in enumerate(query_names_list): inp.append(payloads[n][inp_index]) mod_queries[query_name] = payloads[n][inp_index] url_parts[4] = urlencode(mod_queries) mod_url = urlunparse(url_parts) resp, _ = Get(mod_url, headers=self.project['headers'], cookies=self.project['cookies'], timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, inp, mod_url, outputs) if vulnerable or vulnerable is None: break else: for query_name in query_names: if query_name in query.keys(): for inp in payloads[0]: mod_queries = query.copy() mod_queries[query_name] = inp url_parts[4] = urlencode(mod_queries) mod_url = urlunparse(url_parts) resp, _ = Get(mod_url, headers=self.project['headers'], cookies=self.project['cookies'], timeout=self.req_timout) vulnerable = self.analyze_the_output(module, resp, inp, mod_url, outputs) if vulnerable or vulnerable is None: break
def analyze_the_output(self, module, resp, inp, mod_vector, outputs): """Gets the info about the module and the output as parameters and analyze the output to find if the module is success. Return Boolean value.""" vuln = False if module['output_path'] != 'SAME': resp = Get(url_check(module['output_path'], urlparse(self.project['base_url'])), headers=self.project['headers'], timeout=self.req_timout, cookies=self.project['cookies']) if module['output_type'] == 'DELAY': if module['delay'] + 3 >= resp.elapsed.total_seconds() >= module['delay']: vuln = True if module['output_type'] == 'REFLECT': outputs = inp if isinstance(inp, list) else [inp] if not outputs: printc('[!] This output type is not supported for the module entry points', 'Red', attrs=['bold']) return None if resp and 'response_contents' in module['output_points']: if any(output in str(resp.content) for output in outputs): vuln = True if resp and 'response_headers_names' in module['output_points']: if set(outputs) & set(resp.headers.keys()): vuln = True if resp and 'response_headers_values' in module['output_points']: if set(outputs) & set(resp.headers.values()): vuln = True if resp and 'status_codes' in module['output_points']: if resp.status_code in outputs: vuln = True if vuln: if mod_vector not in self.project['vulnerabilities'][module['severity']].get(module['name'], []): self.project['vulnerabilities'][module['severity']][module['name']] = self.project['vulnerabilities'][ module['severity']].get( module['name'], []) + [mod_vector] save_project(self.project) print(process_icon, c(len(self.project['vulnerabilities'][module['severity']].get(module['name'], [])), 'Red', attrs=['bold']), module['name'].replace('_', ' ') + '\'s', 'Detected.\r', end='') # print(c('\r[+]', 'Blue', attrs=['bold']), # 'Checking', # c(next(iter(mod_vector.values())) if isinstance(mod_vector, dict) else mod_vector, 'DarkOrange3'), # end='') return vuln
def parse_robots_txt(link, base_url): result = { "Sitemap": set(), "User-agent": set(), "Disallow": set(), "Allow": set(), "Noindex": set() } resp, _ = Get(link + "/robots.txt") if resp and resp.status_code == 200: try: for line in resp.content.decode('utf-8').split('\n'): parts = line.split(': ') if len(parts) == 2: url = url_check(parts[1].split('#')[0].strip(), base_url) if url: result[parts[0].strip()].add(url) except KeyError as e: pass return result
def get_subdomains(target): try: ipaddress.ip_address(target) except ValueError as e: print(c('[!] subdomain scan is not supported for this target.', 'Red')) logging.info('subdomain scan is not supported for this target') return set() try: target = urlparse(target).hostname.split('.', 1)[1] except Exception as e: print(c('[x] subdomain scan is not supported for this target.', 'Red')) logging.info('subdomain scan is not supported for this target, {e}') return set() subdomains_list = set() cursor = connect_to_db() if cursor: try: cursor.execute('SELECT ci.NAME_VALUE NAME_VALUE \ FROM certificate_identity ci \ WHERE ci.NAME_TYPE = \'dNSName\' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower(\'%.{}\'));' .format(target)) except Exception as e: logging.error( 'failed to get the subdomain list from crt.sh database') for result in cursor.fetchall(): if len(result) == 1: subdomains_list.add(''.join(result)) # Get subdomains from virustotal url = f'https://www.virustotal.com/ui/domains/{target}/subdomains?limit=40' try: headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Cache-Control': 'max-age=0' } resp, _ = Get(url, headers=headers) vt_subdomains_list = [ resp.json()['data'][ind]['id'] for ind in range(40) ] subdomains_list.update(set(vt_subdomains_list)) except Exception as e: logging.error(e) # Find all the sites that have the same SSL certificate cert = get_cert(target, 443) x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) for ind in range(x509.get_extension_count()): ext = x509.get_extension(ind) if ext.get_short_name() == b'subjectAltName': subdomains_list.update( set([ x.split('DNS:')[1].strip() for x in ext.__str__().split(',') ])) return sorted(subdomains_list)
def crawl(link): Spider.project['queue'].discard(link) # Prevent logout if the user set a Cookies if 'logout' in link: return if link.endswith(COMMON_EXT): Spider.project['files'].add(link) elif link not in Spider.project['links'].keys(): resp, err = Get(link, headers=Spider.project['headers'], cookies=Spider.project['cookies']) if err: return # Add the link to the project links Spider.project['links'][link] = resp.status_code # if link have queries add the link to the queries list parsed_link = urlparse(link) if parsed_link.query: if queries_check(link, Spider.project['base_url'], Spider.queries_hashes): Spider.project['queries'].add(link) # Check if the link contain any contents if not resp.content: return # using 'lxml' for best performance try: soup = bs(resp.content.decode('utf-8'), 'lxml') except UnicodeDecodeError as e: soup = bs(resp.content, 'lxml') except Exception as e: logging.error('failed to creating the page soup.') return # Parse the forms forms = parse_forms(soup, link, Spider.project['base_url']) if forms: Spider.project['forms'] += forms trap = re.search('.*(/.*calendar.*)', link) or re.search( '^.*?(/.+?/).*?(\1).*(\1)', link) if not trap: # Parse URls from the page contents for tag in soup.findAll('a', href=True): url = url_check(tag['href'].split('#')[0], Spider.project['base_url']) if url: Spider.project['queue'].add(url) for tag in soup.findAll(['frame', 'iframe'], src=True): url = url_check(tag['src'].split('#')[0], Spider.project['base_url']) if url: Spider.project['queue'].add(url) for tag in soup.findAll('button', formaction=True): url = url_check(tag['formaction'], Spider.project['base_url']) if url: Spider.project['queue'].add(url)
def yawss_cli(project_path, modules=None): # Start a new project msg, project = load_scan(project_path) if not project: printc('[!] Scan config not found.', 'DeepPink4', attrs=['bold']) return resp, err = Get(project['base_url'], headers=project['headers'], cookies=project['cookies']) if err: printc('[!] Host can not be reached', 'Red', attrs=['bold']) return project['queue'].add(project['base_url']) try: msg, project = load_project(project) printc('[~] ' + msg, 'SteelBlue', attrs=['blink']) if project['enable_crawler']: print(process_icon, 'Parsing robots.txt file.') Spider(project).crawl_robots_txt() print(process_icon, 'Start Crawling.') printc('[~] The spiders are Just Doing Their Best.', 'Grey69') Spider(project).run() # Delete the duplicated forms forms_hashes = set() new_forms = [] for form in project['forms']: form_hash = hashlib.sha1(str(form).encode()).hexdigest() if form_hash in forms_hashes: continue else: forms_hashes.add(form_hash) new_forms.append(form) project['forms'] = new_forms del forms_hashes print('\n' + process_icon, 'Crawling DONE!') if project['subdomains_research']: print(process_icon, 'Subdomain research started.') project['subdomains'] = get_subdomains(project['base_url']) print(process_icon, 'Subdomain research done!') nums_color = 'DarkSeaGreen' print( process_icon, c(f'{len(project["links"])}', nums_color, attrs=['bold']) + ' Links,', c(f'{len(project["forms"])}', nums_color, attrs=['bold']) + ' Forms,', c(f'{len(project["queries"])}', nums_color, attrs=['bold']) + ' Queries', c(f'{len(project["subdomains"])}', nums_color, attrs=['bold']) + ' Subdomains and', c(f'{len(project["files"])}', nums_color, attrs=['bold']) + ' Files Found.') save_project(project) print(process_icon + ' Analysis Engine started.') AnalysisEngine(project, modules).start() save_project(project) except KeyboardInterrupt as e: printc('[X] Ok ok, quitting.', 'Red', attrs=['bold']) save_project(project) except KeyError as e: logging.error( 'Invalid project please check the template and try again') print(c('[!]', 'Red'), e, 'value not found') printc('Invalid project please check the template and try again', 'Red', attrs=['bold']) except Exception as e: logging.error(e) save_project(project) printc(error_icon + '\n\tAn Unexpected Error Occurred! Please check the logs', 'Red', attrs=['bold'])