def recv_info(self, info): m_parsed_url = info.parsed_url m_results = [] #------------------------------------------------------------------ # Find suspicious URLs by matching against known substrings. # Load wordlists m_wordlist_middle = WordListLoader.get_wordlist(Config.plugin_config['middle']) m_wordlist_extensions = WordListLoader.get_wordlist(Config.plugin_config['extensions']) # Add matching keywords at any positions of URL. m_results.extend([SuspiciousURLPath(info, x) for x in m_wordlist_middle if x in m_parsed_url.directory.split("/") or x == m_parsed_url.filebase or x == m_parsed_url.extension]) # Add matching keywords at any positions of URL. m_results.extend([SuspiciousURLPath(info, x) for x in m_wordlist_extensions if m_parsed_url.extension == x]) #------------------------------------------------------------------ # Find suspicious URLs by calculating the Shannon entropy of the hostname. # Idea from: https://github.com/stricaud/urlweirdos/blob/master/src/urlw/plugins/shannon/__init__.py # TODO: test with unicode enabled hostnames! # Check the Shannon entropy for the hostname. hostname = info.parsed_url.hostname entropy = calculate_shannon_entropy(hostname) if entropy > 4.0: m_results.append( SuspiciousURLPath(info, hostname) ) # Check the Shannon entropy for the subdomains. for subdomain in info.parsed_url.hostname.split('.'): if len(subdomain) > 3: entropy = calculate_shannon_entropy(subdomain) if entropy > 4.0: m_results.append( SuspiciousURLPath(info, subdomain) ) #------------------------------------------------------------------ # # # # Get malware suspicious links # # # #------------------------------------------------------------------ p = None m_url = info.url Logger.log_more_verbose("Looking for output links to malware sites") try: allow_redirects = Config.audit_config.follow_redirects or \ (info.depth == 0 and Config.audit_config.follow_first_redirect) p = download(m_url, self.check_download, allow_redirects=allow_redirects) except NetworkException,e: Logger.log_more_verbose("Error while processing %r: %s" % (m_url, str(e)))
def recv_info(self, info): m_parsed_url = info.parsed_url m_results = [] #------------------------------------------------------------------ # Find suspicious URLs by matching against known substrings. # Load wordlists m_wordlist_middle = WordListLoader.get_wordlist( Config.plugin_config['middle']) m_wordlist_extensions = WordListLoader.get_wordlist( Config.plugin_config['extensions']) # Add matching keywords at any positions of URL. m_results.extend([ SuspiciousURL(info, x) for x in m_wordlist_middle if x in m_parsed_url.directory.split("/") or x == m_parsed_url.filebase or x == m_parsed_url.extension ]) # Add matching keywords at any positions of URL. m_results.extend([ SuspiciousURL(info, x) for x in m_wordlist_extensions if m_parsed_url.extension == x ]) #------------------------------------------------------------------ # Find suspicious URLs by calculating the Shannon entropy of the hostname. # Idea from: https://github.com/stricaud/urlweirdos/blob/master/src/urlw/plugins/shannon/__init__.py # TODO: test with unicode enabled hostnames! # Check the Shannon entropy for the hostname. hostname = info.parsed_url.hostname entropy = calculate_shannon_entropy(hostname) if entropy > 4.0: m_results.append(SuspiciousURL(info, hostname)) # Check the Shannon entropy for the subdomains. for subdomain in info.parsed_url.hostname.split('.'): if len(subdomain) > 3: entropy = calculate_shannon_entropy(subdomain) if entropy > 4.0: m_results.append(SuspiciousURL(info, subdomain)) #------------------------------------------------------------------ return m_results
def analyze_url(self, info): m_parsed_url = info.parsed_url m_results = [] Logger.log_more_verbose("Processing URL: %s" % m_parsed_url) #---------------------------------------------------------------------- # Find suspicious URLs by matching against known substrings. # Load wordlists m_wordlist_middle = WordListLoader.get_wordlist(Config.plugin_config['middle']) m_wordlist_extensions = WordListLoader.get_wordlist(Config.plugin_config['extensions']) # Add matching keywords at any positions of URL. m_results.extend([SuspiciousURLPath(info, x) for x in m_wordlist_middle if x in m_parsed_url.directory.split("/") or x == m_parsed_url.filebase or x == m_parsed_url.extension]) # Add matching keywords at any positions of URL. m_results.extend([SuspiciousURLPath(info, x) for x in m_wordlist_extensions if m_parsed_url.extension == x]) #---------------------------------------------------------------------- # Find suspicious URLs by calculating the Shannon entropy of the hostname. # Idea from: https://github.com/stricaud/urlweirdos/blob/master/src/urlw/plugins/shannon/__init__.py # TODO: test with unicode enabled hostnames! # Check the Shannon entropy for the hostname. hostname = info.parsed_url.hostname entropy = calculate_shannon_entropy(hostname) if entropy > 4.0: m_results.append( SuspiciousURLPath(info, hostname) ) # Check the Shannon entropy for the subdomains. for subdomain in info.parsed_url.hostname.split('.'): if len(subdomain) > 3: entropy = calculate_shannon_entropy(subdomain) if entropy > 4.0: m_results.append( SuspiciousURLPath(info, subdomain) ) return m_results