Пример #1
0
    def _generic_vhosts( self, fuzzableRequest ):
        '''
        Test some generic virtual hosts, only do this once.
        '''
        res = []
        base_url = urlParser.baseUrl(fuzzableRequest.getURL())
        
        common_vhost_list = self._get_common_virtualhosts(urlParser.getDomain(base_url))
        
        # Get some responses to compare later
        original_response = self._urlOpener.GET(base_url, useCache=True)
        orig_resp_body = original_response.getBody()
        non_existant = 'iDoNotExistPleaseGoAwayNowOrDie' + createRandAlNum(4)
        self._non_existant_response = self._urlOpener.GET(base_url, useCache=False, \
                                                        headers={'Host': non_existant })
        nonexist_resp_body = self._non_existant_response.getBody()
        
        for common_vhost in common_vhost_list:
            try:
                vhost_response = self._urlOpener.GET( base_url, useCache=False, \
                                                headers={'Host': common_vhost } )
            except w3afException:
                pass
            else:
                vhost_resp_body = vhost_response.getBody()

                # If they are *really* different (not just different by some chars)
                if relative_distance_lt(vhost_resp_body, orig_resp_body, 0.35) and \
                    relative_distance_lt(vhost_resp_body, nonexist_resp_body, 0.35):
                    res.append((common_vhost, vhost_response.id))
        
        return res
Пример #2
0
 def discover(self, fuzzableRequest ):
     '''
     Get www.site.com and site.com and compare responses.
     
     @parameter fuzzableRequest: A fuzzableRequest instance that contains (among other things) the URL to test.
     '''
     if not self._exec :
         # This will remove the plugin from the discovery plugins to be runned.
         raise w3afRunOnce()
         
     else:
         # Only run once
         self._exec = False
         
         if not re.match('\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?',
                                 urlParser.getDomain( fuzzableRequest.getURL() ) ):
             # Only do all this if this is a domain name!
             base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
             original_response = self._urlOpener.GET( base_url, useCache=True )
             
             domain = urlParser.getDomain( fuzzableRequest.getURL() )
             proto = urlParser.getProtocol( fuzzableRequest.getURL() )
             if domain.startswith('www.'):
                 dns_wildcard_url = proto + '://' + domain.replace('www.', '') + '/'
             else:
                 dns_wildcard_url = proto + '://www.' + domain + '/'
             
             self._test_DNS( original_response, dns_wildcard_url )
             self._test_IP( original_response, domain )
             
         return []
Пример #3
0
 def discover(self, fuzzableRequest ):
     '''
     Searches for user directories.
     
     @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                                 (among other things) the URL to test.
     '''
     if not self._run:
         raise w3afRunOnce()
         
     else:
         self._run = False
         self._fuzzable_requests = []
             
         base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
         self._headers = {'Referer': base_url }
         
         # Create a response body to compare with the others
         non_existant_user = '******'
         test_URL = urlParser.urlJoin( base_url, non_existant_user )
         try:
             response = self._urlOpener.GET( test_URL, useCache=True, \
                                                                 headers=self._headers )
             response_body = response.getBody()                
         except:
             raise w3afException('userDir failed to create a non existant signature.')
             
         self._non_existant = response_body.replace( non_existant_user, '')
         
         # Check the users to see if they exist
         url_user_list = self._create_dirs( base_url )
         for url, user in url_user_list :
             om.out.debug('userDir is testing ' + url )
             
             #   Send the requests using threads:
             targs = ( url, user )
             self._tm.startFunction( target=self._do_request, args=targs, ownerObj=self )
             
         # Wait for all threads to finish
         self._tm.join( self )
         
         # Only do this if I already know that users can be identified.
         if kb.kb.getData( 'userDir', 'users' ) != []:
             # AND only run once
             if self._run_OS_ident:
                 self._run_OS_ident = False
                 self._advanced_identification( base_url, 'os' )
             
             if self._run_app_ident:
                 self._run_app_ident = False
                 self._advanced_identification( base_url, 'apps' )
                 
             # Report findings of remote OS, applications, users, etc.
             self._report_findings()
         
         return self._fuzzable_requests
Пример #4
0
    def discover(self, fuzzableRequest ):
        '''
        Get the robots.txt file and parse it.
        
        @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                                      (among other things) the URL to test.
        '''
        if not self._exec:
            # This will remove the plugin from the discovery plugins to be runned.
            raise w3afRunOnce()
        else:
            # Only run once
            self._exec = False
            
            dirs = []
            self._new_fuzzable_requests = []         
            
            base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
            robots_url = urlParser.urlJoin(  base_url , 'robots.txt' )
            http_response = self._urlOpener.GET( robots_url, useCache=True )
            
            if not is_404( http_response ):
                # Save it to the kb!
                i = info.info()
                i.setPluginName(self.getName())
                i.setName('robots.txt file')
                i.setURL( robots_url )
                i.setId( http_response.id )
                i.setDesc( 'A robots.txt file was found at: "'+ robots_url +'".' )
                kb.kb.append( self, 'robots.txt', i )
                om.out.information( i.getDesc() )


                # Work with it...
                dirs.append( robots_url )
                for line in http_response.getBody().split('\n'):
                    
                    line = line.strip()
                    
                    if len(line) > 0 and line[0] != '#' and (line.upper().find('ALLOW') == 0 or\
                    line.upper().find('DISALLOW') == 0 ):
                        
                        url = line[ line.find(':') + 1 : ]
                        url = url.strip()
                        url = urlParser.urlJoin(  base_url , url )
                        dirs.append( url )

            for url in dirs:
                #   Send the requests using threads:
                targs = ( url,  )
                self._tm.startFunction( target=self._get_and_parse, args=targs , ownerObj=self )
            
            # Wait for all threads to finish
            self._tm.join( self )
        
        return self._new_fuzzable_requests
Пример #5
0
    def discover(self, fuzzableRequest):
        """
        Get the file and parse it.
        
        @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                                      (among other things) the URL to test.
        """
        if not self._exec:
            raise w3afRunOnce()
        else:
            # Only run once
            self._exec = False

            base_url = urlParser.baseUrl(fuzzableRequest.getURL())

            ### Google Gears
            for ext in self._extensions:
                for word in file(self._wordlist):
                    manifest_url = urlParser.urlJoin(base_url, word.strip())
                    manifest_url = manifest_url + ext
                    om.out.debug("Google Gears Manifest Testing " + manifest_url)
                    http_response = self._urlOpener.GET(manifest_url, useCache=True)

                    if '"entries":' in http_response and not is_404(http_response):
                        # Save it to the kb!
                        i = info.info()
                        i.setPluginName(self.getName())
                        i.setName("Gears Manifest")
                        i.setURL(manifest_url)
                        i.setId(http_response.id)
                        desc = 'A gears manifest file was found at: "' + manifest_url
                        desc += '".  Each file should be manually reviewed for sensitive'
                        desc += " information that may get cached on the client."
                        i.setDesc(desc)
                        kb.kb.append(self, manifest_url, i)
                        om.out.information(i.getDesc())

            ### CrossDomain.XML
            cross_domain_url = urlParser.urlJoin(base_url, "crossdomain.xml")
            om.out.debug("Checking crossdomain.xml file")
            response = self._urlOpener.GET(cross_domain_url, useCache=True)

            if not is_404(response):
                self._checkResponse(response, "crossdomain.xml")

            ### CrossAccessPolicy.XML
            client_access_url = urlParser.urlJoin(base_url, "clientaccesspolicy.xml")
            om.out.debug("Checking clientaccesspolicy.xml file")
            response = self._urlOpener.GET(client_access_url, useCache=True)

            if not is_404(response):
                self._checkResponse(response, "clientaccesspolicy.xml")

        return []
Пример #6
0
 def discover(self, fuzzableRequest ):
     '''
     It calls the "main" from halberd and writes the results to the kb.
     
     @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                 (among other things) the URL to test.
     '''
     if not self._exec:
         # This will remove the plugin from the discovery plugins to be runned.
         raise w3afRunOnce()
     else:
         self._exec = False
         msg = 'halberd plugin is starting. Original halberd author: Juan M. Bello Rivas ;'
         msg += ' http://halberd.superadditive.com/'
         om.out.information( msg )
         
         self._main( urlParser.baseUrl( fuzzableRequest.getURL() ) )
         
     return []
Пример #7
0
 def discover(self, fuzzableRequest ):
     '''
     Get the sitemap.xml file and parse it.
     
     @parameter fuzzableRequest: A fuzzableRequest instance that contains (among other things) the URL to test.
     '''
     if not self._exec:
         # This will remove the plugin from the discovery plugins to be runned.
         raise w3afRunOnce()
     else:
         # Only run once
         self._exec = False
         self._new_fuzzable_requests = []
         
         base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
         sitemap_url = urlParser.urlJoin(  base_url , 'sitemap.xml' )
         response = self._urlOpener.GET( sitemap_url, useCache=True )
         
         # Remember that httpResponse objects have a faster "__in__" than
         # the one in strings; so string in response.getBody() is slower than
         # string in response
         if '</urlset>' in response and not is_404( response ):
             om.out.debug('Analyzing sitemap.xml file.')
             
             self._new_fuzzable_requests.extend( self._createFuzzableRequests( response ) )
             
             import xml.dom.minidom
             om.out.debug('Parsing xml file with xml.dom.minidom.')
             try:
                 dom = xml.dom.minidom.parseString( response.getBody() )
             except:
                 raise w3afException('Error while parsing sitemap.xml')
             urlList = dom.getElementsByTagName("loc")
             for url in urlList:
                 url = url.childNodes[0].data 
                 #   Send the requests using threads:
                 targs = ( url,  )
                 self._tm.startFunction( target=self._get_and_parse, args=targs , ownerObj=self )
         
             # Wait for all threads to finish
             self._tm.join( self )
     
         return self._new_fuzzable_requests
Пример #8
0
    def discover(self, fuzzableRequest ):
        '''
        GET some files and parse them.
        
        @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                    (among other things) the URL to test.
        '''
        dirs = []
        if not self._exec :
            # This will remove the plugin from the discovery plugins to be runned.
            raise w3afRunOnce()
            
        else:
            # Only run once
            self._exec = False
            
            baseUrl = urlParser.baseUrl( fuzzableRequest.getURL() )
            
            for url, regex_string in self.getOracleData():

                oracle_discovery_URL = urlParser.urlJoin(  baseUrl , url )
                response = self._urlOpener.GET( oracle_discovery_URL, useCache=True )
                
                if not is_404( response ):
                    dirs.extend( self._createFuzzableRequests( response ) )
                    if re.match( regex_string , response.getBody(), re.DOTALL):
                        i = info.info()
                        i.setPluginName(self.getName())
                        i.setName('Oracle application')
                        i.setURL( response.getURL() )
                        i.setDesc( self._parse( url, response ) )
                        i.setId( response.id )
                        kb.kb.append( self, 'info', i )
                        om.out.information( i.getDesc() )
                    else:
                        msg = 'oracleDiscovery found the URL: ' + response.getURL()
                        msg += ' but failed to parse it. The content of the URL is: "'
                        msg += response.getBody() + '".'
                        om.out.debug( msg )
        
        return dirs
Пример #9
0
    def discover(self, fuzzableRequest ):
        '''
        Runs pykto to the site.
        
        @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                                      (among other things) the URL to test.
        '''
        self._new_fuzzable_requests = []
        
        if not self._exec:
            # dont run anymore
            raise w3afRunOnce()
            
        else:
            # run!
            if self._update_scandb:
                self._update_db()
            
            # Run the basic scan (only once)
            if self._first_time:
                self._first_time = False
                url = urlParser.baseUrl( fuzzableRequest.getURL() )
                self._exec = False
                self.__run( url )
            
            # And now mutate if the user configured it...
            if self._mutate_tests:
                
                # If mutations are enabled, I should keep running
                self._exec = True
                
                # Tests are to be mutated
                url = urlParser.getDomainPath( fuzzableRequest.getURL() )
                if url not in self._already_visited:
                    # Save the directories I already have tested
                    self._already_visited.add( url )
                    self.__run( url )

        return self._new_fuzzable_requests
Пример #10
0
    def discover(self, fuzzableRequest ):
        '''
        Get the file and parse it.
        @parameter fuzzableRequest: A fuzzableRequest instance that contains
                                                      (among other things) the URL to test.
        '''
        if not self._exec:
            raise w3afRunOnce()
        else:
            
            if not self._be_recursive:
                # Only run once
                self._exec = False

            self._fuzzable_requests = []
            
            domain_path = urlParser.getDomainPath( fuzzableRequest.getURL() )
            base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
            
            to_test = []
            if not self._tested_base_url:
                to_test.append( base_url )
                self._tested_base_url = True
                
            if domain_path != base_url:
                to_test.append( domain_path )
            
            for base_path in to_test:
                
                #   Send the requests using threads:
                targs = ( base_path,  )
                self._tm.startFunction( target=self._bruteforce_directories, args=targs , ownerObj=self )
            
            # Wait for all threads to finish
            self._tm.join( self )

        return self._fuzzable_requests
Пример #11
0
    def discover(self, fuzzableRequest ):
        '''
        Get the server-status and parse it.
        
        @parameter fuzzableRequest: A fuzzableRequest instance that contains (among other things) the URL to test.
        '''
        res = []
        if not self._exec :
            # This will remove the plugin from the discovery plugins to be runned.
            raise w3afRunOnce()
            
        else:
            # Only run once
            self._exec = False
            
            base_url = urlParser.baseUrl( fuzzableRequest.getURL() )
            server_status_url = urlParser.urlJoin(  base_url , 'server-status' )
            response = self._urlOpener.GET( server_status_url, useCache=True )
            
            if not is_404( response ) and response.getCode() not in range(400, 404):
                msg = 'Apache server-status cgi exists. The URL is: "' + response.getURL() + '".'
                om.out.information( msg )
                
                # Create some simple fuzzable requests
                res.extend( self._createFuzzableRequests( response ) )

                # Get the server version
                # <dl><dt>Server Version: Apache/2.2.9 (Unix)</dt>
                for version in re.findall('<dl><dt>Server Version: (.*?)</dt>', response.getBody()):
                    # Save the results in the KB so the user can look at it
                    i = info.info()
                    i.setPluginName(self.getName())
                    i.setURL( response.getURL() )
                    i.setId( response.id )
                    i.setName( 'Apache Server version' )
                    msg = 'The web server has the apache server status module enabled, '
                    msg += 'which discloses the following remote server version: "' + version + '".'
                    i.setDesc( msg )

                    om.out.information(i.getDesc())
                    kb.kb.append( self, 'server', i )
                
                # Now really parse the file and create custom made fuzzable requests
                regex = '<td>.*?<td nowrap>(.*?)</td><td nowrap>.*? (.*?) HTTP/1'
                for domain, path in re.findall(regex, response.getBody() ):
                    
                    if 'unavailable' in domain:
                        domain = urlParser.getDomain( response.getURL() )
                        
                    foundURL = urlParser.getProtocol( response.getURL() ) + '://' + domain + path
                    # Check if the requested domain and the found one are equal.
                    if urlParser.getDomain( foundURL ) == urlParser.getDomain( response.getURL() ):
                        # They are equal, request the URL and create the fuzzable requests
                        tmpRes = self._urlOpener.GET( foundURL, useCache=True )
                        if not is_404( tmpRes ):
                            res.extend( self._createFuzzableRequests( tmpRes ) )
                    else:
                        # This is a shared hosting server
                        self._shared_hosting_hosts.append( domain )
                
                # Now that we are outsite the for loop, we can report the possible vulns
                if len( self._shared_hosting_hosts ):
                    v = vuln.vuln()
                    v.setPluginName(self.getName())
                    v.setURL( fuzzableRequest.getURL() )
                    v.setId( response.id )
                    self._shared_hosting_hosts = list( set( self._shared_hosting_hosts ) )
                    v['alsoInHosting'] = self._shared_hosting_hosts
                    v.setDesc( 'The web application under test seems to be in a shared hosting.' )
                    v.setName( 'Shared hosting' )
                    v.setSeverity(severity.MEDIUM)
                    
                    kb.kb.append( self, 'sharedHosting', v )
                    om.out.vulnerability( v.getDesc(), severity=v.getSeverity() )
                
                    msg = 'This list of domains, and the domain of the web application under test,'
                    msg += ' all point to the same server:'
                    om.out.vulnerability(msg, severity=severity.MEDIUM )
                    for url in self._shared_hosting_hosts:
                        om.out.vulnerability('- ' + url, severity=severity.MEDIUM )
                
                # Check if well parsed
                elif 'apache' in response.getBody().lower():
                    msg = 'Couldn\'t find any URLs in the apache server status page. Two things can'
                    msg += ' trigger this:\n    - The Apache web server sent a server-status page'
                    msg += ' that the serverStatus plugin failed to parse or,\n    - The remote '
                    msg += ' web server has no traffic. If you are sure about the first one, please'
                    msg += ' report a bug.'
                    om.out.information( msg )
                    om.out.debug('The server-status body is: "'+response.getBody()+'"')
        
        return res
Пример #12
0
    def setOptions(self, optionsMap):
        """
        This method sets all the options that are configured using the user interface 
        generated by the framework using the result of getOptions().
        
        @parameter optionsMap: A dictionary with the options for the plugin.
        @return: No value is returned.
        """
        targetUrls = optionsMap["target"].getValue()

        for targetUrl in targetUrls:
            self._verifyURL(targetUrl)

        for targetUrl in targetUrls:
            if targetUrl.count("file://"):
                try:
                    f = urllib2.urlopen(targetUrl)
                except:
                    raise w3afException("Cannot open target file: " + targetUrl)
                else:
                    for line in f:
                        target_in_file = line.strip()
                        self._verifyURL(target_in_file, fileTarget=False)
                        targetUrls.append(target_in_file)
                    f.close()
                targetUrls.remove(targetUrl)

        # Now we perform a check to see if the user has specified more than one target
        # domain, for example: "http://google.com, http://yahoo.com".
        domainList = [urlParser.getNetLocation(targetURL) for targetURL in targetUrls]
        domainList = list(set(domainList))
        if len(domainList) > 1:
            msg = "You specified more than one target domain: " + ",".join(domainList)
            msg += " . And w3af only supports one target domain at the time."
            raise w3afException(msg)

        # Save in the config, the target URLs, this may be usefull for some plugins.
        cf.cf.save("targets", targetUrls)
        cf.cf.save("targetDomains", [urlParser.getNetLocation(i) for i in targetUrls])
        cf.cf.save("baseURLs", [urlParser.baseUrl(i) for i in targetUrls])

        if targetUrls:
            sessName = [urlParser.getNetLocation(x) for x in targetUrls]
            sessName = "-".join(sessName)
        else:
            sessName = "noTarget"

        cf.cf.save("sessionName", sessName + "-" + time.strftime("%Y-%b-%d_%H-%M-%S"))

        # Advanced target selection
        os = optionsMap["targetOS"].getValueStr()
        if os.lower() in self._operatingSystems:
            cf.cf.save("targetOS", os.lower())
        else:
            raise w3afException("Unknown target operating system: " + os)

        pf = optionsMap["targetFramework"].getValueStr()
        if pf.lower() in self._programmingFrameworks:
            cf.cf.save("targetFramework", pf.lower())
        else:
            raise w3afException("Unknown target programming framework: " + pf)
Пример #13
0
 def discover(self, fuzzableRequest ):
     '''
     @parameter fuzzableRequest: A fuzzableRequest instance that contains (among other things) the URL to test.
     '''
     if not self._run:
         # This will remove the plugin from the discovery plugins to be runned.
         raise w3afRunOnce()
     else:
         # I will only run this one time. All calls to sharedHosting return the same url's
         self._run = False
         
         bing_wrapper = bing( self._urlOpener )
         
         domain = urlParser.getDomain( fuzzableRequest.getURL() )
         if is_private_site( domain ):
             msg = 'sharedHosting plugin is not checking for subdomains for domain: '
             msg += domain + ' because its a private address.' 
             om.out.debug(msg)
             
         else:
             # Get the ip and do the search
             addrinfo = None
             try:
                 addrinfo = socket.getaddrinfo(domain, 0)
             except:
                 raise w3afException('Could not resolve hostname: ' + domain )
             ip_address_list = [info[4][0] for info in addrinfo]
             ip_address_list = list( set(ip_address_list) )
             
             # This is the best way to search, one by one!
             for ip_address in ip_address_list:
                 results = bing_wrapper.getNResults('ip:'+ ip_address, self._result_limit )
                 
                 results = [ urlParser.baseUrl( r.URL ) for r in results ]
                 results = list( set( results ) )
                 
                 # not vuln by default
                 is_vulnerable = False
                 
                 if len(results) > 1:
                     # We may have something...
                     is_vulnerable = True
                     
                     if len(results) == 2:
                         # Maybe we have this case:
                         # [Mon 09 Jun 2008 01:08:26 PM ART] - http://216.244.147.14/
                         # [Mon 09 Jun 2008 01:08:26 PM ART] - http://www.business.com/
                         # Where www.business.com resolves to 216.244.147.14; so we don't really
                         # have more than one domain in the same server.
                         res0 = socket.gethostbyname( urlParser.getDomain( results[0] ) )
                         res1 = socket.gethostbyname( urlParser.getDomain( results[1] ) )
                         if res0 == res1:
                             is_vulnerable = False
                 
                 if is_vulnerable:
                     severityOfThisVuln = severity.MEDIUM
                     v = vuln.vuln()
                     v.setPluginName(self.getName())
                     v.setURL(fuzzableRequest.getURL())
                     v.setId(1)
                     
                     v['alsoInHosting'] = results
                     msg = 'The web application under test seems to be in a shared hosting. '
                     msg += 'This list of domains, and the domain of the web application under '
                     msg += 'test, all point to the same IP address (%s):\n' % ip_address
                     for url in results:
                         domain = urlParser.getDomain(url)
                         msg += '- %s\n' % url
                         kb.kb.append( self, 'domains', domain)
                     v.setDesc( msg )
                     v.setName( 'Shared hosting' )
                     v.setSeverity(severityOfThisVuln)
                     om.out.vulnerability( msg, severity=severityOfThisVuln )
                     kb.kb.append( self, 'sharedHosting', v )
             
     return []
Пример #14
0
                self._report_findings()
            
            return self._fuzzable_requests

    def _do_request( self, mutant, user ):
        '''
        Perform the request and compare.
        
        @return: True when the user was found.
        '''
        try:
            response = self._urlOpener.GET( mutant, useCache=True, headers=self._headers )
        except KeyboardInterrupt,e:
            raise e
        else:
            path = mutant.replace( urlParser.baseUrl( mutant ) , '' )
            response_body = response.getBody().replace( path, '')
            
            if relative_distance_lt(response_body, self._non_existant, 0.7):
                
                # Avoid duplicates
                if user not in [ u['user'] for u in kb.kb.getData( 'userDir', 'users') ]:
                    i = info.info()
                    i.setPluginName(self.getName())
                    i.setName('User directory: ' + response.getURL() )
                    i.setId( response.id )
                    i.setDesc( 'A user directory was found at: ' + response.getURL() )
                    i['user'] = user
                    
                    kb.kb.append( self, 'users', i )
                    
Пример #15
0
    def _get_dead_links(self, fuzzableRequest):
        '''
        Find every link on a HTML document verify if the domain is reachable or not; after that,
        verify if the web found a different name for the target site or if we found a new site that
        is linked. If the link points to a dead site then report it (it could be pointing to some 
        private address or something...)
        '''
        res = []
        
        # Get some responses to compare later
        base_url = urlParser.baseUrl(fuzzableRequest.getURL())
        original_response = self._urlOpener.GET(fuzzableRequest.getURI(), useCache=True)
        base_response = self._urlOpener.GET(base_url, useCache=True)
        base_resp_body = base_response.getBody()
        
        try:
            dp = dpCache.dpc.getDocumentParserFor(original_response)
        except w3afException:
            # Failed to find a suitable parser for the document
            return []
        
        # Set the non existant response
        non_existant = 'iDoNotExistPleaseGoAwayNowOrDie' + createRandAlNum(4) 
        self._non_existant_response = self._urlOpener.GET(base_url, 
                                                useCache=False, headers={'Host': non_existant})
        nonexist_resp_body = self._non_existant_response.getBody()
        
        # Note:
        # - With parsed_references I'm 100% that it's really something in the HTML
        # that the developer intended to add.
        #
        # - The re_references are the result of regular expressions, which in some cases
        # are just false positives.
        #
        # In this case, and because I'm only going to use the domain name of the URL
        # I'm going to trust the re_references also.
        parsed_references, re_references = dp.getReferences()
        parsed_references.extend(re_references)
        
        for link in parsed_references:
            domain = urlParser.getDomain(link)
            
            #
            # First section, find internal hosts using the HTTP Host header:
            #
            if domain not in self._already_queried:
                # If the parsed page has an external link to www.google.com
                # then I'll send a request to the target site, with Host: www.google.com
                # This sucks, but it's cool if the document has a link to 
                # http://some.internal.site.target.com/
                try:
                    vhost_response = self._urlOpener.GET(base_url, useCache=False,
                                                         headers={'Host': domain })
                except w3afException:
                    pass
                else:
                    self._already_queried.add(domain)
                    vhost_resp_body = vhost_response.getBody()
                    
                    # If they are *really* different (not just different by some chars)
                    if relative_distance_lt(vhost_resp_body, base_resp_body, 0.35) and \
                        relative_distance_lt(vhost_resp_body, nonexist_resp_body, 0.35):
                        # and the domain can't just be resolved using a DNS query to
                        # our regular DNS server
                        report = True
                        if self._can_resolve_domain_names:
                            try:
                                socket.gethostbyname(domain)
                            except:
                                # aha! The HTML is linking to a domain that's
                                # hosted in the same server, and the domain name
                                # can NOT be resolved!
                                report = True
                            else:
                                report = False

                        # have found something interesting!
                        if report:
                            res.append( (domain, vhost_response.id) )

            #
            # Second section, find hosts using failed DNS resolutions
            #
            if self._can_resolve_domain_names:
                try:
                    # raises exception when it's not found
                    # socket.gaierror: (-5, 'No address associated with hostname')
                    socket.gethostbyname( domain )
                except:
                    i = info.info()
                    i.setPluginName(self.getName())
                    i.setName('Internal hostname in HTML link')
                    i.setURL( fuzzableRequest.getURL() )
                    i.setMethod( 'GET' )
                    i.setId( original_response.id )
                    msg = 'The content of "'+ fuzzableRequest.getURL() +'" references a non '
                    msg += 'existant domain: "' + link + '". This may be a broken link, or an'
                    msg += ' internal domain name.'
                    i.setDesc( msg )
                    kb.kb.append( self, 'findvhost', i )
                    om.out.information( i.getDesc() )
        
        res = [ r for r in res if r != '']
        
        return res