def test_cachedb_consistency(): mem = VolatileNetworkCache() disk = PersistentNetworkCache() disk.clean(audit) try: print "Testing consistency of in-memory and disk caches..." for x in xrange(100): key = generate_random_string(10) data = generate_random_string(100) helper_test_cachedb_consistency(mem, key, data) helper_test_cachedb_consistency(disk, key, data) print "Testing disk cache compacting and dumping..." disk.compact() disk.dump("test_cachedb.sql") print "Cleaning up the memory cache..." mem.clean(audit) del mem finally: print "Cleaning up the disk cache..." try: disk.clean(audit) finally: disk.close()
def test_auditdb_dump(): main_config = OrchestratorConfig() main_config.ui_mode = "disabled" audit_config = AuditConfig() audit_config.targets = ["www.example.com"] audit_config.audit_db = "sqlite://test_auditdb.db" with PluginTester(main_config, audit_config) as t: disk = t.audit.database assert t.audit.name == "test_auditdb" assert type(disk) is AuditSQLiteDB assert disk.filename == "test_auditdb.db" assert disk.connection_url == "sqlite://test_auditdb.db" print "Testing the audit database dump..." print " -> Writing..." for x in xrange(30): d1 = Url("http://www.example.com/" + generate_random_string()) d2 = Text(generate_random_string()) d3 = UrlDisclosure(d1) d1.add_information(d2) disk.add_data(d1) disk.add_data(d2) disk.add_data(d3) disk.mark_plugin_finished(d1.identity, "some_plugin") disk.mark_plugin_finished(d2.identity, "some_plugin") disk.mark_plugin_finished(d3.identity, "some_plugin") disk.mark_stage_finished(d1.identity, 1) disk.mark_stage_finished(d2.identity, 2) disk.mark_stage_finished(d3.identity, 3) disk.add_shared_values("fake_set_id", ( "string", u"unicode", 100, 200L, 5.0, True, False, complex(1, 1), None, frozenset({"string", 100, 1.0}), (None, True, False), )) disk.put_mapped_values("fake_map_id", ( ("a_string", "string"), ("a_unicode_string", u"unicode"), ("an_integer", 100), ("a_long", 200L), ("a_float", 5.0), ("a_bool", True), ("another_bool", False), ("a_complex", complex(1, 1)), ("none", None), ("a_frozenset", frozenset({"string", 100, 1.0})), ("a_tuple", (None, True, False)), )) print " -> Dumping..." disk.dump("test_auditdb.sql")
def helper_test_auditdb_consistency_setup(audit_name, audit_db): main_config = OrchestratorConfig() main_config.ui_mode = "disabled" audit_config = AuditConfig() audit_config.targets = ["www.example.com"] audit_config.audit_name = audit_name audit_config.audit_db = audit_db with PluginTester(main_config, audit_config) as t: print "--> Testing general consistency..." helper_test_auditdb_general_consistency(t.audit.database) print "--> Testing data consistency..." for x in xrange(100): key = generate_random_string(10) data = generate_random_string(100) helper_test_auditdb_data_consistency(t.audit.database, key, data)
def helper_cachedb_stress(disk, n): print " Testing %d items..." % (n * 2) data1 = "A" * 10000000 data2 = "B" * 10000000 keys = set() for x in xrange(n): key = generate_random_string() keys.add(key) t1 = time.time() print " -> Writing..." for key in keys: disk.set(audit, key, data1, protocol="http") disk.set(audit, key, data2, protocol="https") t2 = time.time() print " -- Checking..." for key in keys: assert disk.exists(audit, key, protocol="http") assert disk.exists(audit, key, protocol="https") t3 = time.time() print " <- Reading..." for key in keys: assert disk.get(audit, key, protocol="http") != disk.get(audit, key, protocol="https") t4 = time.time() print " Write time: %d seconds (%f seconds per item)" % (t2 - t1, (t2 - t1) / (n * 2.0)) print " Check time: %d seconds (%f seconds per item)" % (t3 - t2, (t3 - t2) / (n * 2.0)) print " Read time: %d seconds (%f seconds per item)" % (t4 - t3, (t4 - t3) / (n * 2.0)) print " Total time: %d seconds (%f seconds per item)" % (t4 - t1, (t4 - t1) / (n * 2.0))
def get_error_page(url): """ Generates an error page an get their content. :param url: string with the base Url. :type url: str :return: a string with the content of response. :rtype: str """ # # Generate an error in server to get an error page, using a random string # # Make the URL m_error_url = "%s%s" % (url, generate_random_string()) # Get the request m_error_response = HTTP.get_url(m_error_url) # FIXME handle exceptions! discard_data(m_error_response) m_error_response = m_error_response.data
def helper_auditdb_stress(n, dbname = ":auto:"): main_config = OrchestratorConfig() main_config.ui_mode = "disabled" audit_config = AuditConfig() audit_config.targets = ["www.example.com"] audit_config.audit_db = dbname with PluginTester(main_config, audit_config) as t: disk = t.audit.database assert type(disk) is AuditSQLiteDB print " Testing %d elements..." % (n * 3) t1 = time.time() print " -> Writing..." for x in xrange(n): d1 = Url("http://www.example.com/" + generate_random_string()) d2 = Text(generate_random_string()) d3 = UrlDisclosure(d1) d1.add_information(d2) disk.add_data(d1) disk.add_data(d2) disk.add_data(d3) t2 = time.time() print " -- Reading..." keys = disk.get_data_keys() assert len(keys) == (n * 3) for key in keys: assert disk.has_data_key(key) data = disk.get_data(key) assert data is not None keys = disk.get_data_keys(Data.TYPE_INFORMATION) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_INFORMATION) data = disk.get_data(key, Data.TYPE_INFORMATION) assert data is not None assert data.data_type == Data.TYPE_INFORMATION assert isinstance(data, Text) keys = disk.get_data_keys(Data.TYPE_RESOURCE) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_RESOURCE) data = disk.get_data(key, Data.TYPE_RESOURCE) assert data is not None assert data.data_type == Data.TYPE_RESOURCE assert isinstance(data, Url) keys = disk.get_data_keys(Data.TYPE_VULNERABILITY) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_VULNERABILITY) data = disk.get_data(key, Data.TYPE_VULNERABILITY) assert data is not None assert data.data_type == Data.TYPE_VULNERABILITY assert isinstance(data, UrlDisclosure) t3 = time.time() print " <- Deleting..." for key in keys: disk.remove_data(key) t4 = time.time() print " Write time: %d seconds (%f seconds per element)" % (t2 - t1, (t2 - t1) / (n * 3.0)) print " Read time: %d seconds (%f seconds per element)" % (t3 - t2, (t3 - t2) / (n * 3.0)) print " Delete time: %d seconds (%f seconds per element)" % (t4 - t3, (t4 - t3) / (n * 3.0)) print " Total time: %d seconds (%f seconds per element)" % (t4 - t1, (t4 - t1) / (n * 3.0))
def launch_scan(self, target, **kwargs): """ Launch a new audit in OpenVAS. This is an example code to launch an OpenVAS scan and wait for it to complete:: from threading import Semaphore from functools import partial def my_print_status(i): print str(i) def my_launch_scanner(): Sem = Semaphore(0) # Configure manager = VulnscanManager.connectOpenVAS("localhost", "admin", "admin) # Launch manager.launch_scan( target, profile = "empty", callback_end = partial(lambda x: x.release(), sem), callback_progress = my_print_status ) # Wait Sem.acquire() # Finished scan print "finished!" # >>> my_launch_scanner() # It can take some time # 0 # 10 # 39 # 60 # 90 # finished! :param target: Target to audit. :type target: str :param profile: Scan profile in the OpenVAS server. :type profile: str :param callback_end: If this param is set, the process will run in background and call the function specified in this var when the scan ends. :type callback_end: function :param callback_progress: If this param is set, it will be called every 10 seconds, with the progress percentaje as a float. :type callback_progress: function(float) :return: ID of the audit and ID of the target: (ID_scan, ID_target) :rtype: (str, str) """ profile = kwargs.get("profile", "Full and fast") call_back_end = kwargs.get("callback_end", None) call_back_progress = kwargs.get("callback_progress", None) if not (isinstance(target, basestring) or isinstance(target, Iterable)): raise TypeError("Expected basestring or iterable, got %r instead" % type(target)) if not isinstance(profile, basestring): raise TypeError("Expected string, got %r instead" % type(profile)) # Generate the random names used m_target_name = "golismero_target_%s" % generate_random_string(20) m_job_name = "golismero_scan_%s" % generate_random_string(20) # Create the target try: m_target_id = self.__manager.create_target(m_target_name, target, "Temporal target from golismero OpenVAS plugin") except ServerError, e: raise VulnscanTargetError("The target already exits on the server. Error: %s" % e.message)
def recv_info(self, info): # Get the root domain only. root = info.root # Skip localhost. if root == "localhost": return # Skip root domains we've already processed. if self.state.put(root, True): return # Load the subdomains wordlist. try: wordlist = WordListLoader.get_advanced_wordlist_as_list(Config.plugin_args["wordlist"]) except WordlistNotFound: Logger.log_error_verbose("Wordlist '%s' not found.." % Config.plugin_args["wordlist"]) return except TypeError: Logger.log_error_verbose("Wordlist '%s' is not a file." % Config.plugin_args["wordlist"]) return # Load the subdomains whitelist. try: whitelist = WordListLoader.get_advanced_wordlist_as_list(Config.plugin_config["wordlist"]) except WordlistNotFound: Logger.log_error_verbose("Wordlist '%s' not found.." % Config.plugin_config["wordlist"]) return except TypeError: Logger.log_error_verbose("Wordlist '%s' is not a file." % Config.plugin_config["wordlist"]) return # # Set a base line for dinamyc sub-domains # m_virtual_domains = [] for v in (generate_random_string(40) for x in xrange(3)): l_subdomain = ".".join((v, root)) records = DNS.get_a(l_subdomain, also_CNAME=True) for rec in records: if rec.type == "CNAME": m_virtual_domains.append(rec.target) # If 3 subdomains are the same, set the base domain m_base_domain = None if len(set(m_virtual_domains)) == 1: m_base_domain = m_virtual_domains[0] # Configure the progress notifier. self.progress.set_total(len(wordlist)) self.progress.min_delta = 1 # notify every 1% # For each subdomain in the wordlist... found = 0 results = [] visited = set() for prefix in wordlist: # Mark as completed before actually trying. # We can't put this at the end of the loop where it belongs, # because the "continue" statements would skip over this too. self.progress.add_completed() # Build the domain name. name = ".".join((prefix, root)) # Skip if out of scope. if name not in Config.audit_scope: continue # Resolve the subdomain. records = DNS.get_a(name, also_CNAME=True) records.extend( DNS.get_aaaa(name, also_CNAME=True) ) # If no DNS records were found, skip. if not records: continue # If CNAME is the base domain, skip chk = [True for x in records if x.type == "CNAME" and x.target == m_base_domain] if len(chk) > 0 and all(chk): continue # We found a subdomain! found += 1 Logger.log_more_verbose( "Subdomain found: %s" % name) # Create the Domain object for the subdomain. domain = Domain(name) results.append(domain) # # Check for Domain disclosure # if prefix not in whitelist: d = DomainDisclosure(name, risk = 0, level = "low", title = "Possible subdomain leak", description = "A subdomain was discovered which may be an unwanted information disclosure." ) d.add_resource(domain) results.append(d) # For each DNs record, grab the address or name. # Skip duplicated records. for rec in records: if rec.type == "CNAME": location = rec.target elif rec.type in ("A", "AAAA"): location = rec.address else: # should not happen... results.append(rec) domain.add_information(rec) continue if location not in visited: visited.add(location) results.append(rec) domain.add_information(rec) # Log the results. if found: Logger.log( "Found %d subdomains for root domain: %s" % (found, root)) else: Logger.log_verbose( "No subdomains found for root domain: %s" % root) # Return the results. return results
def launch_scan(self, target, **kwargs): """ Launch a new audit in OpenVAS. This is an example code to launch an OpenVAS scan and wait for it to complete:: from threading import Semaphore from functools import partial def my_print_status(i): print str(i) def my_launch_scanner(): Sem = Semaphore(0) # Configure manager = VulnscanManager.connectOpenVAS("localhost", "admin", "admin) # Launch manager.launch_scan( target, profile = "empty", callback_end = partial(lambda x: x.release(), sem), callback_progress = my_print_status ) # Wait Sem.acquire() # Finished scan print "finished!" # >>> my_launch_scanner() # It can take some time # 0 # 10 # 39 # 60 # 90 # finished! :param target: Target to audit. :type target: str :param profile: Scan profile in the OpenVAS server. :type profile: str :param callback_end: If this param is set, the process will run in background and call the function specified in this var when the scan ends. :type callback_end: function :param callback_progress: If this param is set, it will be called every 10 seconds, with the progress percentaje as a float. :type callback_progress: function(float) :return: ID of the audit and ID of the target: (ID_scan, ID_target) :rtype: (str, str) """ profile = kwargs.get("profile", "Full and fast") call_back_end = kwargs.get("callback_end", None) call_back_progress = kwargs.get("callback_progress", None) if not (isinstance(target, basestring) or isinstance(target, Iterable)): raise TypeError("Expected basestring or iterable, got %r instead" % type(target)) if not isinstance(profile, basestring): raise TypeError("Expected string, got %r instead" % type(profile)) # Generate the random names used m_target_name = "golismero_target_%s" % generate_random_string(20) m_job_name = "golismero_scan_%s" % generate_random_string(20) # Create the target try: m_target_id = self.__manager.create_target( m_target_name, target, "Temporal target from golismero OpenVAS plugin") except ServerError, e: raise VulnscanTargetError( "The target already exits on the server. Error: %s" % e.message)
def test_input(self): assert len(generate_random_string(40)) == 40
def test_negative(self): assert len(generate_random_string(-4)) == 0
def helper_auditdb_stress(n): main_config = OrchestratorConfig() main_config.ui_mode = "disabled" audit_config = AuditConfig() audit_config.targets = ["www.example.com"] audit_config.audit_db = "sqlite://" with PluginTester(main_config, audit_config) as t: disk = t.audit.database assert type(disk) is AuditSQLiteDB print " Testing %d elements..." % (n * 3) t1 = time.time() print " -> Writing..." for x in xrange(n): d1 = Url("http://www.example.com/" + generate_random_string()) d2 = Text(generate_random_string()) d3 = UrlDisclosure(d1) d1.add_information(d2) disk.add_data(d1) disk.add_data(d2) disk.add_data(d3) t2 = time.time() print " -- Reading..." keys = disk.get_data_keys() assert len(keys) == (n * 3) for key in keys: assert disk.has_data_key(key) data = disk.get_data(key) assert data is not None keys = disk.get_data_keys(Data.TYPE_INFORMATION) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_INFORMATION) data = disk.get_data(key, Data.TYPE_INFORMATION) assert data is not None assert data.data_type == Data.TYPE_INFORMATION assert isinstance(data, Text) keys = disk.get_data_keys(Data.TYPE_RESOURCE) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_RESOURCE) data = disk.get_data(key, Data.TYPE_RESOURCE) assert data is not None assert data.data_type == Data.TYPE_RESOURCE assert isinstance(data, Url) keys = disk.get_data_keys(Data.TYPE_VULNERABILITY) assert len(keys) == n for key in keys: assert disk.has_data_key(key, Data.TYPE_VULNERABILITY) data = disk.get_data(key, Data.TYPE_VULNERABILITY) assert data is not None assert data.data_type == Data.TYPE_VULNERABILITY assert isinstance(data, UrlDisclosure) t3 = time.time() print " <- Deleting..." for key in keys: disk.remove_data(key) t4 = time.time() print " Write time: %d seconds (%f seconds per element)" % ( t2 - t1, (t2 - t1) / (n * 3.0)) print " Read time: %d seconds (%f seconds per element)" % ( t3 - t2, (t3 - t2) / (n * 3.0)) print " Delete time: %d seconds (%f seconds per element)" % ( t4 - t3, (t4 - t3) / (n * 3.0)) print " Total time: %d seconds (%f seconds per element)" % ( t4 - t1, (t4 - t1) / (n * 3.0))
def run(self, info): # Get the root domain only. root = info.root # Skip localhost. if root == "localhost": return # Skip root domains we've already processed. if self.state.put(root, True): return # Load the subdomains wordlist. try: wordlist = WordListLoader.get_wordlist_as_list(Config.plugin_args["wordlist"]) except WordlistNotFound: Logger.log_error_verbose("Wordlist '%s' not found.." % Config.plugin_args["wordlist"]) return except TypeError: Logger.log_error_verbose("Wordlist '%s' is not a file." % Config.plugin_args["wordlist"]) return # Load the subdomains whitelist. try: whitelist = WordListLoader.get_wordlist_as_list(Config.plugin_config["wordlist"]) except WordlistNotFound: Logger.log_error_verbose("Wordlist '%s' not found.." % Config.plugin_config["wordlist"]) return except TypeError: Logger.log_error_verbose("Wordlist '%s' is not a file." % Config.plugin_config["wordlist"]) return # # Set a base line for dinamyc sub-domains # m_virtual_domains = [] for v in (generate_random_string(40) for x in xrange(3)): l_subdomain = ".".join((v, root)) records = DNS.get_a(l_subdomain, also_CNAME=True) for rec in records: if rec.type == "CNAME": m_virtual_domains.append(rec.target) # If 3 subdomains are the same, set the base domain m_base_domain = None if len(set(m_virtual_domains)) == 1: m_base_domain = m_virtual_domains[0] # Configure the progress notifier. self.progress.set_total(len(wordlist)) self.progress.min_delta = 1 # notify every 1% # For each subdomain in the wordlist... found = 0 results = [] visited = set() for prefix in wordlist: # Mark as completed before actually trying. # We can't put this at the end of the loop where it belongs, # because the "continue" statements would skip over this too. self.progress.add_completed() # Build the domain name. name = ".".join((prefix, root)) # Skip if out of scope. if name not in Config.audit_scope: continue # Resolve the subdomain. records = DNS.get_a(name, also_CNAME=True) records.extend( DNS.get_aaaa(name, also_CNAME=True) ) # If no DNS records were found, skip. if not records: continue # If CNAME is the base domain, skip chk = [True for x in records if x.type == "CNAME" and x.target == m_base_domain] if len(chk) > 0 and all(chk): continue # We found a subdomain! found += 1 Logger.log_more_verbose( "Subdomain found: %s" % name) # Create the Domain object for the subdomain. domain = Domain(name) results.append(domain) # # Check for Domain disclosure # if prefix not in whitelist: d = DomainDisclosure(domain, risk = 0, level = "low", title = "Possible subdomain leak", description = "A subdomain was discovered which may be an unwanted information disclosure." ) results.append(d) # For each DNs record, grab the address or name. # Skip duplicated records. for rec in records: if rec.type == "CNAME": location = rec.target elif rec.type in ("A", "AAAA"): location = rec.address else: # should not happen... results.append(rec) domain.add_information(rec) continue if location not in visited: visited.add(location) results.append(rec) domain.add_information(rec) # Log the results. if found: Logger.log( "Found %d subdomains for root domain: %s" % (found, root)) else: Logger.log_verbose( "No subdomains found for root domain: %s" % root) # Return the results. return results