def test_auditdb_dump():
main_config = OrchestratorConfig()
main_config.ui_mode = "disabled"
audit_config = AuditConfig()
audit_config.targets = ["www.example.com"]
audit_config.audit_db = "sqlite://test_auditdb.db"
with PluginTester(main_config, audit_config) as t:
disk = t.audit.database
assert t.audit.name == "test_auditdb"
assert type(disk) is AuditSQLiteDB
assert disk.filename == "test_auditdb.db"
assert disk.connection_url == "sqlite://test_auditdb.db"
print "Testing the audit database dump..."
print " -> Writing..."
for x in xrange(30):
d1 = Url("http://www.example.com/" + generate_random_string())
d2 = Text(generate_random_string())
d3 = UrlDisclosure(d1)
d1.add_information(d2)
disk.add_data(d1)
disk.add_data(d2)
disk.add_data(d3)
disk.mark_plugin_finished(d1.identity, "some_plugin")
disk.mark_plugin_finished(d2.identity, "some_plugin")
disk.mark_plugin_finished(d3.identity, "some_plugin")
disk.mark_stage_finished(d1.identity, 1)
disk.mark_stage_finished(d2.identity, 2)
disk.mark_stage_finished(d3.identity, 3)
disk.add_shared_values("fake_set_id", (
"string",
u"unicode",
100,
200L,
5.0,
True,
False,
complex(1, 1),
None,
frozenset({"string", 100, 1.0}),
(None, True, False),
))
disk.put_mapped_values("fake_map_id", (
("a_string", "string"),
("a_unicode_string", u"unicode"),
("an_integer", 100),
("a_long", 200L),
("a_float", 5.0),
("a_bool", True),
("another_bool", False),
("a_complex", complex(1, 1)),
("none", None),
("a_frozenset", frozenset({"string", 100, 1.0})),
("a_tuple", (None, True, False)),
))
print " -> Dumping..."
disk.dump("test_auditdb.sql")
def helper_test_auditdb_data_consistency(db, key, data):
assert isinstance(db, BaseAuditDB)
# Test the database start and end times.
db.set_audit_times(None, None)
assert db.get_audit_times() == (None, None)
db.set_audit_start_time(1)
assert db.get_audit_times() == (1, None)
db.set_audit_stop_time(2)
assert db.get_audit_times() == (1, 2)
db.set_audit_start_time(None)
assert db.get_audit_times() == (None, 2)
db.set_audit_stop_time(None)
assert db.get_audit_times() == (None, None)
# Create some fake data and add it to the database.
d1 = Url("http://www.example.com/" + key)
d2 = Text(data)
d3 = UrlDisclosure(d1)
d1.add_information(d2)
assert d1.links == {d2.identity, d3.identity}
assert d2.links == {d1.identity}
assert d3.links == {d1.identity}
db.add_data(d1)
db.add_data(d2)
db.add_data(d3)
# Test has_data_key().
assert db.has_data_key(d1.identity)
assert db.has_data_key(d2.identity)
assert db.has_data_key(d3.identity)
assert db.has_data_key(d1.identity, d1.data_type)
assert db.has_data_key(d2.identity, d2.data_type)
assert db.has_data_key(d3.identity, d3.data_type)
# Test get_data().
d1p = db.get_data(d1.identity)
d2p = db.get_data(d2.identity)
d3p = db.get_data(d3.identity)
assert d1p is not None
assert d2p is not None
assert d3p is not None
assert d1p.identity == d1.identity
assert d2p.identity == d2.identity
assert d3p.identity == d3.identity
assert d1p.links == d1.links
assert d2p.links == d2.links
assert d3p.links == d3.links
# Test get_data_types().
assert db.get_data_types((d1.identity, d2.identity, d3.identity)) == {
(d1.data_type, d1.resource_type), (d2.data_type, d2.information_type),
(d3.data_type, d3.vulnerability_type)
}
# Test get_data_count().
assert db.get_data_count() == 3
assert db.get_data_count(d1.data_type) == 1
assert db.get_data_count(d2.data_type) == 1
assert db.get_data_count(d3.data_type) == 1
assert db.get_data_count(d1.data_type, d1.resource_type) == 1
assert db.get_data_count(d2.data_type, d2.information_type) == 1
assert db.get_data_count(d3.data_type, d3.vulnerability_type) == 1
# Test get_many_data().
assert {
x.identity
for x in db.get_many_data((d1.identity, d2.identity, d3.identity))
} == {d1.identity, d2.identity, d3.identity}
# Test stage and plugin completion logic.
# XXX TODO
# Test remove_data().
db.remove_data(d1.identity)
db.remove_data(d2.identity)
db.remove_data(d3.identity)
assert not db.has_data_key(d1.identity)
assert not db.has_data_key(d2.identity)
assert not db.has_data_key(d3.identity)
assert not db.has_data_key(d1.identity, d1.data_type)
assert not db.has_data_key(d2.identity, d2.data_type)
assert not db.has_data_key(d3.identity, d3.data_type)
assert db.get_data_count() == 0
assert db.get_data_count(d1.data_type) == 0
assert db.get_data_count(d2.data_type) == 0
assert db.get_data_count(d3.data_type) == 0
assert db.get_data_count(d1.data_type, d1.resource_type) == 0
assert db.get_data_count(d2.data_type, d2.information_type) == 0
assert db.get_data_count(d3.data_type, d3.vulnerability_type) == 0
assert db.get_data_types((d1.identity, d2.identity, d3.identity)) == set()
assert db.get_data(d1.identity) == None
assert db.get_data(d2.identity) == None
assert db.get_data(d3.identity) == None
def helper_auditdb_stress(n):
main_config = OrchestratorConfig()
main_config.ui_mode = "disabled"
audit_config = AuditConfig()
audit_config.targets = ["www.example.com"]
audit_config.audit_db = "sqlite://"
with PluginTester(main_config, audit_config) as t:
disk = t.audit.database
assert type(disk) is AuditSQLiteDB
print " Testing %d elements..." % (n * 3)
t1 = time.time()
print " -> Writing..."
for x in xrange(n):
d1 = Url("http://www.example.com/" + generate_random_string())
d2 = Text(generate_random_string())
d3 = UrlDisclosure(d1)
d1.add_information(d2)
disk.add_data(d1)
disk.add_data(d2)
disk.add_data(d3)
t2 = time.time()
print " -- Reading..."
keys = disk.get_data_keys()
assert len(keys) == (n * 3)
for key in keys:
assert disk.has_data_key(key)
data = disk.get_data(key)
assert data is not None
keys = disk.get_data_keys(Data.TYPE_INFORMATION)
assert len(keys) == n
for key in keys:
assert disk.has_data_key(key, Data.TYPE_INFORMATION)
data = disk.get_data(key, Data.TYPE_INFORMATION)
assert data is not None
assert data.data_type == Data.TYPE_INFORMATION
assert isinstance(data, Text)
keys = disk.get_data_keys(Data.TYPE_RESOURCE)
assert len(keys) == n
for key in keys:
assert disk.has_data_key(key, Data.TYPE_RESOURCE)
data = disk.get_data(key, Data.TYPE_RESOURCE)
assert data is not None
assert data.data_type == Data.TYPE_RESOURCE
assert isinstance(data, Url)
keys = disk.get_data_keys(Data.TYPE_VULNERABILITY)
assert len(keys) == n
for key in keys:
assert disk.has_data_key(key, Data.TYPE_VULNERABILITY)
data = disk.get_data(key, Data.TYPE_VULNERABILITY)
assert data is not None
assert data.data_type == Data.TYPE_VULNERABILITY
assert isinstance(data, UrlDisclosure)
t3 = time.time()
print " <- Deleting..."
for key in keys:
disk.remove_data(key)
t4 = time.time()
print " Write time: %d seconds (%f seconds per element)" % (
t2 - t1, (t2 - t1) / (n * 3.0))
print " Read time: %d seconds (%f seconds per element)" % (
t3 - t2, (t3 - t2) / (n * 3.0))
print " Delete time: %d seconds (%f seconds per element)" % (
t4 - t3, (t4 - t3) / (n * 3.0))
print " Total time: %d seconds (%f seconds per element)" % (
t4 - t1, (t4 - t1) / (n * 3.0))
def helper_data_links():
# Create some dummy data.
from golismero.api.data.resource.url import URL
from golismero.api.data.information.text import Text
from golismero.api.data.vulnerability.information_disclosure.url_disclosure import UrlDisclosure
d1 = URL("http://www.example.com/")
d2 = Text("some text")
d3 = UrlDisclosure(d1)
d1.add_information(d2)
# Test data_type, data_subtype, etc.
print "Testing Data type checks..."
assert d1.data_type == Data.TYPE_RESOURCE
assert d1.data_subtype == URL.data_subtype
assert d1.resource_type == d1.data_subtype
assert d2.data_type == Data.TYPE_INFORMATION
assert d2.data_subtype == Text.data_subtype
assert d2.information_type == d2.data_subtype
assert d3.data_type == Data.TYPE_VULNERABILITY
assert d3.data_subtype == UrlDisclosure.data_subtype
assert d3.vulnerability_type == d3.data_subtype
# Test validate_link_minimums().
print "Testing Data.validate_link_minimums()..."
d1.validate_link_minimums()
d2.validate_link_minimums()
d3.validate_link_minimums()
# Test the links property.
print "Testing Data.links..."
assert d1.links == {d2.identity, d3.identity}
assert d2.links == {d1.identity}
assert d3.links == {d1.identity}
# Test the get_links method.
print "Testing Data.get_links()..."
assert d1.get_links(d1.data_type) == set()
assert d1.get_links(d1.data_type, d1.resource_type) == set()
assert d1.get_links(d2.data_type) == {d2.identity}
assert d1.get_links(d2.data_type, d2.information_type) == {d2.identity}
assert d1.get_links(d3.data_type) == {d3.identity}
assert d1.get_links(d3.data_type, d3.vulnerability_type) == {d3.identity}
assert d2.get_links(d2.data_type) == set()
assert d2.get_links(d2.data_type, d2.information_type) == set()
assert d2.get_links(d1.data_type) == {d1.identity}
assert d2.get_links(d1.data_type, d1.resource_type) == {d1.identity}
assert d2.get_links(d3.data_type) == set()
assert d2.get_links(d3.data_type, d3.vulnerability_type) == set()
assert d3.get_links(d3.data_type) == set()
assert d3.get_links(d3.data_type, d3.vulnerability_type) == set()
assert d3.get_links(d1.data_type) == {d1.identity}
assert d3.get_links(d1.data_type, d1.resource_type) == {d1.identity}
assert d3.get_links(d2.data_type) == set()
assert d3.get_links(d2.data_type, d2.information_type) == set()
# Test the linked_data property.
# There should be no accesses to the database since all data is local.
print "Testing Data.linked_data..."
assert {x.identity for x in d1.linked_data} == {d2.identity, d3.identity}
assert {x.identity for x in d2.linked_data} == {d1.identity}
assert {x.identity for x in d3.linked_data} == {d1.identity}
# Test the get_linked_data() method.
# There should be no accesses to the database since all data is local.
print "Testing Data.get_linked_data()..."
assert {x.identity for x in d1.find_linked_data(d1.data_type)} == set()
assert {
x.identity
for x in d1.find_linked_data(d1.data_type, d1.resource_type)
} == set()
assert {x.identity
for x in d1.find_linked_data(d2.data_type)} == {d2.identity}
assert {
x.identity
for x in d1.find_linked_data(d2.data_type, d2.information_type)
} == {d2.identity}
assert {x.identity
for x in d1.find_linked_data(d3.data_type)} == {d3.identity}
assert {
x.identity
for x in d1.find_linked_data(d3.data_type, d3.vulnerability_type)
} == {d3.identity}
assert {x.identity for x in d2.find_linked_data(d2.data_type)} == set()
assert {
x.identity
for x in d2.find_linked_data(d2.data_type, d2.information_type)
} == set()
assert {x.identity
for x in d2.find_linked_data(d1.data_type)} == {d1.identity}
assert {
x.identity
for x in d2.find_linked_data(d1.data_type, d1.resource_type)
} == {d1.identity}
assert {x.identity for x in d2.find_linked_data(d3.data_type)} == set()
assert {
x.identity
for x in d2.find_linked_data(d3.data_type, d3.vulnerability_type)
} == set()
assert {x.identity for x in d3.find_linked_data(d3.data_type)} == set()
assert {
x.identity
for x in d3.find_linked_data(d3.data_type, d3.vulnerability_type)
} == set()
assert {x.identity
for x in d3.find_linked_data(d1.data_type)} == {d1.identity}
assert {
x.identity
for x in d3.find_linked_data(d1.data_type, d1.resource_type)
} == {d1.identity}
assert {x.identity for x in d3.find_linked_data(d2.data_type)} == set()
assert {
x.identity
for x in d3.find_linked_data(d2.data_type, d2.information_type)
} == set()
# Test the associated_* properties.
# There should be no accesses to the database since all data is local.
print "Testing Data.associated_*..."
assert {x.identity for x in d1.associated_resources} == set()
assert {x.identity for x in d1.associated_informations} == {d2.identity}
assert {x.identity for x in d1.associated_vulnerabilities} == {d3.identity}
assert {x.identity for x in d2.associated_informations} == set()
assert {x.identity for x in d2.associated_resources} == {d1.identity}
assert {x.identity for x in d2.associated_vulnerabilities} == set()
assert {x.identity for x in d3.associated_vulnerabilities} == set()
assert {x.identity for x in d3.associated_resources} == {d1.identity}
assert {x.identity for x in d3.associated_informations} == set()
# Test the get_associated_*_by_category() methods.
# There should be no accesses to the database since all data is local.
print "Testing Data.get_associated_*_by_category()..."
assert {
x.identity
for x in d1.get_associated_resources_by_category(d1.resource_type)
} == set()
assert {
x.identity
for x in d1.get_associated_informations_by_category(
d2.information_type)
} == {d2.identity}
assert {
x.identity
for x in d1.get_associated_vulnerabilities_by_category(
d3.vulnerability_type)
} == {d3.identity}
assert {
x.identity
for x in d2.get_associated_informations_by_category(
d2.information_type)
} == set()
assert {
x.identity
for x in d2.get_associated_resources_by_category(d1.resource_type)
} == {d1.identity}
assert {
x.identity
for x in d2.get_associated_vulnerabilities_by_category(
d3.vulnerability_type)
} == set()
assert {
x.identity
for x in d3.get_associated_vulnerabilities_by_category(
d3.vulnerability_type)
} == set()
assert {
x.identity
for x in d3.get_associated_resources_by_category(d1.resource_type)
} == {d1.identity}
assert {
x.identity
for x in d3.get_associated_informations_by_category(
d2.information_type)
} == set()
# Test TempDataStorage.on_finish().
print "Testing LocalDataCache.on_finish() on ideal conditions..."
result = LocalDataCache.on_finish([d2, d3], d1)
assert set(result) == set([d1, d2, d3])
d1.validate_link_minimums()
d2.validate_link_minimums()
d3.validate_link_minimums()
assert d1.links == {d2.identity, d3.identity}
assert d2.links == {d1.identity}
assert d3.links == {d1.identity}
assert d1.get_links(d1.data_type) == set()
assert d1.get_links(d1.data_type, d1.resource_type) == set()
assert d1.get_links(d2.data_type) == {d2.identity}
assert d1.get_links(d2.data_type, d2.information_type) == {d2.identity}
assert d1.get_links(d3.data_type) == {d3.identity}
assert d1.get_links(d3.data_type, d3.vulnerability_type) == {d3.identity}
assert d2.get_links(d2.data_type) == set()
assert d2.get_links(d2.data_type, d2.information_type) == set()
assert d2.get_links(d1.data_type) == {d1.identity}
assert d2.get_links(d1.data_type, d1.resource_type) == {d1.identity}
assert d2.get_links(d3.data_type) == set()
assert d2.get_links(d3.data_type, d3.vulnerability_type) == set()
assert d3.get_links(d3.data_type) == set()
assert d3.get_links(d3.data_type, d3.vulnerability_type) == set()
assert d3.get_links(d1.data_type) == {d1.identity}
assert d3.get_links(d1.data_type, d1.resource_type) == {d1.identity}
assert d3.get_links(d2.data_type) == set()
assert d3.get_links(d2.data_type, d2.information_type) == set()