def test_purge_reports(self):
now = time.time()
c = yield Cracker(ip_address="192.168.1.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
c2 = yield Cracker(ip_address="192.168.1.2", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
c3 = yield Cracker(ip_address="192.168.1.3", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now)
yield controllers.purge_legacy_addresses()
crackers = yield Cracker.all()
self.assertEqual(len(crackers), 3, "Should still have three crackers after purging legacy")
reports = yield Report.all()
self.assertEqual(len(reports), 4, "Should still have four reports after purging legacy")
yield controllers.purge_reported_addresses()
crackers = yield Cracker.all()
self.assertEqual(len(crackers), 0, "Should have no crackers after purging reports")
reports = yield Report.all()
self.assertEqual(len(reports), 0, "Should have no reports after purging reports")
def test_add_cracker(self):
cracker_ip = "127.0.0.1"
now = time.time()
cracker = Cracker(ip_address=cracker_ip,
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0,
resiliency=0)
cracker = yield cracker.save()
c2 = yield controllers.get_cracker(cracker_ip)
yield self.assertIsNotNone(c2)
self.assertEqual(c2.ip_address, cracker.ip_address,
"Save and re-fetch cracker from database")
self.assertEqual(c2.id, cracker.id,
"Save and re-fetch cracker from database")
self.assertEqual(c2.first_time, cracker.first_time,
"Save and re-fetch cracker from database")
self.assertEqual(c2.latest_time, cracker.latest_time,
"Save and re-fetch cracker from database")
self.assertEqual(c2.total_reports, cracker.total_reports,
"Save and re-fetch cracker from database")
self.assertEqual(c2.current_reports, cracker.current_reports,
"Save and re-fetch cracker from database")
self.assertEqual(c2.id, cracker.id,
"Save and re-fetch cracker from database")
def prepare_stats(self):
self.stats_settings()
now = time.time()
c1 = yield Cracker(ip_address="192.168.1.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c2 = yield Cracker(ip_address="192.168.1.2",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
yield controllers.add_report_to_cracker(c1,
"127.0.0.1",
when=now - 25 * 3600)
yield controllers.add_report_to_cracker(c1, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c1, "127.0.0.3", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now + 1)
yield Registry.DBPOOL.runInteraction(stats.fixup_history_txn)
yesterday = datetime.date.today() - datetime.timedelta(days=1)
yield Registry.DBPOOL.runInteraction(stats.update_country_history_txn,
yesterday,
include_history=True)
yield stats.update_stats_cache()
def test_fixup(self):
now = time.time()
c1 = yield Cracker(ip_address="194.109.6.92",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c2 = yield Cracker(ip_address="192.30.252.128",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
hosts = [c1, c2]
config.stats_resolve_hostnames = True
stats.fixup_crackers(hosts)
self.assertEqual(c1.hostname, "www.xs4all.nl",
"Reverse DNS of www.xs4all.nl")
self.assertEqual(c1.country, "Netherlands",
"Testing geoip of www.xs4all.nl")
self.assertEqual(c2.hostname, "github.com",
"Reverse DNS of github.com")
self.assertEqual(c2.country, "United States",
"Testing geoip of github.com")
def test_purge_ip(self):
now = time.time()
c = yield Cracker(ip_address="192.168.211.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c2 = yield Cracker(ip_address="192.168.211.2",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c3 = yield Cracker(ip_address="192.168.211.3",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now)
legacy = yield Legacy(ip_address="192.168.211.1",
retrieved_time=now).save()
legacy = yield Legacy(ip_address="192.168.211.2",
retrieved_time=now).save()
yield controllers.purge_ip("192.168.211.1")
crackers = yield Cracker.find(orderby='ip_address ASC')
self.assertEqual(len(crackers), 2,
"Should still have two crackers after purging one")
self.assertEquals(crackers[0].ip_address, "192.168.211.2",
"Should remove the right cracker")
self.assertEquals(crackers[1].ip_address, "192.168.211.3",
"Should remove the right cracker")
reports = yield Report.all()
self.assertEqual(
len(reports), 1,
"Should still have one report left after purging cracker with three reports"
)
self.assertEquals(reports[0].ip_address, "127.0.0.3",
"Should remove the right report")
legacy = yield Legacy.all()
self.assertEqual(
len(legacy), 1,
"Should still have one legacy reports after purging one")
self.assertEquals(legacy[0].ip_address, "192.168.211.2",
"Should remove the right legacy host")
def test_add_cracker(self):
cracker_ip = "127.0.0.1"
now = time.time()
cracker = Cracker(ip_address=cracker_ip, first_time=now, latest_time=now, total_reports=0, current_reports=0, resiliency=0)
cracker = yield cracker.save()
c2 = yield controllers.get_cracker(cracker_ip)
yield self.assertIsNotNone(c2)
self.assertEqual(c2.ip_address, cracker.ip_address, "Save and re-fetch cracker from database")
self.assertEqual(c2.id, cracker.id, "Save and re-fetch cracker from database")
self.assertEqual(c2.first_time, cracker.first_time, "Save and re-fetch cracker from database")
self.assertEqual(c2.latest_time, cracker.latest_time, "Save and re-fetch cracker from database")
self.assertEqual(c2.total_reports, cracker.total_reports, "Save and re-fetch cracker from database")
self.assertEqual(c2.current_reports, cracker.current_reports, "Save and re-fetch cracker from database")
self.assertEqual(c2.id, cracker.id, "Save and re-fetch cracker from database")
def test_purge_ip(self):
now = time.time()
c = yield Cracker(ip_address="192.168.211.1", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
c2 = yield Cracker(ip_address="192.168.211.2", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
c3 = yield Cracker(ip_address="192.168.211.3", first_time=now, latest_time=now, total_reports=0, current_reports=0).save()
yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now)
legacy = yield Legacy(ip_address="192.168.211.1", retrieved_time=now).save()
legacy = yield Legacy(ip_address="192.168.211.2", retrieved_time=now).save()
yield controllers.purge_ip("192.168.211.1")
crackers = yield Cracker.find(orderby='ip_address ASC')
self.assertEqual(len(crackers), 2, "Should still have two crackers after purging one")
self.assertEquals(crackers[0].ip_address, "192.168.211.2", "Should remove the right cracker")
self.assertEquals(crackers[1].ip_address, "192.168.211.3", "Should remove the right cracker")
reports = yield Report.all()
self.assertEqual(len(reports), 1, "Should still have one report left after purging cracker with three reports")
self.assertEquals(reports[0].ip_address, "127.0.0.3", "Should remove the right report")
legacy = yield Legacy.all()
self.assertEqual(len(legacy), 1, "Should still have one legacy reports after purging one")
self.assertEquals(legacy[0].ip_address, "192.168.211.2", "Should remove the right legacy host")
def test_purge_reports(self):
now = time.time()
c = yield Cracker(ip_address="192.168.1.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c2 = yield Cracker(ip_address="192.168.1.2",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c3 = yield Cracker(ip_address="192.168.1.3",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
yield controllers.add_report_to_cracker(c, "127.0.0.1", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.2", when=now)
yield controllers.add_report_to_cracker(c, "127.0.0.9", when=now)
yield controllers.add_report_to_cracker(c2, "127.0.0.3", when=now)
yield controllers.purge_legacy_addresses()
crackers = yield Cracker.all()
self.assertEqual(
len(crackers), 3,
"Should still have three crackers after purging legacy")
reports = yield Report.all()
self.assertEqual(
len(reports), 4,
"Should still have four reports after purging legacy")
yield controllers.purge_reported_addresses()
crackers = yield Cracker.all()
self.assertEqual(len(crackers), 0,
"Should have no crackers after purging reports")
reports = yield Report.all()
self.assertEqual(len(reports), 0,
"Should have no reports after purging reports")
def test_add_report(self):
now = time.time()
yield Cracker(ip_address="192.168.1.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c = yield controllers.get_cracker("192.168.1.1")
yield self.assertIsNotNone(c)
yield controllers.add_report_to_cracker(c, "127.0.0.1")
r = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"],
limit=1)
returnValue(self.assertIsNotNone(r, "Added report is in database"))
def test_add_multiple_reports(self):
now = time.time()
yield Cracker(ip_address="192.168.1.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
c = yield controllers.get_cracker("192.168.1.1")
yield self.assertIsNotNone(c)
yield controllers.add_report_to_cracker(c, "127.0.0.1", now)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 1, "First added report is in database")
# Add second report shortly after first
yield controllers.add_report_to_cracker(c, "127.0.0.1", now + 1)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 1,
"Second added report should be ignored")
# Add second report after 24 hours
yield controllers.add_report_to_cracker(c, "127.0.0.1",
now + 24 * 3600 + 1)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 2,
"Second report should be added after 24 hours ")
# Add third report shortly after second, should be ignored
yield controllers.add_report_to_cracker(c, "127.0.0.1",
now + 24 * 3600 + 10)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(
len(reports), 2,
"Third report shortly after second should be ignored")
# Add third report after again 24 hours
yield controllers.add_report_to_cracker(c, "127.0.0.1",
now + 2 * 24 * 3600 + 20)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 3,
"Third report after again 24 hours, should be added")
# Add fourth report
time_added = now + 2 * 24 * 3600 + 30
yield controllers.add_report_to_cracker(c, "127.0.0.1", time_added)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"],
orderby='latest_report_time asc')
self.assertEqual(len(reports), 3, "Fourth report, should be merged")
self.assertEqual(reports[-1].latest_report_time, time_added,
"Latest report time should be updated")
self.assertEquals(c.current_reports, 1,
"Only one unique reporter should be counted")
self.assertEquals(c.total_reports, 6,
"Cracker reported six timed in total")
# Perform maintenance, expire original report
yield controllers.perform_maintenance(limit=now + 1)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 2,
"Maintenance should remove oldest report")
yield c.refresh()
self.assertEqual(c.current_reports, 1,
"Maintenance should still leave one unique reporter")
# Perform maintenance, expire second report
yield controllers.perform_maintenance(limit=now + 24 * 3600 + 11)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 1,
"Maintenance should remove one more report")
yield c.refresh()
self.assertEqual(c.current_reports, 1,
"Maintenance should still leave one unique reporter")
# Perform maintenance again, expire last report and cracker
yield controllers.perform_maintenance(limit=now + 2 * 24 * 3600 + 31)
reports = yield Report.find(
where=["cracker_id=? and ip_address=?", c.id, "127.0.0.1"])
self.assertEqual(len(reports), 0,
"Maintenance should remove last report")
cracker = yield controllers.get_cracker("192.168.1.1")
self.assertIsNone(cracker, "Maintenance should remove cracker")
def test_get_qualifying_crackers(self):
now = time.time()
c = yield Cracker(ip_address="192.168.1.1",
first_time=now,
latest_time=now,
total_reports=0,
current_reports=0).save()
# First test: cracker without reports should not be reported
hosts = yield controllers.get_qualifying_crackers(1, 0, now, 50, [])
self.assertEqual(len(hosts), 0,
"Cracker without reports should not be returned")
client_ip = "1.1.1.1"
yield controllers.add_report_to_cracker(c, client_ip, when=now)
hosts = yield controllers.get_qualifying_crackers(
1, -1, now - 1, 50, [])
self.assertEqual(
len(hosts), 1,
"When one report, get_new_hosts with resilience <0 should return one host"
)
hosts = yield controllers.get_qualifying_crackers(
2, -1, now - 1, 50, [])
self.assertEqual(
len(hosts), 0,
"When one report, get_new_hosts with resilience 0 and threshold 2 should return empty list"
)
client_ip = "1.1.1.2"
yield controllers.add_report_to_cracker(c, client_ip, when=now + 3600)
hosts = yield controllers.get_qualifying_crackers(
2, 3500, now - 1, 50, [])
self.assertEqual(len(hosts), 1,
"Two reports should result in a result")
hosts = yield controllers.get_qualifying_crackers(
2, 3500, now - 1, 50, [c.ip_address])
self.assertEqual(len(hosts), 0,
"Two reports, remove reported host from list")
hosts = yield controllers.get_qualifying_crackers(
3, 3500, now - 1, 50, [])
self.assertEqual(len(hosts), 0, "Two reports, asked for three")
hosts = yield controllers.get_qualifying_crackers(
2, 4000, now - 1, 50, [])
self.assertEqual(len(hosts), 0, "Two reports, not enough resiliency")
logging.debug("Testing d2")
client_ip = "1.1.1.3"
yield controllers.add_report_to_cracker(c, client_ip, when=now + 7200)
hosts = yield controllers.get_qualifying_crackers(
2, 3500, now + 3601, 50, [])
self.assertEqual(len(hosts), 0, "Condition (d2)")
logging.debug("Testing d1")
client_ip = "1.1.1.3"
yield controllers.add_report_to_cracker(c,
client_ip,
when=now + 7200 + 24 * 3600 +
1)
hosts = yield controllers.get_qualifying_crackers(
2, 24 * 3600 + 1, now + 3601, 50, [])
self.assertEqual(len(hosts), 1, "Condition (d1)")