def testScanBenign(self):
"Tests the dummy scanner's scan method."
malicious, si = dummy.scan(self.benignfile)
self.assertFalse(malicious)
expected = (self.scannerfile, self.version, trim_microseconds(self.curtime))
actual = (si.scannervv, si.sigversion, trim_microseconds(si.sigdate))
self.assertEquals(expected, actual)
def testScanMalicious(self):
"Tests the dummy scanner's scan method with an evil file."
malicious, si = dummy.scan(self.maliciousfile)
self.assertTrue(malicious)
expected = (self.scannerfile, self.version, trim_microseconds(self.curtime))
actual = (si.scannervv, si.sigversion, trim_microseconds(si.sigdate))
self.assertEquals(expected, actual)
def testSaveLoadHits(self):
'If there were hits on this log, preserve them.'
vases = ('http://allyourvases.rblong.to.us', 1234, '', Safety.possibly_benign)
data = [vases]
now = trim_microseconds(datetime.utcnow())
capacity = 45
scannervv = 'Unit Test Sig Info'
sigversion = '0.12345'
si = SigInfo(scannervv, sigversion, now)
# Create ScanLog
sl = scanlog.ScanLog(capacity, si, self.slpath)
# Add data to ScanLog
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sl.add(uo, safety))
sl.hits = 35
sl.save()
del sl
# Load
sl = scanlog.ScanLog.load(self.slpath)
# Get items.
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sl.get(uo)
self.assertEqual(safety.ismalicious, value.ismalicious)
self.assertEqual(int(sl.hits), 35)
def testInitAddSaveLoadGetNoIncludeContent(self):
"Test that two different urls with the same content are found with only one added."
name = "Testbox"
now = trim_microseconds(datetime.utcnow())
si = SigInfo("Test Scanner Version 0.14", "signature version 9.12.10", now)
sd = scandigest.ScanDigest(23, si, self.sdpath, nonce=12345)
urlbase = "http://www.froogly.com/iownyour%sbase.aspx"
bases = ["first", "second", "third", "home"]
urlobjects = []
for base in bases:
uo = UrlObject(urlbase % base, 3425, nonce=name, hash="e0dbc25fdb98a7e3")
urlobjects.append(uo)
sd.add(uo)
sd.save()
# Load from file now.
sd = scandigest.ScanDigest.load(self.sdpath)
self.assertTrue(sd.saved)
self.assertEqual(sd.maxcapacity, 23)
self.assertEqual(sd.siginfo, si)
# Change the urls, but keep the hash. The urls should still be
# found because their hashes were stored.
altered_urlobjects = []
for base in bases:
url = urlbase % base
uo = UrlObject(url + "altered", 3425, nonce=name, hash="e0dbc25fdb98a7e3")
altered_urlobjects.append(uo)
for uo in altered_urlobjects:
self.assertTrue(sd.get(uo))
def testLengthCorrespondsToUrlsAdded(self):
'The length of a ScanDigest should equal the number of urls added.'
bases = ('http://allyourbases.rblong.to.us', 2345, '', Safety.possibly_malicious)
vases = ('http://allyourvases.rblong.to.us', 1234, '', Safety.possibly_benign)
cases = ('http://allyourcases.rblong.to.us', 6358, '', Safety.possibly_malicious)
faces = ('http://allyourfaces.rblong.to.us', 39217, '', Safety.possibly_benign)
data = [('http://www.google.com', 1546, 'e60f0c7b96e7ca2f0948ab1c31d', Safety.benign),
('http://westealyourpasswd.com', 9823, 'b849b8e3a659f8d4cac675a', Safety.malicious),
('http://www.mit.edu', 12309, '6aedc8dcf2ca3efe938d9268c40ae56', Safety.benign),
('', 22434, '8eaf73dac9d0b083cadefa34', Safety.benign),
('', 66575, '8caf73dac9d0e183cadefa32', Safety.malicious),
('http://youhavebeenpowned.cz', 0, '', Safety.malicious),
('http://reallysafe.com', 136, '', Safety.benign),
bases,
vases,
cases,
faces]
now = trim_microseconds(datetime.utcnow())
self.db['maxcapacity'] = '%s' % (23)
self.db['scannervv'] = '%s' % 'Test Scanner Version 0.14'
self.db['sigversion'] = '%s' % 'signature version 9.12.10'
self.db['sigtimestamp'] = '%s' % now
self.db['hits'] = '%s' % (25)
self.db.close()
# Now open as ScanLog
sl = scanlog.ScanLog.load(self.slpath)
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sl.add(uo, safety))
self.assertEqual(len(sl), len(data))
def setUpClass(cls):
'Initialize sqlite in-memory for use by tests.'
cls.engine = create_engine('sqlite:///:memory:')
Session = sessionmaker(bind=cls.engine)
cls.session = Session()
Base.metadata.create_all(cls.engine)
now = trim_microseconds(datetime.utcnow())
cls.siginfo = SigInfo('Test Model Scanner v1.0', 'Generic Signature 0.3', now)
def testLoadFreshDb(self):
'Starting off with a fresh db.'
si = SigInfo('Unknown Scanner Version', 'Unknown Signature Version',
trim_microseconds(datetime.utcnow()))
sl = scanlog.ScanLog.load(self.slpath + '_newdb')
self.assertTrue(sl.saved)
self.assertEqual(sl.maxcapacity, 300)
self.assertEqual(sl.siginfo, si)
def testVerifySaveStaleFile(self):
"A scandigest that is older than expected should raise a DigestModifiedTimeError."
fd, tmp = tempfile.mkstemp(dir=path.dirname(self.sdpath))
os.write(fd, "The brown lazy fox held on for dear life while the cow jumped over the moon.")
os.close(fd)
now = trim_microseconds(datetime.utcnow())
si = SigInfo("Test Scanner Version 0.14", "signature version 9.12.10", now)
sd = scandigest.ScanDigest(23, si, self.sdpath, nonce=12345)
sd.filename = tmp
self.assertRaises(DigestModifiedTimeError, sd.verify_save, (time.time() + 1000))
self.removeTempFile(tmp)
def testVerifySaveZeroBytes(self):
"A zero-sized scandigest should raise ZeroSizedDigestError."
fd, tmp = tempfile.mkstemp(dir=path.dirname(self.sdpath))
os.write(fd, "")
os.close(fd)
now = trim_microseconds(datetime.utcnow())
si = SigInfo("Test Scanner Version 0.14", "signature version 9.12.10", now)
sd = scandigest.ScanDigest(23, si, self.sdpath, nonce=12345)
sd.filename = tmp
self.assertRaises(ZeroSizedDigestError, sd.verify_save)
self.removeTempFile(tmp)
def testVerifySaveRecentFile(self):
before_write = time.time()
fd, tmp = tempfile.mkstemp(dir=path.dirname(self.sdpath))
os.write(fd, "The brown lazy fox held on for dear life while the cow jumped over the moon.")
os.close(fd)
now = trim_microseconds(datetime.utcnow())
si = SigInfo("Test Scanner Version 0.14", "signature version 9.12.10", now)
sd = scandigest.ScanDigest(23, si, self.sdpath, nonce=12345)
sd.filename = tmp
# No assertion, as verify_save should run without raising an error in this case.
sd.verify_save(age=before_write)
self.removeTempFile(tmp)
def testMicrosecondsDoNotRainOnTheParade(self):
'If the sigtimestamp has microseconds, ignore them.'
now = datetime.utcnow()
now_sans_micro = trim_microseconds(now)
self.db['maxcapacity'] = '%s' % (23)
self.db['scannervv'] = '%s' % 'Test Scanner Version 0.14'
self.db['sigversion'] = '%s' % 'signature version 9.12.10'
self.db['sigtimestamp'] = '%s' % now
self.db.close()
# Now open as ScanLog
sl = scanlog.ScanLog.load(self.slpath)
self.assertEqual(sl.siginfo.sigdate, now_sans_micro)
def testFillBeyondCapacity(self):
'Create a new ScanLog object, save items to it, save it, load it, stuff it full.'
bases = ('http://allyourbases.rblong.to.us', 2345, '', Safety.possibly_malicious)
vases = ('http://allyourvases.rblong.to.us', 1234, '', Safety.possibly_benign)
cases = ('http://allyourcases.rblong.to.us', 6358, '', Safety.possibly_malicious)
faces = ('http://allyourfaces.rblong.to.us', 39217, '', Safety.possibly_benign)
empty0 = ('', 0, '', Safety.benign)
empty1 = ('', 235, '', Safety.malicious)
empty2 = ('', 3342, '', Safety.possibly_malicious)
empty3 = ('', 0, '', Safety.possibly_benign)
data = [('http://www.google.com', 1546, 'e60f0c7b96e7ca2f0948ab1c31d', Safety.benign),
('http://westealyourpasswd.com', 9823, 'b849b8e3a659f8d4cac675a', Safety.malicious),
('http://www.mit.edu', 12309, '6aedc8dcf2ca3efe938d9268c40ae56', Safety.benign),
('', 22434, '8eaf73dac9d0b083cadefa34', Safety.benign),
('', 66575, '8caf73dac9d0e183cadefa32', Safety.malicious),
('http://youhavebeenpowned.cz', 0, '', Safety.malicious),
('http://reallysafe.com', 136, '', Safety.benign),
bases,
vases,
cases]
empty = [empty0, empty1, empty2, empty3]
now = trim_microseconds(datetime.utcnow())
capacity = 10 # Capacity is based on number of urls added.
scannervv = 'Unit Test Sig Info'
sigversion = '0.12345'
si = SigInfo(scannervv, sigversion, now)
# Create ScanLog
sl = scanlog.ScanLog(capacity, si, self.slpath)
# Add data to ScanLog
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sl.add(uo, safety))
for url, size, hash, safety in empty:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertFalse(sl.add(uo, safety))
sl.save()
del sl
# Load
sl = scanlog.ScanLog.load(self.slpath)
# Get items.
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sl.get(uo)
self.assertEqual(safety.ismalicious, value.ismalicious)
for url, size, hash, safety in empty:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sl.get(uo)
self.assertIsNone(value)
with self.assertRaises(ContainerFullError):
face_uo = UrlObject(faces[0], faces[1], hash=faces[2], nonce=self.name)
sl.add(face_uo, faces[3])
def testInitAddSaveLoadGet(self):
'Create a new ScanLog object, save items to it, save it, load it.'
# We need data in an object to test get, so we'll test add. In order to test
# add we need to load an object or create a new one. In order to load one, we
# need to save one, so we might as well test all of them.
# Initialize data.
bases = ('http://allyourbases.rblong.to.us', 2345, '', Safety.possibly_malicious)
vases = ('http://allyourvases.rblong.to.us', 1234, '', Safety.possibly_benign)
empty0 = ('', 0, '', Safety.benign)
empty1 = ('', 235, '', Safety.malicious)
empty2 = ('', 3342, '', Safety.possibly_malicious)
empty3 = ('', 0, '', Safety.possibly_benign)
data = [('http://www.google.com', 1546, 'e60f0c7b96e7ca2f0948ab1c31d', Safety.benign),
('http://westealyourpasswd.com', 9823, 'b849b8e3a659f8d4cac675a', Safety.malicious),
('http://www.mit.edu', 12309, '6aedc8dcf2ca3efe938d9268c40ae56', Safety.benign),
('', 22434, '8eaf73dac9d0b083cadefa34', Safety.benign),
('', 66575, '8caf73dac9d0e183cadefa32', Safety.malicious),
('http://youhavebeenpowned.cz', 0, '', Safety.malicious),
('http://reallysafe.com', 136, '', Safety.benign),
bases,
vases]
empty = [empty0, empty1, empty2, empty3]
now = trim_microseconds(datetime.utcnow())
capacity = 45
scannervv = 'Unit Test Sig Info'
sigversion = '0.12345'
si = SigInfo(scannervv, sigversion, now)
# Create ScanLog
sl = scanlog.ScanLog(capacity, si, self.slpath)
# Add data to ScanLog
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sl.add(uo, safety))
for url, size, hash, safety in empty:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertFalse(sl.add(uo, safety))
sl.save()
del sl
# Load
sl = scanlog.ScanLog.load(self.slpath)
# Get items.
for url, size, hash, safety in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sl.get(uo)
self.assertEqual(safety.ismalicious, value.ismalicious)
for url, size, hash, safety in empty:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sl.get(uo)
self.assertIsNone(value)
def testLoadExistingDb(self):
'Create a db, save using normal db interface, then load from it using scanlog.'
# setUp creates. Now we populate the db.
now = trim_microseconds(datetime.utcnow())
self.db['maxcapacity'] = '%s' % (23)
self.db['scannervv'] = '%s' % 'Test Scanner Version 0.14'
self.db['sigversion'] = '%s' % 'signature version 9.12.10'
self.db['sigtimestamp'] = '%s' % now
si = SigInfo('Test Scanner Version 0.14', 'signature version 9.12.10', now)
# Save the data.
self.db.close()
# Load a ScanLog from the db.
sl = scanlog.ScanLog.load(self.slpath)
self.assertTrue(sl.saved)
self.assertEqual(sl.maxcapacity, 23)
self.assertEqual(sl.siginfo, si)
def testSaveDigest(self):
"Create a digest, save, then load from it using ScanDigest.load."
name = "Testbox"
now = trim_microseconds(datetime.utcnow())
si = SigInfo("Test Scanner Version 0.14", "signature version 9.12.10", now)
sd = scandigest.ScanDigest(23, si, self.sdpath, nonce=12345)
urlbase = "http://www.froogly.com/iownyour%sbase.aspx"
bases = ["first", "second", "third", "home"]
urlobjects = []
for base in bases:
uo = UrlObject(urlbase % base, 3425, nonce=name, hash="e0dbc25fdb98a7e3")
urlobjects.append(uo)
sd.add(uo)
sd.save()
# Load from file.
sd = scandigest.ScanDigest.load(self.sdpath)
self.assertTrue(sd.saved)
self.assertEqual(sd.maxcapacity, 23)
self.assertEqual(sd.siginfo, si)
for uo in urlobjects:
self.assertTrue(sd.get(uo))
def testCreateLoadToCapacityLoadThenOverflow(self):
"Create a ScanDigest object, stuff it to overflowing."
bases = ("http://allyourbases.rblong.to.us", 2345, "")
vases = ("http://allyourvases.rblong.to.us", 1234, "")
cases = ("http://allyourcases.rblong.to.us", 4262, "")
faces = ("http://allyourfaces.rblong.to.us", 9013, "")
data = [
("http://www.google.com", 1546, "e60f0c7b96e7ca2f0948ab1c31d"),
("http://westealyourpasswd.com", 9823, "b849b8e3a659f8d4cac675a"),
("http://www.mit.edu", 12309, "6aedc8dcf2ca3efe938d9268c40ae56"),
("", 22434, "8eaf73dac9d0b083cadefa34"),
("", 66575, "8caf73dac9d0e183cadefa32"),
("http://youhavebeenpowned.cz", 0, ""),
("http://reallysafe.com", 136, ""),
bases,
vases,
cases,
]
now = trim_microseconds(datetime.utcnow())
capacity = 10 # Capacity is based on number of urls added.
scannervv = "Unit Test Sig Info"
sigversion = "0.12345"
si = SigInfo(scannervv, sigversion, now)
# Create ScanDigest
sd = scandigest.ScanDigest(capacity, si, self.sdpath)
# Add data to ScanDigest
for url, size, hash in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sd.add(uo))
sd.save()
del sd
# Load and get items.
sd = scandigest.ScanDigest.load(self.sdpath)
for url, size, hash in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
value = sd.get(uo)
self.assertTrue(value)
# Try to add and save faces.
face_uo = UrlObject(faces[0], faces[1], nonce=self.name, hash=faces[2])
self.assertRaises(ContainerFullError, sd.add, face_uo)
def testSizeMatchesUrlsAdded(self):
"Add urls to a ScanDigest, some with contents hashes, size is number of urls added."
bases = ("http://allyourbases.rblong.to.us", 2345, "")
vases = ("http://allyourvases.rblong.to.us", 1234, "")
cases = ("http://allyourcases.rblong.to.us", 4262, "")
faces = ("http://allyourfaces.rblong.to.us", 9013, "")
data = [
("http://www.google.com", 1546, "e60f0c7b96e7ca2f0948ab1c31d"),
("http://westealyourpasswd.com", 9823, "b849b8e3a659f8d4cac675a"),
("http://www.mit.edu", 12309, "6aedc8dcf2ca3efe938d9268c40ae56"),
("", 22434, "8eaf73dac9d0b083cadefa34"),
("", 66575, "8caf73dac9d0e183cadefa32"),
("http://youhavebeenpowned.cz", 0, ""),
("http://reallysafe.com", 136, ""),
bases,
vases,
cases,
]
now = trim_microseconds(datetime.utcnow())
capacity = 25
scannervv = "Unit Test Sig Info"
sigversion = "0.12345"
si = SigInfo(scannervv, sigversion, now)
# Create ScanDigest
sd = scandigest.ScanDigest(capacity, si, self.sdpath)
# Add data to ScanDigest
for url, size, hash in data:
uo = UrlObject(url, size, nonce=self.name, hash=hash)
self.assertTrue(sd.add(uo))
self.assertEqual(len(sd), len(data))
sd.save()
self.assertEqual(len(sd), len(data))
del sd
# Load and get items.
sd = scandigest.ScanDigest.load(self.sdpath)
self.assertEqual(len(sd), len(data))
def testGetSigInfo(self):
"Should return a SigInfo object"
si = dummy.getSigInfo()
actual = (si.scannervv, si.sigversion, trim_microseconds(si.sigdate))
expected = (self.scannerfile, self.version, trim_microseconds(self.curtime))
self.assertEqual(expected, actual)
