def testNormalize(self):
value = '123content-location*23'
self.assertTrue(Clue.normalize(value) == 'content_location_23')
value = 'content/location'
self.assertTrue(Clue.normalize(value) == 'content_location')
value = '*content/location123'
self.assertTrue(Clue.normalize(value) == '_content_location123')
def test_normalize(self):
value = '123content-location*23'
self.failUnless(Clue.normalize(value) == 'content_location_23')
value = 'content/location'
self.failUnless(Clue.normalize(value) == 'content_location')
value = '*content/location123'
self.failUnless(Clue.normalize(value) == '_content_location123')
class TestStorage(unittest.TestCase):
def setUp(self):
self.clue = Clue()
self.clue.setTimestamp(100)
self.clue.headers = eval(r"""[
('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
('Content-Location', ' index.html.en'),
('Vary', ' negotiate,accept-language,accept-charset'),
('TCN', ' choice'),
('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
('Accept-Ranges', ' bytes'),
('Content-Length', ' 1456'),
('Keep-Alive', ' timeout=15, max=100'),
('Connection', ' Keep-Alive'),
('Content-Type', ' text/html; charset=ISO-8859-1'),
('Content-Language', ' en')
]""")
self.clue.parse(self.clue.headers)
self.filename = os.path.join('tests', 'data', 'test.clues')
def tearDown(self):
pass
def testSimpleSaveAndLoad(self):
try:
Halberd.clues.file.save(self.filename, [self.clue])
clues = Halberd.clues.file.load(self.filename)
finally:
os.unlink(self.filename)
self.assertTrue(len(clues) == 1)
self.assertTrue(clues[0] == self.clue)
def testNormalize(self):
value = "123content-location*23"
self.failUnless(Clue.normalize(value) == "content_location_23")
value = "content/location"
self.failUnless(Clue.normalize(value) == "content_location")
value = "*content/location123"
self.failUnless(Clue.normalize(value) == "_content_location123")
def ignore_changing_fields(clues):
"""Tries to detect and ignore MIME fields with ever changing content.
Some servers might include fields varying with time, randomly, etc. Those
fields are likely to alter the clue's digest and interfer with L{analyze},
producing many false positives and making the scan useless. This function
detects those fields and recalculates each clue's digest so they can be
safely analyzed again.
@param clues: Sequence of clues.
@type clues: C{list} or C{tuple}
"""
from Halberd.clues.Clue import Clue
different = diff_fields(clues)
# First alter Clue to be able to cope with the varying fields.
ignored = []
for field in different:
method = "_get_" + Clue.normalize(field)
if not hasattr(Clue, method):
logger.debug("ignoring %s", field)
ignored.append(method)
setattr(Clue, method, lambda s, f: None)
for clue in clues:
Clue.parse(clue, clue.headers)
for method in ignored:
# We want to leave the Clue class as before because a MIME field
# causing trouble for the current scan might be the source of precious
# information for another scan.
delattr(Clue, method)
return clues
def testNormalize(self):
value = '123content-location*23'
self.failUnless(Clue.normalize(value) == 'content_location_23')
value = 'content/location'
self.failUnless(Clue.normalize(value) == 'content_location')
value = '*content/location123'
self.failUnless(Clue.normalize(value) == '_content_location123')
def ignore_changing_fields(clues):
"""Tries to detect and ignore MIME fields with ever changing content.
Some servers might include fields varying with time, randomly, etc. Those
fields are likely to alter the clue's digest and interfer with L{analyze},
producing many false positives and making the scan useless. This function
detects those fields and recalculates each clue's digest so they can be
safely analyzed again.
@param clues: Sequence of clues.
@type clues: C{list} or C{tuple}
"""
from Halberd.clues.Clue import Clue
different = diff_fields(clues)
# First alter Clue to be able to cope with the varying fields.
ignored = []
for field in different:
method = '_get_' + Clue.normalize(field)
if not hasattr(Clue, method):
logger.debug('ignoring %s', field)
ignored.append(method)
setattr(Clue, method, lambda s, f: None)
for clue in clues:
Clue.parse(clue, clue.headers)
for method in ignored:
# We want to leave the Clue class as before because a MIME field
# causing trouble for the current scan might be the source of precious
# information for another scan.
delattr(Clue, method)
return clues
class TestClue(unittest.TestCase):
def setUp(self):
self.clue = Clue()
def tearDown(self):
pass
def test_count(self):
self.failUnlessEqual(self.clue.get_count(), 1)
self.clue.inc_count()
self.failUnlessEqual(self.clue.get_count(), 2)
self.clue.inc_count(21)
self.failUnlessEqual(self.clue.get_count(), 23)
self.failUnlessRaises(ValueError, self.clue.inc_count, 0)
self.failUnlessRaises(ValueError, self.clue.inc_count, -7)
def test_normalize(self):
value = '123content-location*23'
self.failUnless(Clue.normalize(value) == 'content_location_23')
value = 'content/location'
self.failUnless(Clue.normalize(value) == 'content_location')
value = '*content/location123'
self.failUnless(Clue.normalize(value) == '_content_location123')
def test_recompute(self):
# Check for invalid digest computations.
self.clue.parse('Test: abc\r\nSomething: blah\r\n\r\n')
self.assertRaises(AssertionError, self.clue._updateDigest, )
class TestClue(unittest.TestCase):
def setUp(self):
self.clue = Clue()
def tearDown(self):
pass
def testCount(self):
self.failUnlessEqual(self.clue.getCount(), 1)
self.clue.incCount()
self.failUnlessEqual(self.clue.getCount(), 2)
self.clue.incCount(21)
self.failUnlessEqual(self.clue.getCount(), 23)
self.failUnlessRaises(ValueError, self.clue.incCount, 0)
self.failUnlessRaises(ValueError, self.clue.incCount, -7)
def testNormalize(self):
value = "123content-location*23"
self.failUnless(Clue.normalize(value) == "content_location_23")
value = "content/location"
self.failUnless(Clue.normalize(value) == "content_location")
value = "*content/location123"
self.failUnless(Clue.normalize(value) == "_content_location123")
def testRecompute(self):
# Check for invalid digest computations.
self.clue.parse("Test: abc\r\nSomething: blah\r\n\r\n")
self.assertRaises(AssertionError, self.clue._updateDigest)
def setUp(self):
self.clue = Clue()
self.clue.setTimestamp(100)
self.clue.headers = eval(r"""[
('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
('Content-Location', ' index.html.en'),
('Vary', ' negotiate,accept-language,accept-charset'),
('TCN', ' choice'),
('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
('Accept-Ranges', ' bytes'),
('Content-Length', ' 1456'),
('Keep-Alive', ' timeout=15, max=100'),
('Connection', ' Keep-Alive'),
('Content-Type', ' text/html; charset=ISO-8859-1'),
('Content-Language', ' en')
]""")
self.clue.parse(self.clue.headers)
self.filename = os.path.join('tests', 'data', 'test.clues')
class TestStorage(unittest.TestCase):
def setUp(self):
self.clue = Clue()
self.clue.setTimestamp(100)
self.clue.headers = eval(r"""[
('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
('Content-Location', ' index.html.en'),
('Vary', ' negotiate,accept-language,accept-charset'),
('TCN', ' choice'),
('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
('Accept-Ranges', ' bytes'),
('Content-Length', ' 1456'),
('Keep-Alive', ' timeout=15, max=100'),
('Connection', ' Keep-Alive'),
('Content-Type', ' text/html; charset=ISO-8859-1'),
('Content-Language', ' en')
]""")
self.clue.parse(self.clue.headers)
self.filename = os.path.join('tests', 'data', 'test.clues')
def tearDown(self):
pass
def testSimpleSaveAndLoad(self):
try:
Halberd.clues.file.save(self.filename, [self.clue])
clues = Halberd.clues.file.load(self.filename)
finally:
os.unlink(self.filename)
self.failUnless(len(clues) == 1)
self.failUnless(clues[0] == self.clue)
def setUp(self):
self.clue = Clue()
self.clue.setTimestamp(100)
self.clue.headers = eval(r"""[
('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
('Content-Location', ' index.html.en'),
('Vary', ' negotiate,accept-language,accept-charset'),
('TCN', ' choice'),
('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
('Accept-Ranges', ' bytes'),
('Content-Length', ' 1456'),
('Keep-Alive', ' timeout=15, max=100'),
('Connection', ' Keep-Alive'),
('Content-Type', ' text/html; charset=ISO-8859-1'),
('Content-Language', ' en')
]""")
self.clue.parse(self.clue.headers)
self.filename = os.path.join('tests', 'data', 'test.clues')
def load(filename):
"""Load clues from file.
:param filename: Name of the files where the clues are stored.
@type filename: C{str}
:return: Clues extracted from the file.
@rtype: C{list}
@raise InvalidFile: In case there's a problem while reinterpreting the
clues.
"""
cluefp = open(filename, 'r')
reader = csv.reader(cluefp)
clues = []
for tup in reader:
try:
count, localtime, headers = tup
except ValueError:
raise InvalidFile('Cannot unpack fields')
# Recreate the current clue.
clue = Clue()
try:
clue._count = int(count)
clue._local = float(localtime)
except ValueError:
raise InvalidFile('Could not convert fields')
# This may be risky from a security standpoint.
clue.headers = eval(headers, {}, {})
if not (isinstance(clue.headers, types.ListType) or
isinstance(clue.headers, types.TupleType)):
raise InvalidFile('Wrong clue header field')
clue.parse(clue.headers)
clues.append(clue)
cluefp.close()
return clues
def setUp(self):
self.clue = Clue()
def setUp(self):
self.clue = Clue()