Exemplo n.º 1
0
def ignore_changing_fields(clues):
    """Tries to detect and ignore MIME fields with ever changing content.

    Some servers might include fields varying with time, randomly, etc. Those
    fields are likely to alter the clue's digest and interfer with L{analyze},
    producing many false positives and making the scan useless. This function
    detects those fields and recalculates each clue's digest so they can be
    safely analyzed again.

    @param clues: Sequence of clues.
    @type clues: C{list} or C{tuple}
    """
    from Halberd.clues.Clue import Clue

    different = diff_fields(clues)

    # First alter Clue to be able to cope with the varying fields.
    ignored = []
    for field in different:
        method = "_get_" + Clue.normalize(field)
        if not hasattr(Clue, method):
            logger.debug("ignoring %s", field)
            ignored.append(method)
            setattr(Clue, method, lambda s, f: None)

    for clue in clues:
        Clue.parse(clue, clue.headers)

    for method in ignored:
        # We want to leave the Clue class as before because a MIME field
        # causing trouble for the current scan might be the source of precious
        # information for another scan.
        delattr(Clue, method)

    return clues
Exemplo n.º 2
0
 def testNormalize(self):
     value = '123content-location*23'
     self.failUnless(Clue.normalize(value) == 'content_location_23')
     value = 'content/location'
     self.failUnless(Clue.normalize(value) == 'content_location')
     value = '*content/location123'
     self.failUnless(Clue.normalize(value) == '_content_location123')
Exemplo n.º 3
0
 def testNormalize(self):
     value = "123content-location*23"
     self.failUnless(Clue.normalize(value) == "content_location_23")
     value = "content/location"
     self.failUnless(Clue.normalize(value) == "content_location")
     value = "*content/location123"
     self.failUnless(Clue.normalize(value) == "_content_location123")
Exemplo n.º 4
0
class TestClue(unittest.TestCase):
    def setUp(self):
        self.clue = Clue()

    def tearDown(self):
        pass

    def testCount(self):
        self.failUnlessEqual(self.clue.getCount(), 1)
        self.clue.incCount()
        self.failUnlessEqual(self.clue.getCount(), 2)
        self.clue.incCount(21)
        self.failUnlessEqual(self.clue.getCount(), 23)

        self.failUnlessRaises(ValueError, self.clue.incCount, 0)
        self.failUnlessRaises(ValueError, self.clue.incCount, -7)

    def testNormalize(self):
        value = "123content-location*23"
        self.failUnless(Clue.normalize(value) == "content_location_23")
        value = "content/location"
        self.failUnless(Clue.normalize(value) == "content_location")
        value = "*content/location123"
        self.failUnless(Clue.normalize(value) == "_content_location123")

    def testRecompute(self):
        # Check for invalid digest computations.
        self.clue.parse("Test: abc\r\nSomething: blah\r\n\r\n")
        self.assertRaises(AssertionError, self.clue._updateDigest)
Exemplo n.º 5
0
class TestStorage(unittest.TestCase):

    def setUp(self):
        self.clue = Clue()
        self.clue.setTimestamp(100)
        self.clue.headers = eval(r"""[
            ('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
            ('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
            ('Content-Location', ' index.html.en'),
            ('Vary', ' negotiate,accept-language,accept-charset'),
            ('TCN', ' choice'),
            ('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
            ('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
            ('Accept-Ranges', ' bytes'),
            ('Content-Length', ' 1456'),
            ('Keep-Alive', ' timeout=15, max=100'),
            ('Connection', ' Keep-Alive'),
            ('Content-Type', ' text/html; charset=ISO-8859-1'),
            ('Content-Language', ' en')
        ]""")
        self.clue.parse(self.clue.headers)

        self.filename = os.path.join('tests', 'data', 'test.clues')

    def tearDown(self):
        pass


    def testSimpleSaveAndLoad(self):
        try:
            Halberd.clues.file.save(self.filename, [self.clue])
            clues = Halberd.clues.file.load(self.filename)
        finally:
            os.unlink(self.filename)

        self.failUnless(len(clues) == 1)
        self.failUnless(clues[0] == self.clue)
Exemplo n.º 6
0
    def setUp(self):
        self.clue = Clue()
        self.clue.setTimestamp(100)
        self.clue.headers = eval(r"""[
            ('Date', ' Tue, 24 Feb 2004 17:09:05 GMT'),
            ('Server', ' Apache/2.0.48 (Unix) DAV/2 SVN/0.35.1'),
            ('Content-Location', ' index.html.en'),
            ('Vary', ' negotiate,accept-language,accept-charset'),
            ('TCN', ' choice'),
            ('Last-Modified', ' Sat, 22 Nov 2003 15:56:12 GMT'),
            ('ETag', ' "252ff0-5b0-3b5aff00;253006-961-3b5aff00"'),
            ('Accept-Ranges', ' bytes'),
            ('Content-Length', ' 1456'),
            ('Keep-Alive', ' timeout=15, max=100'),
            ('Connection', ' Keep-Alive'),
            ('Content-Type', ' text/html; charset=ISO-8859-1'),
            ('Content-Language', ' en')
        ]""")
        self.clue.parse(self.clue.headers)

        self.filename = os.path.join('tests', 'data', 'test.clues')
Exemplo n.º 7
0
def load(filename):
    """Load clues from file.

    :param filename: Name of the files where the clues are stored.
    @type filename: C{str}

    :return: Clues extracted from the file.
    @rtype: C{list}

    @raise InvalidFile: In case there's a problem while reinterpreting the
    clues.
    """
    cluefp = open(filename, 'r')
    reader = csv.reader(cluefp)

    clues = []
    for tup in reader:
        try:
            count, localtime, headers = tup
        except ValueError:
            raise InvalidFile('Cannot unpack fields')

        # Recreate the current clue.
        clue = Clue()
        try:
            clue._count = int(count)
            clue._local = float(localtime)
        except ValueError:
            raise InvalidFile('Could not convert fields')

        # This may be risky from a security standpoint.
        clue.headers = eval(headers, {}, {})
        if not (isinstance(clue.headers, types.ListType) or
                isinstance(clue.headers, types.TupleType)):
            raise InvalidFile('Wrong clue header field')
        clue.parse(clue.headers)

        clues.append(clue)

    cluefp.close()
    return clues
Exemplo n.º 8
0
 def setUp(self):
     self.clue = Clue()