Exemple #1
0
 def test_get_domain(self):
     tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
     testdata=[
         ('bla.com','bla.com'),
         ('co.uk','co.uk'),
         ('bla.co.uk','bla.co.uk'),
         ('bla.blubb.co.uk','blubb.co.uk'),
         ('bla.org.co.uk','bla.org.co.uk'),
     ]
     for fqdn,expecteddomain in testdata:
         domain = tldmagic.get_domain(fqdn)
         self.assertEqual(domain,expecteddomain, "Expected Domain %s from %s, but got %s"%(expecteddomain,fqdn,domain))
Exemple #2
0
 def test_get_tld(self):
     tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
     testdata=[
         ('bla.com','com'),
         ('co.uk','co.uk'),
         ('bla.co.uk','co.uk'),
         ('bla.blubb.co.uk','co.uk'),
         ('bla.org.co.uk','org.co.uk'),
     ]
     for fqdn,expectedtld in testdata:
         tld = tldmagic.get_tld(fqdn)
         self.assertEqual(tld,expectedtld, "Expected TLD %s from %s, but got %s"%(expectedtld,fqdn,tld))
Exemple #3
0
 def test_get_tld_count(self):
     tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
     testdata=[
         ('bla.com',1),
         ('co.uk',2),
         ('bla.co.uk',2),
         ('bla.blubb.co.uk',2),
         ('bla.org.co.uk',3),
     ]
     for fqdn,expectedtldcount in testdata:
         count = tldmagic.get_tld_count(fqdn)
         self.assertEqual(count,expectedtldcount, "Expected TLD count %s from %s, but got %s"%(expectedtldcount,fqdn,count))
Exemple #4
0
 def test_get_tld(self):
     tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
     testdata = [
         ('bla.com', 'com'),
         ('co.uk', 'co.uk'),
         ('bla.co.uk', 'co.uk'),
         ('bla.blubb.co.uk', 'co.uk'),
         ('bla.org.co.uk', 'org.co.uk'),
     ]
     for fqdn, expectedtld in testdata:
         tld = tldmagic.get_tld(fqdn)
         self.assertEqual(
             tld, expectedtld, "Expected TLD %s from %s, but got %s" %
             (expectedtld, fqdn, tld))
Exemple #5
0
 def test_get_domain(self):
     tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
     testdata = [
         ('bla.com', 'bla.com'),
         ('co.uk', 'co.uk'),
         ('bla.co.uk', 'bla.co.uk'),
         ('bla.blubb.co.uk', 'blubb.co.uk'),
         ('bla.org.co.uk', 'bla.org.co.uk'),
     ]
     for fqdn, expecteddomain in testdata:
         domain = tldmagic.get_domain(fqdn)
         self.assertEqual(
             domain, expecteddomain,
             "Expected Domain %s from %s, but got %s" %
             (expecteddomain, fqdn, domain))
Exemple #6
0
 def test_get_tld_count(self):
     tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
     testdata = [
         ('bla.com', 1),
         ('co.uk', 2),
         ('bla.co.uk', 2),
         ('bla.blubb.co.uk', 2),
         ('bla.org.co.uk', 3),
     ]
     for fqdn, expectedtldcount in testdata:
         count = tldmagic.get_tld_count(fqdn)
         self.assertEqual(
             count, expectedtldcount,
             "Expected TLD count %s from %s, but got %s" %
             (expectedtldcount, fqdn, count))
Exemple #7
0
    def _init_tldmagic(self):
        init_tldmagic = False
        extratlds = []

        if self.extratlds is None:
            extratldfile = self.config.get(self.section, 'extra_tld_file')
            if extratldfile and os.path.exists(extratldfile):
                self.extratlds = FileList(extratldfile, lowercase=True)
                init_tldmagic = True

        if self.extratlds is not None:
            extratlds = self.extratlds.get_list()
            if self.lasttlds != extratlds:  # extra tld file changed
                self.lasttlds = extratlds
                init_tldmagic = True

        if self.tldmagic is None or init_tldmagic:
            self.tldmagic = TLDMagic()
            for tld in extratlds:  # add extra tlds to tldmagic
                self.tldmagic.add_tld(tld)
Exemple #8
0
class DomainAction(ScannerPlugin):
    """Perform Action based on Domains in message body"""
    def __init__(self, config, section=None):
        ScannerPlugin.__init__(self, config, section)
        self.logger = self._logger()

        self.requiredvars = {
            'blacklistconfig': {
                'default': '/etc/fuglu/rbl.conf',
                'description': 'RBL Lookup config file',
            },
            'checksubdomains': {
                'default':
                'yes',
                'description':
                'check subdomains as well (from top to bottom, eg. example.com, bla.example.com, blubb.bla.example.com',
            },
            'action': {
                'default': 'reject',
                'description': 'action on hit (reject, delete, etc)',
            },
            'message': {
                'default': '5.7.1 black listed URL ${domain} by ${blacklist}',
                'description': 'message template for rejects/ok messages',
            },
            'maxdomains': {
                'default': '10',
                'description':
                'maximum number of domains to check per message',
            },
            'extra_tld_file': {
                'default':
                '',
                'description':
                'directory containing files with extra TLDs (2TLD or inofficial TLDs)'
            },
        }

        self.rbllookup = None
        self.tldmagic = None
        self.extratlds = None
        self.lasttlds = None

    def _init_tldmagic(self):
        init_tldmagic = False
        extratlds = []

        if self.extratlds is None:
            extratldfile = self.config.get(self.section, 'extra_tld_file')
            if extratldfile and os.path.exists(extratldfile):
                self.extratlds = FileList(extratldfile, lowercase=True)
                init_tldmagic = True

        if self.extratlds is not None:
            extratlds = self.extratlds.get_list()
            if self.lasttlds != extratlds:  # extra tld file changed
                self.lasttlds = extratlds
                init_tldmagic = True

        if self.tldmagic is None or init_tldmagic:
            self.tldmagic = TLDMagic()
            for tld in extratlds:  # add extra tlds to tldmagic
                self.tldmagic.add_tld(tld)

    def examine(self, suspect):
        if not DOMAINMAGIC_AVAILABLE:
            self.logger.info('Not scanning - Domainmagic not available')
            return DUNNO

        if self.rbllookup is None:
            self.rbllookup = RBLLookup()
            self.rbllookup.from_config(
                self.config.get(self.section, 'blacklistconfig'))
        self._init_tldmagic()

        urls = suspect.get_tag('body.uris', defaultvalue=[])
        #self.logger.info("Body URIs to check: %s"%urls)
        domains = set(map(fqdn_from_uri, urls))

        counter = 0
        for domain in domains:
            counter += 1
            if counter > self.config.getint(self.section, 'maxdomains'):
                self.logger.info("maximum number of domains reached")
                break

            tldcount = self.tldmagic.get_tld_count(domain)
            parts = domain.split('.')

            if self.config.getboolean(self.section, 'checksubdomains'):
                subrange = range(tldcount + 1, len(parts) + 1)
            else:
                subrange = [tldcount + 1]

            for subindex in subrange:
                subdomain = '.'.join(parts[-subindex:])

                listings = self.rbllookup.listings(subdomain)
                for identifier, humanreadable in iter(listings.items()):
                    self.logger.info(
                        "%s : url host %s flagged as %s because %s" %
                        (suspect.id, domain, identifier, humanreadable))
                    return string_to_actioncode(
                        self.config.get(self.section, 'action'),
                        self.config), apply_template(
                            self.config.get(self.section, 'message'), suspect,
                            dict(domain=domain, blacklist=identifier))

        return DUNNO

    def lint(self):
        allok = True
        if not DOMAINMAGIC_AVAILABLE:
            print(
                "ERROR: domainmagic lib or one of its dependencies (dnspython/pygeoip) is not installed!"
            )
            allok = False

        if allok:
            allok = self.check_config()

        if allok:
            extratldfile = self.config.get(self.section, 'extra_tld_file')
            if extratldfile and not os.path.exists(extratldfile):
                allok = False
                print('WARNING: invalid extra_tld_file %s specified' %
                      extratldfile)

        return allok