def test_get_tld_count(self): tldmagic = TLDMagic(['com','co.uk','org.co.uk']) testdata=[ ('bla.com',1), ('co.uk',2), ('bla.co.uk',2), ('bla.blubb.co.uk',2), ('bla.org.co.uk',3), ] for fqdn,expectedtldcount in testdata: count = tldmagic.get_tld_count(fqdn) self.assertEqual(count,expectedtldcount, "Expected TLD count %s from %s, but got %s"%(expectedtldcount,fqdn,count))
def test_get_tld_count(self): tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk']) testdata = [ ('bla.com', 1), ('co.uk', 2), ('bla.co.uk', 2), ('bla.blubb.co.uk', 2), ('bla.org.co.uk', 3), ] for fqdn, expectedtldcount in testdata: count = tldmagic.get_tld_count(fqdn) self.assertEqual( count, expectedtldcount, "Expected TLD count %s from %s, but got %s" % (expectedtldcount, fqdn, count))
class DomainAction(ScannerPlugin): """Perform Action based on Domains in message body""" def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.requiredvars = { 'blacklistconfig': { 'default': '/etc/fuglu/rbl.conf', 'description': 'RBL Lookup config file', }, 'checksubdomains': { 'default': 'yes', 'description': 'check subdomains as well (from top to bottom, eg. example.com, bla.example.com, blubb.bla.example.com', }, 'action': { 'default': 'reject', 'description': 'action on hit (reject, delete, etc)', }, 'message': { 'default': '5.7.1 black listed URL ${domain} by ${blacklist}', 'description': 'message template for rejects/ok messages', }, 'maxdomains': { 'default': '10', 'description': 'maximum number of domains to check per message', }, 'extra_tld_file': { 'default': '', 'description': 'directory containing files with extra TLDs (2TLD or inofficial TLDs)' }, } self.rbllookup = None self.tldmagic = None self.extratlds = None self.lasttlds = None def _init_tldmagic(self): init_tldmagic = False extratlds = [] if self.extratlds is None: extratldfile = self.config.get(self.section, 'extra_tld_file') if extratldfile and os.path.exists(extratldfile): self.extratlds = FileList(extratldfile, lowercase=True) init_tldmagic = True if self.extratlds is not None: extratlds = self.extratlds.get_list() if self.lasttlds != extratlds: # extra tld file changed self.lasttlds = extratlds init_tldmagic = True if self.tldmagic is None or init_tldmagic: self.tldmagic = TLDMagic() for tld in extratlds: # add extra tlds to tldmagic self.tldmagic.add_tld(tld) def examine(self, suspect): if not DOMAINMAGIC_AVAILABLE: self.logger.info('Not scanning - Domainmagic not available') return DUNNO if self.rbllookup is None: self.rbllookup = RBLLookup() self.rbllookup.from_config( self.config.get(self.section, 'blacklistconfig')) self._init_tldmagic() urls = suspect.get_tag('body.uris', defaultvalue=[]) #self.logger.info("Body URIs to check: %s"%urls) domains = set(map(fqdn_from_uri, urls)) counter = 0 for domain in domains: counter += 1 if counter > self.config.getint(self.section, 'maxdomains'): self.logger.info("maximum number of domains reached") break tldcount = self.tldmagic.get_tld_count(domain) parts = domain.split('.') if self.config.getboolean(self.section, 'checksubdomains'): subrange = range(tldcount + 1, len(parts) + 1) else: subrange = [tldcount + 1] for subindex in subrange: subdomain = '.'.join(parts[-subindex:]) listings = self.rbllookup.listings(subdomain) for identifier, humanreadable in iter(listings.items()): self.logger.info( "%s : url host %s flagged as %s because %s" % (suspect.id, domain, identifier, humanreadable)) return string_to_actioncode( self.config.get(self.section, 'action'), self.config), apply_template( self.config.get(self.section, 'message'), suspect, dict(domain=domain, blacklist=identifier)) return DUNNO def lint(self): allok = True if not DOMAINMAGIC_AVAILABLE: print( "ERROR: domainmagic lib or one of its dependencies (dnspython/pygeoip) is not installed!" ) allok = False if allok: allok = self.check_config() if allok: extratldfile = self.config.get(self.section, 'extra_tld_file') if extratldfile and not os.path.exists(extratldfile): allok = False print('WARNING: invalid extra_tld_file %s specified' % extratldfile) return allok