def test_get_domain(self):
tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
testdata=[
('bla.com','bla.com'),
('co.uk','co.uk'),
('bla.co.uk','bla.co.uk'),
('bla.blubb.co.uk','blubb.co.uk'),
('bla.org.co.uk','bla.org.co.uk'),
]
for fqdn,expecteddomain in testdata:
domain = tldmagic.get_domain(fqdn)
self.assertEqual(domain,expecteddomain, "Expected Domain %s from %s, but got %s"%(expecteddomain,fqdn,domain))
def test_get_tld(self):
tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
testdata=[
('bla.com','com'),
('co.uk','co.uk'),
('bla.co.uk','co.uk'),
('bla.blubb.co.uk','co.uk'),
('bla.org.co.uk','org.co.uk'),
]
for fqdn,expectedtld in testdata:
tld = tldmagic.get_tld(fqdn)
self.assertEqual(tld,expectedtld, "Expected TLD %s from %s, but got %s"%(expectedtld,fqdn,tld))
def test_get_tld_count(self):
tldmagic = TLDMagic(['com','co.uk','org.co.uk'])
testdata=[
('bla.com',1),
('co.uk',2),
('bla.co.uk',2),
('bla.blubb.co.uk',2),
('bla.org.co.uk',3),
]
for fqdn,expectedtldcount in testdata:
count = tldmagic.get_tld_count(fqdn)
self.assertEqual(count,expectedtldcount, "Expected TLD count %s from %s, but got %s"%(expectedtldcount,fqdn,count))
def test_get_tld(self):
tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
testdata = [
('bla.com', 'com'),
('co.uk', 'co.uk'),
('bla.co.uk', 'co.uk'),
('bla.blubb.co.uk', 'co.uk'),
('bla.org.co.uk', 'org.co.uk'),
]
for fqdn, expectedtld in testdata:
tld = tldmagic.get_tld(fqdn)
self.assertEqual(
tld, expectedtld, "Expected TLD %s from %s, but got %s" %
(expectedtld, fqdn, tld))
def test_get_domain(self):
tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
testdata = [
('bla.com', 'bla.com'),
('co.uk', 'co.uk'),
('bla.co.uk', 'bla.co.uk'),
('bla.blubb.co.uk', 'blubb.co.uk'),
('bla.org.co.uk', 'bla.org.co.uk'),
]
for fqdn, expecteddomain in testdata:
domain = tldmagic.get_domain(fqdn)
self.assertEqual(
domain, expecteddomain,
"Expected Domain %s from %s, but got %s" %
(expecteddomain, fqdn, domain))
def test_get_tld_count(self):
tldmagic = TLDMagic(['com', 'co.uk', 'org.co.uk'])
testdata = [
('bla.com', 1),
('co.uk', 2),
('bla.co.uk', 2),
('bla.blubb.co.uk', 2),
('bla.org.co.uk', 3),
]
for fqdn, expectedtldcount in testdata:
count = tldmagic.get_tld_count(fqdn)
self.assertEqual(
count, expectedtldcount,
"Expected TLD count %s from %s, but got %s" %
(expectedtldcount, fqdn, count))
def _init_tldmagic(self):
init_tldmagic = False
extratlds = []
if self.extratlds is None:
extratldfile = self.config.get(self.section, 'extra_tld_file')
if extratldfile and os.path.exists(extratldfile):
self.extratlds = FileList(extratldfile, lowercase=True)
init_tldmagic = True
if self.extratlds is not None:
extratlds = self.extratlds.get_list()
if self.lasttlds != extratlds: # extra tld file changed
self.lasttlds = extratlds
init_tldmagic = True
if self.tldmagic is None or init_tldmagic:
self.tldmagic = TLDMagic()
for tld in extratlds: # add extra tlds to tldmagic
self.tldmagic.add_tld(tld)
class DomainAction(ScannerPlugin):
"""Perform Action based on Domains in message body"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'blacklistconfig': {
'default': '/etc/fuglu/rbl.conf',
'description': 'RBL Lookup config file',
},
'checksubdomains': {
'default':
'yes',
'description':
'check subdomains as well (from top to bottom, eg. example.com, bla.example.com, blubb.bla.example.com',
},
'action': {
'default': 'reject',
'description': 'action on hit (reject, delete, etc)',
},
'message': {
'default': '5.7.1 black listed URL ${domain} by ${blacklist}',
'description': 'message template for rejects/ok messages',
},
'maxdomains': {
'default': '10',
'description':
'maximum number of domains to check per message',
},
'extra_tld_file': {
'default':
'',
'description':
'directory containing files with extra TLDs (2TLD or inofficial TLDs)'
},
}
self.rbllookup = None
self.tldmagic = None
self.extratlds = None
self.lasttlds = None
def _init_tldmagic(self):
init_tldmagic = False
extratlds = []
if self.extratlds is None:
extratldfile = self.config.get(self.section, 'extra_tld_file')
if extratldfile and os.path.exists(extratldfile):
self.extratlds = FileList(extratldfile, lowercase=True)
init_tldmagic = True
if self.extratlds is not None:
extratlds = self.extratlds.get_list()
if self.lasttlds != extratlds: # extra tld file changed
self.lasttlds = extratlds
init_tldmagic = True
if self.tldmagic is None or init_tldmagic:
self.tldmagic = TLDMagic()
for tld in extratlds: # add extra tlds to tldmagic
self.tldmagic.add_tld(tld)
def examine(self, suspect):
if not DOMAINMAGIC_AVAILABLE:
self.logger.info('Not scanning - Domainmagic not available')
return DUNNO
if self.rbllookup is None:
self.rbllookup = RBLLookup()
self.rbllookup.from_config(
self.config.get(self.section, 'blacklistconfig'))
self._init_tldmagic()
urls = suspect.get_tag('body.uris', defaultvalue=[])
#self.logger.info("Body URIs to check: %s"%urls)
domains = set(map(fqdn_from_uri, urls))
counter = 0
for domain in domains:
counter += 1
if counter > self.config.getint(self.section, 'maxdomains'):
self.logger.info("maximum number of domains reached")
break
tldcount = self.tldmagic.get_tld_count(domain)
parts = domain.split('.')
if self.config.getboolean(self.section, 'checksubdomains'):
subrange = range(tldcount + 1, len(parts) + 1)
else:
subrange = [tldcount + 1]
for subindex in subrange:
subdomain = '.'.join(parts[-subindex:])
listings = self.rbllookup.listings(subdomain)
for identifier, humanreadable in iter(listings.items()):
self.logger.info(
"%s : url host %s flagged as %s because %s" %
(suspect.id, domain, identifier, humanreadable))
return string_to_actioncode(
self.config.get(self.section, 'action'),
self.config), apply_template(
self.config.get(self.section, 'message'), suspect,
dict(domain=domain, blacklist=identifier))
return DUNNO
def lint(self):
allok = True
if not DOMAINMAGIC_AVAILABLE:
print(
"ERROR: domainmagic lib or one of its dependencies (dnspython/pygeoip) is not installed!"
)
allok = False
if allok:
allok = self.check_config()
if allok:
extratldfile = self.config.get(self.section, 'extra_tld_file')
if extratldfile and not os.path.exists(extratldfile):
allok = False
print('WARNING: invalid extra_tld_file %s specified' %
extratldfile)
return allok
