def examine(self, suspect):
helo_name = suspect.get_value('helo_name')
if helo_name is None:
self.logger.error('missing helo')
return DUNNO
helo_tld = helo_name.split('.')[-1].lower()
#initialize loaders
tld_file = self.config.get(self.section, 'tldfile')
if self.tld_loader is None:
self.tld_loader = FileList(tld_file,
lowercase=True,
minimum_time_between_reloads=3600)
if helo_tld in self.tld_loader.get_list():
return DUNNO, ''
exceptionfile = self.config.get(self.section, 'exceptionfile')
if self.exception_loader is None:
self.exception_loader = FileList(exceptionfile,
lowercase=True,
minimum_time_between_reloads=10)
if helo_tld in self.exception_loader.get_list():
return DUNNO, ''
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
dict(helo_tld=helo_tld))
action = self.config.get(self.section, "on_fail")
return action, message
def ip_whitelisted(self, addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file = self.config.get(self.section, 'ip_whitelist_file',
'').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader = FileList(ip_whitelist_file,
lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist = self.ip_whitelist_loader.get_list()
if HAVE_NETADDR:
self.ip_whitelist = [IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist = plainlist
if HAVE_NETADDR:
checkaddr = IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def enforce_domain(self, to_domain):
global SETTINGSCACHE
dbconnection = self.config.get(self.section, 'dbconnection',
'').strip()
domainlist = self.config.get(self.section, 'domainlist')
enforce = False
if domainlist.strip() == '':
enforce = True
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if self.selective_domain_loader is None:
self.selective_domain_loader = FileList(domainfile,
lowercase=True)
if to_domain in self.selective_domain_loader.get_list():
enforce = True
elif domainlist.startswith('sql:') and dbconnection != '':
if SETTINGSCACHE is None:
SETTINGSCACHE = SettingsCache()
sqlquery = domainlist[4:]
enforce = get_domain_setting(to_domain, dbconnection, sqlquery,
SETTINGSCACHE, False, self.logger)
return enforce
def examine(self,suspect):
helo_name=suspect.get_value('helo_name')
if helo_name is None :
self.logger.error('missing helo')
return DUNNO
helo_tld=helo_name.split('.')[-1].lower()
#initialize loaders
tld_file=self.config.get(self.section,'tldfile')
if self.tld_loader is None:
self.tld_loader=FileList(tld_file,lowercase=True,minimum_time_between_reloads=3600)
if helo_tld in self.tld_loader.get_list():
return DUNNO,''
exceptionfile=self.config.get(self.section,'exceptionfile')
if self.exception_loader is None:
self.exception_loader=FileList(exceptionfile,lowercase=True,minimum_time_between_reloads=10)
if helo_tld in self.exception_loader.get_list():
return DUNNO,''
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(helo_tld=helo_tld))
action=self.config.get(self.section,"on_fail")
return action, message
def check_this_domain(self, from_domain):
global SETTINGSCACHE
do_check = False
selective_sender_domain_file = self.config.get(
self.section, 'domain_selective_spf_file', '').strip()
if selective_sender_domain_file != '' and os.path.exists(
selective_sender_domain_file):
if self.selective_domain_loader is None:
self.selective_domain_loader = FileList(
selective_sender_domain_file, lowercase=True)
if from_domain.lower() in self.selective_domain_loader.get_list():
do_check = True
if not do_check:
dbconnection = self.config.get(self.section, 'dbconnection',
'').strip()
sqlquery = self.config.get(self.section, 'domain_sql_query')
if dbconnection != '' and SQLALCHEMY_AVAILABLE:
if SETTINGSCACHE is None:
SETTINGSCACHE = SettingsCache()
do_check = get_domain_setting(from_domain, dbconnection,
sqlquery, SETTINGSCACHE, False,
self.logger)
elif dbconnection != '' and not SQLALCHEMY_AVAILABLE:
self.logger.error(
'dbconnection specified but sqlalchemy not available - skipping db lookup'
)
return do_check
def check_this_domain(self, from_domain):
do_check = False
selective_sender_domain_file = self.config.get(
self.section, 'domain_selective_spf_file', '').strip()
if selective_sender_domain_file != '' and os.path.exists(
selective_sender_domain_file):
if self.selective_domain_loader is None:
self.selective_domain_loader = FileList(
selective_sender_domain_file, lowercase=True)
if from_domain.lower() in self.selective_domain_loader.get_list():
do_check = True
if not do_check:
dbconnection = self.config.get(self.section, 'dbconnection',
'').strip()
sqlquery = self.config.get(self.section, 'domain_sql_query')
if dbconnection != '' and SQL_EXTENSION_ENABLED:
cache = get_default_cache()
do_check = get_domain_setting(from_domain, dbconnection,
sqlquery, cache, self.section,
False, self.logger)
elif dbconnection != '' and not SQL_EXTENSION_ENABLED:
self.logger.error(
'dbconnection specified but sqlalchemy not available - skipping db lookup'
)
return do_check
def _is_whitelisted(self, from_domain):
whitelist_file = self.config.get(self.section, 'whitelist_file',
'').strip()
if whitelist_file == '':
return False
whitelisted = False
self.whitelist = FileList(whitelist_file, lowercase=True)
if from_domain in self.whitelist.get_list():
whitelisted = True
return whitelisted
def __init__(self,
filename=None,
strip=True,
skip_empty=True,
skip_comments=True,
lowercase=False,
additional_filters=None,
minimum_time_between_reloads=5):
self.addresses = {}
self.names = {}
FileList.__init__(self, filename, strip, skip_empty, skip_comments,
lowercase, additional_filters,
minimum_time_between_reloads)
def ip_whitelisted(self,addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file')
if ip_whitelist_file is not None:
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist=self.ip_whitelist_loader.get_list()
if have_netaddr:
self.ip_whitelist=[IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist=plainlist
if have_netaddr:
checkaddr=IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def ip_whitelisted(self,addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file', '').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist=self.ip_whitelist_loader.get_list()
if HAVE_NETADDR:
self.ip_whitelist=[IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist=plainlist
if HAVE_NETADDR:
checkaddr=IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def _is_whitelisted(self, from_domain):
whitelist_file = self.config.get(self.section,'whitelist_file').strip()
if whitelist_file == '':
return False
whitelisted = False
self.whitelist = FileList(whitelist_file,lowercase=True)
if from_domain in self.whitelist.get_list():
whitelisted = True
return whitelisted
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'filename':{
'default':'/etc/postomaat/complexrules.cf',
'description':'File containing rules',
},
}
self.ruleparser=ComplexRuleParser()
self.filereloader=FileList()
class CreativeTLD(ScannerPlugin):
""" Reject clients with unofficial TLD in rdns """
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'action':{
'default':'REJECT',
'description':'Action if sender uses invalid TLD',
},
'message':{
'default':'forged rDNS TLD',
},
'tldfile':{
'default':'/etc/mail/tlds-alpha-by-domain.txt',
},
}
self.filelist=FileList(filename=None,strip=True, skip_empty=True, skip_comments=True,lowercase=True,minimum_time_between_reloads=86400)
def examine(self,suspect):
retaction,retmessage = DUNNO,''
revclient=suspect.get_value('reverse_client_name')
self.filelist.filename=self.config.get(self.section,'domainsfile')
tlds = self.filelist.get_list()
if revclient is None or revclient.strip()=='unknown' or '.' not in revclient:
return DUNNO,''
tld=revclient.split('.')[-1].lower()
if tld not in tlds:
retaction=self.config.get(self.section,'action').strip()
retmessage=self.config.get(self.section,'message').strip()
return retaction,retmessage
def lint(self):
lint_ok=True
retaction=self.config.get(self.section,'action').strip().lower()
reasonable_actions=[REJECT,DEFER,DEFER_IF_PERMIT,FILTER,HOLD,PREPEND,WARN]
if retaction not in reasonable_actions:
print "are you sure about action '%s' ?"%retaction
print "I'd expect one of %s"%(",".join(reasonable_actions))
lint_ok=False
if not self.checkConfig():
print 'Error checking config'
lint_ok = False
return lint_ok
def __str__(self):
return "Creative TLD"
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'action': {
'default': 'REJECT',
'description': 'Action if sender uses invalid TLD',
},
'message': {
'default': 'forged rDNS TLD',
},
'tldfile': {
'default': '/etc/mail/tlds-alpha-by-domain.txt',
},
}
self.filelist = FileList(filename=None,
strip=True,
skip_empty=True,
skip_comments=True,
lowercase=True,
minimum_time_between_reloads=86400)
def examine(self,suspect):
if not have_spf:
return DUNNO
client_address=suspect.get_value('client_address')
helo_name=suspect.get_value('helo_name')
sender=suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check"%client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email=='' or sender_email is None:
return DUNNO
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file')
if selective_sender_domain_file!='':
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True)
try:
sender_domain = extract_domain(sender_email)
if sender_domain is None:
return DUNNO
except ValueError as e:
self.logger.warning(str(e))
return DUNNO
if not sender_domain.lower() in self.selective_domain_loader.get_list():
return DUNNO
result, explanation = spf.check2(client_address, sender_email, helo_name)
suspect.tags['spf']=result
if result!='none':
self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation))
action = DUNNO
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(result=result,explanation=explanation))
configopt='on_%s'%result
if self.config.has_option(self.section,configopt):
action=self.config.get(self.section,configopt)
return action, message
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'action':{
'default':'REJECT',
'description':'Action if sender uses invalid TLD',
},
'message':{
'default':'forged rDNS TLD',
},
'tldfile':{
'default':'/etc/mail/tlds-alpha-by-domain.txt',
},
}
self.filelist=FileList(filename=None,strip=True, skip_empty=True, skip_comments=True,lowercase=True,minimum_time_between_reloads=86400)
def enforce_domain(self, to_domain):
dbconnection = self.config.get(self.section,'dbconnection', '').strip()
domainlist = self.config.get(self.section,'domainlist')
enforce = False
if domainlist.strip() == '':
enforce = True
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(domainfile,lowercase=True)
if to_domain in self.selective_domain_loader.get_list():
enforce = True
elif domainlist.startswith('sql:') and dbconnection != '':
cache = get_default_cache()
sqlquery = domainlist[4:]
enforce = get_domain_setting(to_domain, dbconnection, sqlquery, cache, self.section, False, self.logger)
return enforce
def check_this_domain(self, from_domain):
do_check = False
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file','').strip()
if selective_sender_domain_file != '' and os.path.exists(selective_sender_domain_file):
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True)
if from_domain.lower() in self.selective_domain_loader.get_list():
do_check = True
if not do_check:
dbconnection = self.config.get(self.section, 'dbconnection', '').strip()
sqlquery = self.config.get(self.section, 'domain_sql_query')
if dbconnection!='' and SQL_EXTENSION_ENABLED:
cache = get_default_cache()
do_check = get_domain_setting(from_domain, dbconnection, sqlquery, cache, self.section, False, self.logger)
elif dbconnection!='' and not SQL_EXTENSION_ENABLED:
self.logger.error('dbconnection specified but sqlalchemy not available - skipping db lookup')
return do_check
class SPFPlugin(ScannerPlugin):
"""This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/
by default, it just logs the result (test mode)
to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg:
on_fail = REJECT
valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral'
"""
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'ip_whitelist_file':{
'default':'',
'description':'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted',
},
'domain_selective_spf_file':{
'default':'',
'description':'if this is non-empty, only sender domains in this file will be checked for SPF',
},
'on_fail':{
'default':'DUNNO',
'description':'Action for SPF fail.',
},
'on_softfail':{
'default':'DUNNO',
'description':'Action for SPF softfail.',
},
'messagetemplate':{
'default':'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}'
}
}
self.ip_whitelist_loader=None
self.ip_whitelist=[] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available
self.selective_domain_loader=None
def is_private_address(self,addr):
if addr=='127.0.0.1' or addr=='::1' or addr.startswith('10.') or addr.startswith('192.168.') or addr.startswith('fe80:'):
return True
if not addr.startswith('172.'):
return False
for i in range(16,32):
if addr.startswith('172.%s'%i):
return True
return False
def ip_whitelisted(self,addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file')
if ip_whitelist_file is not None:
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist=self.ip_whitelist_loader.get_list()
if have_netaddr:
self.ip_whitelist=[IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist=plainlist
if have_netaddr:
checkaddr=IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def examine(self,suspect):
if not have_spf:
return DUNNO
client_address=suspect.get_value('client_address')
helo_name=suspect.get_value('helo_name')
sender=suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check"%client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email=='' or sender_email is None:
return DUNNO
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file')
if selective_sender_domain_file!='':
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True)
try:
sender_domain = extract_domain(sender_email)
if sender_domain is None:
return DUNNO
except ValueError as e:
self.logger.warning(str(e))
return DUNNO
if not sender_domain.lower() in self.selective_domain_loader.get_list():
return DUNNO
result, explanation = spf.check2(client_address, sender_email, helo_name)
suspect.tags['spf']=result
if result!='none':
self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation))
action = DUNNO
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(result=result,explanation=explanation))
configopt='on_%s'%result
if self.config.has_option(self.section,configopt):
action=self.config.get(self.section,configopt)
return action, message
def lint(self):
lint_ok = True
if not have_spf:
print 'pyspf or pydns module not installed - this plugin will do nothing'
lint_ok = False
if not have_netaddr:
print 'WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation'
if not self.checkConfig():
print 'Error checking config'
lint_ok = False
return lint_ok
def __str__(self):
return "SPF"
def __init__(self, filename, **kw):
FileList.__init__(self, filename, **kw)
self.geoip = None
class EnforceTLS(ScannerPlugin):
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.selective_domain_loader = None
self.requiredvars={
'domainlist':{
'default':'',
'description':"""
if this is empty, all recipient domains will be forced to use TLS
txt:<filename> - get from simple textfile which lists one domain per line
sql:<statement> - get from sql database :domain will be replaced with the actual domain name. must return field enforce_inbound_tls
""",
},
'dbconnection':{
'default':"mysql://root@localhost/enforcetls?charset=utf8",
'description':'SQLAlchemy Connection string',
},
'action':{
'default':'DEFER',
'description':'Action if connection is not TLS encrypted. set to DUNNO, DEFER, REJECT',
},
'messagetemplate':{
'default':'Unencrypted connection. This recipient requires TLS'
}
}
def enforce_domain(self, to_domain):
dbconnection = self.config.get(self.section,'dbconnection', '').strip()
domainlist = self.config.get(self.section,'domainlist')
enforce = False
if domainlist.strip() == '':
enforce = True
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(domainfile,lowercase=True)
if to_domain in self.selective_domain_loader.get_list():
enforce = True
elif domainlist.startswith('sql:') and dbconnection != '':
cache = get_default_cache()
sqlquery = domainlist[4:]
enforce = get_domain_setting(to_domain, dbconnection, sqlquery, cache, self.section, False, self.logger)
return enforce
def examine(self, suspect):
encryption_protocol = suspect.get_value('encryption_protocol')
recipient=suspect.get_value('recipient')
rcpt_email = strip_address(recipient)
if rcpt_email=='' or rcpt_email is None:
return DUNNO
enforce = self.enforce_domain(extract_domain(rcpt_email))
action = DUNNO
message = None
if enforce and encryption_protocol == '':
action=string_to_actioncode(self.config.get(self.section, 'action'))
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect)
return action, message
def lint(self):
lint_ok = True
if not self.checkConfig():
print('Error checking config')
lint_ok = False
if lint_ok:
domainlist = self.config.get(self.section,'domainlist')
if domainlist.strip() == '':
print('Enforcing TLS for all domains')
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if not os.path.exists(domainfile):
print('Cannot find domain file %s' % domainfile)
lint_ok = False
elif domainlist.startswith('sql:'):
sqlquery = domainlist[4:]
if not sqlquery.lower().startswith('select '):
lint_ok = False
print('SQL statement must be a SELECT query')
if not SQL_EXTENSION_ENABLED:
print('SQLAlchemy not available, cannot use sql backend')
if lint_ok:
dbconnection = self.config.get(self.section, 'dbconnection')
try:
conn=get_session(dbconnection)
conn.execute(sqlquery, {'domain':'example.com'})
except Exception as e:
lint_ok = False
print(str(e))
else:
lint_ok = False
print('Could not determine domain list backend type')
return lint_ok
def __str__(self):
return "EnforceTLS"
class HELOTLDPlugin(ScannerPlugin):
"""
This plugin rejects messages if the HELO uses an invalid TLD
"""
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'tldfile':{
'default':'/etc/mail/tlds-alpha-by-domain.txt',
'description':'filename containing official TLDs. Add a cronjob to dowload this.',
},
'exceptionfile':{
'default':'/etc/mail/tlds-exceptions.txt',
'description':'additional tld file with local exceptions',
},
'on_fail':{
'default':'REJECT',
'description':'Action to take if the TLD is invalid',
},
'messagetemplate':{
'default':"""HELO ${helo_name} contains forged/unresolvable TLD '.${helo_tld}'"""
}
}
self.tld_loader=None
self.exception_loader=None
def examine(self,suspect):
helo_name=suspect.get_value('helo_name')
if helo_name is None :
self.logger.error('missing helo')
return DUNNO
helo_tld=helo_name.split('.')[-1].lower()
#initialize loaders
tld_file=self.config.get(self.section,'tldfile')
if self.tld_loader is None:
self.tld_loader=FileList(tld_file,lowercase=True,minimum_time_between_reloads=3600)
if helo_tld in self.tld_loader.get_list():
return DUNNO,''
exceptionfile=self.config.get(self.section,'exceptionfile')
if self.exception_loader is None:
self.exception_loader=FileList(exceptionfile,lowercase=True,minimum_time_between_reloads=10)
if helo_tld in self.exception_loader.get_list():
return DUNNO,''
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(helo_tld=helo_tld))
action=self.config.get(self.section,"on_fail")
return action, message
def lint(self):
lint_ok = True
tld_file=self.config.get(self.section,'tldfile')
exceptionfile=self.config.get(self.section,'exceptionfile')
if not os.path.exists(tld_file):
print("TLD file %s not found"%tld_file)
lint_ok = False
if not os.path.exists(exceptionfile):
print("TLD exception file %s not found"%exceptionfile)
lint_ok = False
if not self.checkConfig():
print('Error checking config')
lint_ok = False
return lint_ok
def __str__(self):
return "HeloTLD"
class ComplexRules(ScannerPlugin):
""" """
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'filename':{
'default':'/etc/postomaat/complexrules.cf',
'description':'File containing rules',
},
}
self.ruleparser=ComplexRuleParser()
self.filereloader=FileList()
def examine(self,suspect):
if not PYPARSING_AVAILABLE:
return DUNNO,''
filename=self.config.get(self.section,'filename').strip()
if not os.path.exists(filename):
self.logger.error("Rulefile %s does not exist"%filename)
return DUNNO,''
self.filereloader.filename=filename
newcontent=self.filereloader._reload_if_necessary()
if newcontent:
self.ruleparser.clear_rules()
reloadok=self.ruleparser.rules_from_list(self.filereloader.content)
numrules=len(self.ruleparser.rules)
if reloadok:
okmsg="all rules ok"
else:
okmsg="some rules failed to load"
self.logger.info("Rule reload complete, %s rules now active, (%s)"%(numrules,okmsg))
retaction,retmessage=self.ruleparser.apply(suspect.values)
return retaction,retmessage
def lint(self):
if not PYPARSING_AVAILABLE:
print("pyparsing is not installed, can not use complex rules")
return False
if RE2_AVAILABLE:
print("Using re2(google) library")
if not self.checkConfig():
print('Error checking config')
return False
filename=self.config.get(self.section,'filename').strip()
if not os.path.exists(filename):
print("Rulefile %s does not exist"%filename)
return False
self.filereloader.filename=filename
newcontent=self.filereloader._reload_if_necessary()
assert newcontent
self.ruleparser.clear_rules()
ok= self.ruleparser.rules_from_list(self.filereloader.content)
rulecount=len(self.ruleparser.rules)
print("%s rules ok"%(rulecount))
return ok
def __str__(self):
return "Complex Rules"
class HELOTLDPlugin(ScannerPlugin):
"""
This plugin rejects messages if the HELO uses an invalid TLD
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'tldfile': {
'default':
'/etc/mail/tlds-alpha-by-domain.txt',
'description':
'filename containing official TLDs. Add a cronjob to dowload this.',
},
'exceptionfile': {
'default': '/etc/mail/tlds-exceptions.txt',
'description': 'additional tld file with local exceptions',
},
'on_fail': {
'default': 'REJECT',
'description': 'Action to take if the TLD is invalid',
},
'messagetemplate': {
'default':
"""HELO ${helo_name} contains forged/unresolvable TLD '.${helo_tld}'"""
}
}
self.tld_loader = None
self.exception_loader = None
def examine(self, suspect):
helo_name = suspect.get_value('helo_name')
if helo_name is None:
self.logger.error('missing helo')
return DUNNO
helo_tld = helo_name.split('.')[-1].lower()
#initialize loaders
tld_file = self.config.get(self.section, 'tldfile')
if self.tld_loader is None:
self.tld_loader = FileList(tld_file,
lowercase=True,
minimum_time_between_reloads=3600)
if helo_tld in self.tld_loader.get_list():
return DUNNO, ''
exceptionfile = self.config.get(self.section, 'exceptionfile')
if self.exception_loader is None:
self.exception_loader = FileList(exceptionfile,
lowercase=True,
minimum_time_between_reloads=10)
if helo_tld in self.exception_loader.get_list():
return DUNNO, ''
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
dict(helo_tld=helo_tld))
action = self.config.get(self.section, "on_fail")
return action, message
def lint(self):
lint_ok = True
tld_file = self.config.get(self.section, 'tldfile')
exceptionfile = self.config.get(self.section, 'exceptionfile')
if not os.path.exists(tld_file):
print("TLD file %s not found" % tld_file)
lint_ok = False
if not os.path.exists(exceptionfile):
print("TLD exception file %s not found" % exceptionfile)
lint_ok = False
if not self.checkConfig():
print('Error checking config')
lint_ok = False
return lint_ok
def __str__(self):
return "HeloTLD"
class EnforceTLS(ScannerPlugin):
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.selective_domain_loader = None
self.requiredvars={
'domainlist':{
'default':'',
'description':"""
if this is empty, all recipient domains will be forced to use TLS
txt:<filename> - get from simple textfile which lists one domain per line
sql:<statement> - get from sql database :domain will be replaced with the actual domain name. must return field enforce_inbound_tls
""",
},
'dbconnection':{
'default':"mysql://root@localhost/enforcetls?charset=utf8",
'description':'SQLAlchemy Connection string',
},
'action':{
'default':'DEFER',
'description':'Action if connection is not TLS encrypted. set to DUNNO, DEFER, REJECT',
},
'messagetemplate':{
'default':'Unencrypted connection. This recipient requires TLS'
}
}
def enforce_domain(self, to_domain):
dbconnection = self.config.get(self.section,'dbconnection', '').strip()
domainlist = self.config.get(self.section,'domainlist')
enforce = False
if domainlist.strip() == '':
enforce = True
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(domainfile,lowercase=True)
if to_domain in self.selective_domain_loader.get_list():
enforce = True
elif domainlist.startswith('sql:') and dbconnection != '':
cache = get_default_cache()
sqlquery = domainlist[4:]
enforce = get_domain_setting(to_domain, dbconnection, sqlquery, cache, self.section, False, self.logger)
return enforce
def examine(self, suspect):
encryption_protocol = suspect.get_value('encryption_protocol')
recipient=suspect.get_value('recipient')
rcpt_email = strip_address(recipient)
if rcpt_email=='' or rcpt_email is None:
return DUNNO
enforce = self.enforce_domain(extract_domain(rcpt_email))
action = DUNNO
message = None
if enforce and encryption_protocol == '':
action=string_to_actioncode(self.config.get(self.section, 'action'))
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect)
return action, message
def lint(self):
lint_ok = True
if not self.checkConfig():
print('Error checking config')
lint_ok = False
if lint_ok:
domainlist = self.config.get(self.section,'domainlist')
if domainlist.strip() == '':
print('Enforcing TLS for all domains')
elif domainlist.startswith('txt:'):
domainfile = domainlist[4:]
if not os.path.exists(domainfile):
print('Cannot find domain file %s' % domainfile)
lint_ok = False
elif domainlist.startswith('sql:'):
sqlquery = domainlist[4:]
if not sqlquery.lower().startswith('select '):
lint_ok = False
print('SQL statement must be a SELECT query')
if not SQL_EXTENSION_ENABLED:
print('SQLAlchemy not available, cannot use sql backend')
if lint_ok:
dbconnection = self.config.get(self.section, 'dbconnection')
try:
conn=get_session(dbconnection)
conn.execute(sqlquery, {'domain':'example.com'})
except Exception as e:
lint_ok = False
print(str(e))
else:
lint_ok = False
print('Could not determine domain list backend type')
return lint_ok
def __str__(self):
return "EnforceTLS"
class CreativeTLD(ScannerPlugin):
""" Reject clients with unofficial TLD in rdns """
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'action': {
'default': 'REJECT',
'description': 'Action if sender uses invalid TLD',
},
'message': {
'default': 'forged rDNS TLD',
},
'tldfile': {
'default': '/etc/mail/tlds-alpha-by-domain.txt',
},
}
self.filelist = FileList(filename=None,
strip=True,
skip_empty=True,
skip_comments=True,
lowercase=True,
minimum_time_between_reloads=86400)
def examine(self, suspect):
retaction, retmessage = DUNNO, ''
revclient = suspect.get_value('reverse_client_name')
self.filelist.filename = self.config.get(self.section, 'domainsfile')
tlds = self.filelist.get_list()
if revclient is None or revclient.strip(
) == 'unknown' or '.' not in revclient:
return DUNNO, ''
tld = revclient.split('.')[-1].lower()
if tld not in tlds:
retaction = self.config.get(self.section, 'action').strip()
retmessage = self.config.get(self.section, 'message').strip()
return retaction, retmessage
def lint(self):
lint_ok = True
retaction = self.config.get(self.section, 'action').strip().lower()
reasonable_actions = [
REJECT, DEFER, DEFER_IF_PERMIT, FILTER, HOLD, PREPEND, WARN
]
if retaction not in reasonable_actions:
print("are you sure about action '%s' ?" % retaction)
print("I'd expect one of %s" % (",".join(reasonable_actions)))
lint_ok = False
if not self.checkConfig():
print('Error checking config')
lint_ok = False
return lint_ok
def __str__(self):
return "Creative TLD"
class SPFPlugin(ScannerPlugin):
"""This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/
by default, it just logs the result (test mode)
to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg:
on_fail = REJECT
valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral'
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'ip_whitelist_file': {
'default':
'',
'description':
'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted',
},
'domain_selective_spf_file': {
'default':
'',
'description':
'if this is non-empty, only sender domains in this file will be checked for SPF',
},
'dbconnection': {
'default':
"mysql://root@localhost/spfcheck?charset=utf8",
'description':
'SQLAlchemy Connection string. Leave empty to disable SQL lookups',
},
'domain_sql_query': {
'default':
"SELECT check_spf from domain where domain_name=:domain",
'description':
'get from sql database :domain will be replaced with the actual domain name. must return field check_spf',
},
'on_fail': {
'default': 'DUNNO',
'description': 'Action for SPF fail.',
},
'on_softfail': {
'default': 'DUNNO',
'description': 'Action for SPF softfail.',
},
'messagetemplate': {
'default':
'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}'
}
}
self.ip_whitelist_loader = None
self.ip_whitelist = [
] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available
self.selective_domain_loader = None
def check_this_domain(self, from_domain):
global SETTINGSCACHE
do_check = False
selective_sender_domain_file = self.config.get(
self.section, 'domain_selective_spf_file', '').strip()
if selective_sender_domain_file != '' and os.path.exists(
selective_sender_domain_file):
if self.selective_domain_loader is None:
self.selective_domain_loader = FileList(
selective_sender_domain_file, lowercase=True)
if from_domain.lower() in self.selective_domain_loader.get_list():
do_check = True
if not do_check:
dbconnection = self.config.get(self.section, 'dbconnection',
'').strip()
sqlquery = self.config.get(self.section, 'domain_sql_query')
if dbconnection != '' and SQLALCHEMY_AVAILABLE:
if SETTINGSCACHE is None:
SETTINGSCACHE = SettingsCache()
do_check = get_domain_setting(from_domain, dbconnection,
sqlquery, SETTINGSCACHE, False,
self.logger)
elif dbconnection != '' and not SQLALCHEMY_AVAILABLE:
self.logger.error(
'dbconnection specified but sqlalchemy not available - skipping db lookup'
)
return do_check
def is_private_address(self, addr):
if addr == '127.0.0.1' or addr == '::1' or addr.startswith(
'10.') or addr.startswith('192.168.') or addr.startswith(
'fe80:'):
return True
if not addr.startswith('172.'):
return False
for i in range(16, 32):
if addr.startswith('172.%s' % i):
return True
return False
def ip_whitelisted(self, addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file = self.config.get(self.section, 'ip_whitelist_file',
'').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader = FileList(ip_whitelist_file,
lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist = self.ip_whitelist_loader.get_list()
if HAVE_NETADDR:
self.ip_whitelist = [IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist = plainlist
if HAVE_NETADDR:
checkaddr = IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def examine(self, suspect):
if not HAVE_SPF:
return DUNNO
client_address = suspect.get_value('client_address')
helo_name = suspect.get_value('helo_name')
sender = suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check" %
client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email == '' or sender_email is None:
return DUNNO
sender_domain = extract_domain(sender_email)
if sender_domain is None:
self.logger.error('no domain found in sender address %s' %
sender_email)
return DUNNO
if not self.check_this_domain(sender_domain):
self.logger.debug('skipping SPF check for %s' % sender_domain)
return DUNNO
result, explanation = spf.check2(client_address, sender_email,
helo_name)
suspect.tags['spf'] = result
if result != 'none':
self.logger.info(
'SPF client=%s, sender=%s, h=%s result=%s : %s' %
(client_address, sender_email, helo_name, result, explanation))
action = DUNNO
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
dict(result=result, explanation=explanation))
configopt = 'on_%s' % result
if self.config.has_option(self.section, configopt):
action = string_to_actioncode(
self.config.get(self.section, configopt))
return action, message
def lint(self):
lint_ok = True
if not HAVE_SPF:
print 'pyspf or pydns module not installed - this plugin will do nothing'
lint_ok = False
if not HAVE_NETADDR:
print 'WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation'
if not self.checkConfig():
print 'Error checking config'
lint_ok = False
selective_sender_domain_file = self.config.get(
self.section, 'domain_selective_spf_file', '').strip()
if selective_sender_domain_file != '' and not os.path.exists(
selective_sender_domain_file):
print "domain_selective_spf_file %s does not exist" % selective_sender_domain_file
lint_ok = False
ip_whitelist_file = self.config.get(self.section, 'ip_whitelist_file',
'').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
print "ip_whitelist_file %s does not exist - IP whitelist is disabled" % ip_whitelist_file
lint_ok = False
sqlquery = self.config.get(self.section, 'domain_sql_query')
dbconnection = self.config.get(self.section, 'dbconnection',
'').strip()
if not SQLALCHEMY_AVAILABLE and dbconnection != '':
print 'SQLAlchemy not available, cannot use SQL backend'
lint_ok = False
elif dbconnection == '':
print 'No DB connection defined. Disabling SQL backend'
else:
if not sqlquery.lower().startswith('select '):
lint_ok = False
print 'SQL statement must be a SELECT query'
if lint_ok:
try:
conn = get_session(dbconnection)
conn.execute(sqlquery, {'domain': 'example.com'})
except Exception as e:
lint_ok = False
print str(e)
return lint_ok
def __str__(self):
return "SPF"
class EBLLookup(ScannerPlugin):
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.whitelist = None
self.requiredvars={
'whitelist_file':{
'default':'/etc/postomaat/conf.d/ebl-whitelist.txt',
'description':'path to file containing whitelisted sender domains',
},
'dnszone':{
'default':'ebl.msbl.org',
'description':'the DNS zone to query. defaults to ebl.msbl.org',
},
'hash': {
'default':'sha1',
'description':'hash function used by DNS zone. Use one of sha1, md5'
},
'response':{
'default':'127.0.0.2',
'description':'expected response of zone query',
},
'messagetemplate':{
'default':'${sender} listed by ${dnszone} for ${message}'
},
'normalisation':{
'default':'ebl',
'description':'type of normalisation to be applied to email addresses before hashing. choose one of ebl (full normalisation according to ebl.msbl.org standard), low (lowercase only)'
},
'decode_srs':{
'default':'0',
'description':'decode SRS encoded sender addresses before lookup'
},
'check_srs_only':{
'default':'0',
'description':'only check decoded SRS sender addresses against the blacklist zone'
},
}
def _is_whitelisted(self, from_domain):
whitelist_file = self.config.get(self.section,'whitelist_file').strip()
if whitelist_file == '':
return False
whitelisted = False
self.whitelist = FileList(whitelist_file,lowercase=True)
if from_domain in self.whitelist.get_list():
whitelisted = True
return whitelisted
def _email_normalise_ebl(self, address):
if not '@' in address:
self.logger.error('Not an email address: %s' % address)
return address
address = address.lower()
lhs, domain = address.rsplit('@',1)
domainparts = domain.split('.')
if 'googlemail' in domainparts: # replace googlemail with gmail
tld = '.'.join(domainparts[1:])
domain = 'gmail.%s' % tld
domainparts = ['gmail', tld]
if '+' in lhs: # strip all + tags
lhs = lhs.split('+')[0]
if 'gmail' in domainparts: # discard periods in gmail
lhs = lhs.replace('.', '')
if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo
lhs = lhs.split('-')[0]
lhs = re.sub('^(envelope-from|id|r|receiver)=', '', lhs) # strip mail log prefixes
return '%s@%s' % (lhs, domain)
def _email_normalise_low(self, address):
address = address.lower()
return address
def _email_normalise(self, address):
n = self.config.get(self.section,'normalisation')
if n == 'ebl':
address = self._email_normalise_ebl(address)
elif n == 'low':
address = self._email_normalise_low(address)
return address
def _create_hash(self, value):
hashtype = self.config.get(self.section,'hash').lower()
if hashtype == 'sha1':
myhash = sha1(value.encode('utf-8')).hexdigest()
elif hashtype == 'md5':
myhash = md5(value).hexdigest()
else:
myhash = ''
return myhash
def _ebl_lookup(self, addr_hash):
listed = False
message = None
dnszone = self.config.get(self.section,'dnszone').strip()
response = self.config.get(self.section,'response').strip()
query = '%s.%s' % (addr_hash, dnszone)
result = lookup(query)
if result is not None:
for rec in result:
if rec == response:
listed = True
result = lookup(query, qtype=QTYPE_TXT)
if result:
message = result[0]
break
return listed, message
def _is_srs(self, addr):
if addr.startswith('SRS0=') or addr.startswith('SRS1='):
return True
return False
def _decode_srs(self, addr):
srs = SRSDecode()
return srs.reverse(addr)
def examine(self, suspect):
if not DNSQUERY_EXTENSION_ENABLED:
return DUNNO
from_address=suspect.get_value('sender')
if from_address is None:
self.logger.warning('No FROM address found')
return DEFER_IF_PERMIT,'internal policy error (no from address)'
from_address=strip_address(from_address)
if self.config.getboolean(self.section,'check_srs_only') and not self._is_srs(from_address):
self.logger.info('skipping non SRS address %s' % from_address)
return DUNNO
if HAVE_SRS and self.config.getboolean(self.section,'decode_srs'):
from_address = self._decode_srs(from_address)
from_domain=extract_domain(from_address)
if self._is_whitelisted(from_domain):
return DUNNO
from_address = self._email_normalise(from_address)
addr_hash = self._create_hash(from_address)
listed, message = self._ebl_lookup(addr_hash)
if listed:
values = {
'dnszone': self.config.get(self.section,'dnszone','').strip(),
'message': message,
}
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect, values)
return REJECT, message
else:
return DUNNO
def lint(self):
dnszone = self.config.get(self.section,'dnszone','').strip()
print('querying zone %s' % dnszone)
lint_ok = True
if not self.checkConfig():
print('Error checking config')
lint_ok = False
if not DNSQUERY_EXTENSION_ENABLED:
print("no DNS resolver library available - this plugin will do nothing")
lint_ok = False
if self.config.getboolean(self.section,'decode_srs') and not HAVE_SRS:
print('decode_srs enabled but SRS library is not available')
lint_ok = False
hashtype = self.config.get(self.section,'hash').lower()
if hashtype not in ['sha1', 'md5']:
lint_ok = False
print('unsupported hash type %s' % hashtype)
normalisation = self.config.get(self.section,'normalisation')
if normalisation not in ['ebl', 'low']:
lint_ok = False
print('unsupported normalisation type %s' % normalisation)
addr_hash = self._create_hash('*****@*****.**')
listed, message = self._ebl_lookup(addr_hash)
if not listed:
lint_ok = False
print('test entry not found in dns zone')
else:
print('test entry found in dns zone: %s' % message)
if lint_ok:
whitelist_file = self.config.get(self.section,'whitelist_file')
if whitelist_file.strip() == '':
print('No whitelist defined')
return lint_ok
class EBLLookup(ScannerPlugin):
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.whitelist = None
self.requiredvars = {
'whitelist_file': {
'default':
'/etc/postomaat/conf.d/ebl-whitelist.txt',
'description':
'path to file containing whitelisted sender domains',
},
'dnszone': {
'default': 'ebl.msbl.org',
'description':
'the DNS zone to query. defaults to ebl.msbl.org',
},
'response': {
'default': '127.0.0.2',
'description': 'expected response of zone query',
},
'messagetemplate': {
'default': '${sender} listed by ${dnszone} for ${message}'
}
}
def _is_whitelisted(self, from_domain):
whitelist_file = self.config.get(self.section, 'whitelist_file',
'').strip()
if whitelist_file == '':
return False
whitelisted = False
self.whitelist = FileList(whitelist_file, lowercase=True)
if from_domain in self.whitelist.get_list():
whitelisted = True
return whitelisted
def _email_normalise(self, address):
if not '@' in address:
self.logger.error('Not an email address: %s' % address)
return address
address = address.lower()
lhs, domain = address.split('@', 1)
domainparts = domain.split('.')
if 'googlemail' in domainparts: # replace googlemail with gmail
tld = domainparts.split('.', 1)
domainparts = 'gmail.%s' % tld
if '+' in lhs: # strip all + tags
lhs = lhs.split('+')[0]
if 'gmail' in domainparts: # discard periods in gmail
lhs = lhs.replace('.', '')
if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo
lhs = lhs.split('-')[0]
lhs = re.sub('^(envelope-from|id|r|receiver)=', '',
lhs) # strip mail log prefixes
return '%s@%s' % (lhs, domain)
def _create_hash(self, value):
myhash = sha1(value).hexdigest()
return myhash
def _ebl_lookup(self, addr_hash):
listed = False
message = None
dnszone = self.config.get(self.section, 'dnszone', '').strip()
response = self.config.get(self.section, 'response', '').strip()
query = '%s.%s' % (addr_hash, dnszone)
result = lookup(query)
if result is not None:
for rec in result:
if rec == response:
listed = True
result = lookup(query, qtype='TXT')
if result:
message = result[0]
break
return listed, message
def examine(self, suspect):
if not HAVE_DNS:
return DUNNO
from_address = suspect.get_value('sender')
if from_address is None:
self.logger.warning('No FROM address found')
return DEFER_IF_PERMIT, 'internal policy error (no from address)'
from_address = strip_address(from_address)
from_domain = extract_domain(from_address)
if self._is_whitelisted(from_domain):
return DUNNO
from_address = self._email_normalise(from_address)
addr_hash = self._create_hash(from_address)
listed, message = self._ebl_lookup(addr_hash)
if listed:
values = {
'dnszone': self.config.get(self.section, 'dnszone',
'').strip(),
'message': message,
}
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
values)
return REJECT, message
else:
return DUNNO
def lint(self):
lint_ok = True
if not self.checkConfig():
print 'Error checking config'
lint_ok = False
if not HAVE_DNS:
print "no DNS resolver library available - this plugin will do nothing"
lint_ok = False
dnszone = self.config.get(self.section, 'dnszone', '').strip()
print 'querying zone %s' % dnszone
addr_hash = self._create_hash('*****@*****.**')
listed, message = self._ebl_lookup(addr_hash)
if not listed:
lint_ok = False
print 'test entry not found in dns zone'
else:
print 'test entry found in dns zone: %s' % message
if lint_ok:
whitelist_file = self.config.get(self.section, 'whitelist_file')
if whitelist_file.strip() == '':
print 'No whitelist defined'
return lint_ok
class SPFPlugin(ScannerPlugin):
"""This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/
by default, it just logs the result (test mode)
to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg:
on_fail = REJECT
valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral'
"""
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.logger=self._logger()
self.requiredvars={
'ip_whitelist_file':{
'default':'',
'description':'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted',
},
'domain_selective_spf_file':{
'default':'',
'description':'if this is non-empty, only sender domains in this file will be checked for SPF',
},
'dbconnection':{
'default':"mysql://root@localhost/spfcheck?charset=utf8",
'description':'SQLAlchemy Connection string. Leave empty to disable SQL lookups',
},
'domain_sql_query':{
'default':"SELECT check_spf from domain where domain_name=:domain",
'description':'get from sql database :domain will be replaced with the actual domain name. must return field check_spf',
},
'on_fail':{
'default':'DUNNO',
'description':'Action for SPF fail.',
},
'on_softfail':{
'default':'DUNNO',
'description':'Action for SPF softfail.',
},
'messagetemplate':{
'default':'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}'
}
}
if HAVE_NETADDR:
self.private_nets = [
IPNetwork('10.0.0.0/8'), # private network
IPNetwork('127.0.0.0/8'), # localhost
IPNetwork('169.254.0.0/16'), # link local
IPNetwork('172.16.0.0/12'), # private network
IPNetwork('192.168.0.0/16'), # private network
IPNetwork('fe80::/10'), # ipv6 link local
IPNetwork('::1/128'), # localhost
]
else:
self.private_nets = None
self.ip_whitelist_loader=None
self.ip_whitelist=[] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available
self.selective_domain_loader=None
def check_this_domain(self, from_domain):
do_check = False
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file','').strip()
if selective_sender_domain_file != '' and os.path.exists(selective_sender_domain_file):
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True)
if from_domain.lower() in self.selective_domain_loader.get_list():
do_check = True
if not do_check:
dbconnection = self.config.get(self.section, 'dbconnection', '').strip()
sqlquery = self.config.get(self.section, 'domain_sql_query')
if dbconnection!='' and SQL_EXTENSION_ENABLED:
cache = get_default_cache()
do_check = get_domain_setting(from_domain, dbconnection, sqlquery, cache, self.section, False, self.logger)
elif dbconnection!='' and not SQL_EXTENSION_ENABLED:
self.logger.error('dbconnection specified but sqlalchemy not available - skipping db lookup')
return do_check
def is_private_address(self,addr):
if HAVE_NETADDR:
ipaddr = IPAddress(addr)
private = False
for net in self.private_nets:
if ipaddr in net:
private = True
break
return private
else:
if addr=='127.0.0.1' or addr=='::1' or addr.startswith('10.') or addr.startswith('192.168.') or addr.startswith('fe80:'):
return True
if not addr.startswith('172.'):
return False
for i in range(16,32):
if addr.startswith('172.%s'%i):
return True
return False
def ip_whitelisted(self,addr):
if self.is_private_address(addr):
return True
#check ip whitelist
ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file', '').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
plainlist = []
if self.ip_whitelist_loader is None:
self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True)
if self.ip_whitelist_loader.file_changed():
plainlist=self.ip_whitelist_loader.get_list()
if HAVE_NETADDR:
self.ip_whitelist=[IPNetwork(x) for x in plainlist]
else:
self.ip_whitelist=plainlist
if HAVE_NETADDR:
checkaddr=IPAddress(addr)
for net in self.ip_whitelist:
if checkaddr in net:
return True
else:
if addr in plainlist:
return True
return False
def examine(self,suspect):
if not HAVE_SPF:
return DUNNO
client_address=suspect.get_value('client_address')
helo_name=suspect.get_value('helo_name')
sender=suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check"%client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email=='' or sender_email is None:
return DUNNO
sender_domain = extract_domain(sender_email)
if sender_domain is None:
self.logger.error('no domain found in sender address %s' % sender_email)
return DUNNO
if not self.check_this_domain(sender_domain):
self.logger.debug('skipping SPF check for %s' % sender_domain)
return DUNNO
result, explanation = spf.check2(client_address, sender_email, helo_name)
suspect.tags['spf'] = result
if result != 'none':
self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation))
action = DUNNO
message = apply_template(self.config.get(self.section, 'messagetemplate'), suspect, dict(result=result, explanation=explanation))
configopt = 'on_%s' % result
if self.config.has_option(self.section, configopt):
action=string_to_actioncode(self.config.get(self.section, configopt))
return action, message
def lint(self):
lint_ok = True
if not HAVE_SPF:
print('pyspf or pydns module not installed - this plugin will do nothing')
lint_ok = False
if not HAVE_NETADDR:
print('WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation')
if not self.checkConfig():
print('Error checking config')
lint_ok = False
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file','').strip()
if selective_sender_domain_file != '' and not os.path.exists(selective_sender_domain_file):
print("domain_selective_spf_file %s does not exist" % selective_sender_domain_file)
lint_ok = False
ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file', '').strip()
if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file):
print("ip_whitelist_file %s does not exist - IP whitelist is disabled" % ip_whitelist_file)
lint_ok = False
sqlquery = self.config.get(self.section, 'domain_sql_query')
dbconnection = self.config.get(self.section, 'dbconnection', '').strip()
if not SQL_EXTENSION_ENABLED and dbconnection != '':
print('SQLAlchemy not available, cannot use SQL backend')
lint_ok = False
elif dbconnection == '':
print('No DB connection defined. Disabling SQL backend')
else:
if not sqlquery.lower().startswith('select '):
lint_ok = False
print('SQL statement must be a SELECT query')
if lint_ok:
try:
conn=get_session(dbconnection)
conn.execute(sqlquery, {'domain':'example.com'})
except Exception as e:
lint_ok = False
print(str(e))
return lint_ok
def __str__(self):
return "SPF"
class EBLLookup(ScannerPlugin):
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.whitelist = None
self.requiredvars = {
'whitelist_file': {
'default':
'/etc/postomaat/conf.d/ebl-whitelist.txt',
'description':
'path to file containing whitelisted sender domains',
},
'dnszone': {
'default': 'ebl.msbl.org',
'description':
'the DNS zone to query. defaults to ebl.msbl.org',
},
'hash': {
'default':
'sha1',
'description':
'hash function used by DNS zone. Use one of sha1, md5'
},
'response': {
'default': '127.0.0.2',
'description': 'expected response of zone query',
},
'messagetemplate': {
'default': '${sender} listed by ${dnszone} for ${message}'
},
'normalisation': {
'default':
'ebl',
'description':
'type of normalisation to be applied to email addresses before hashing. choose one of ebl (full normalisation according to ebl.msbl.org standard), low (lowercase only)'
},
'decode_srs': {
'default': '0',
'description':
'decode SRS encoded sender addresses before lookup'
},
'check_srs_only': {
'default':
'0',
'description':
'only check decoded SRS sender addresses against the blacklist zone'
},
}
def _is_whitelisted(self, from_domain):
whitelist_file = self.config.get(self.section,
'whitelist_file').strip()
if whitelist_file == '':
return False
whitelisted = False
self.whitelist = FileList(whitelist_file, lowercase=True)
if from_domain in self.whitelist.get_list():
whitelisted = True
return whitelisted
def _email_normalise_ebl(self, address):
if not '@' in address:
self.logger.error('Not an email address: %s' % address)
return address
address = address.lower()
lhs, domain = address.rsplit('@', 1)
domainparts = domain.split('.')
if 'googlemail' in domainparts: # replace googlemail with gmail
tld = '.'.join(domainparts[1:])
domain = 'gmail.%s' % tld
domainparts = ['gmail', tld]
if '+' in lhs: # strip all + tags
lhs = lhs.split('+')[0]
if 'gmail' in domainparts: # discard periods in gmail
lhs = lhs.replace('.', '')
if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo
lhs = lhs.split('-')[0]
lhs = re.sub('^(envelope-from|id|r|receiver)=', '',
lhs) # strip mail log prefixes
return '%s@%s' % (lhs, domain)
def _email_normalise_low(self, address):
address = address.lower()
return address
def _email_normalise(self, address):
n = self.config.get(self.section, 'normalisation')
if n == 'ebl':
address = self._email_normalise_ebl(address)
elif n == 'low':
address = self._email_normalise_low(address)
return address
def _create_hash(self, value):
hashtype = self.config.get(self.section, 'hash').lower()
if hashtype == 'sha1':
myhash = sha1(value.encode('utf-8')).hexdigest()
elif hashtype == 'md5':
myhash = md5(value).hexdigest()
else:
myhash = ''
return myhash
def _ebl_lookup(self, addr_hash):
listed = False
message = None
dnszone = self.config.get(self.section, 'dnszone').strip()
response = self.config.get(self.section, 'response').strip()
query = '%s.%s' % (addr_hash, dnszone)
result = lookup(query)
if result is not None:
for rec in result:
if rec == response:
listed = True
result = lookup(query, qtype=QTYPE_TXT)
if result:
message = result[0]
break
return listed, message
def _is_srs(self, addr):
if addr.startswith('SRS0=') or addr.startswith('SRS1='):
return True
return False
def _decode_srs(self, addr):
srs = SRSDecode()
return srs.reverse(addr)
def examine(self, suspect):
if not DNSQUERY_EXTENSION_ENABLED:
return DUNNO
from_address = suspect.get_value('sender')
if from_address is None:
self.logger.warning('No FROM address found')
return DEFER_IF_PERMIT, 'internal policy error (no from address)'
from_address = strip_address(from_address)
if self.config.getboolean(
self.section,
'check_srs_only') and not self._is_srs(from_address):
self.logger.info('skipping non SRS address %s' % from_address)
return DUNNO
if HAVE_SRS and self.config.getboolean(self.section, 'decode_srs'):
from_address = self._decode_srs(from_address)
from_domain = extract_domain(from_address)
if self._is_whitelisted(from_domain):
return DUNNO
from_address = self._email_normalise(from_address)
addr_hash = self._create_hash(from_address)
listed, message = self._ebl_lookup(addr_hash)
if listed:
values = {
'dnszone': self.config.get(self.section, 'dnszone',
'').strip(),
'message': message,
}
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
values)
return REJECT, message
else:
return DUNNO
def lint(self):
dnszone = self.config.get(self.section, 'dnszone', '').strip()
print('querying zone %s' % dnszone)
lint_ok = True
if not self.checkConfig():
print('Error checking config')
lint_ok = False
if not DNSQUERY_EXTENSION_ENABLED:
print(
"no DNS resolver library available - this plugin will do nothing"
)
lint_ok = False
if self.config.getboolean(self.section, 'decode_srs') and not HAVE_SRS:
print('decode_srs enabled but SRS library is not available')
lint_ok = False
hashtype = self.config.get(self.section, 'hash').lower()
if hashtype not in ['sha1', 'md5']:
lint_ok = False
print('unsupported hash type %s' % hashtype)
normalisation = self.config.get(self.section, 'normalisation')
if normalisation not in ['ebl', 'low']:
lint_ok = False
print('unsupported normalisation type %s' % normalisation)
addr_hash = self._create_hash('*****@*****.**')
listed, message = self._ebl_lookup(addr_hash)
if not listed:
lint_ok = False
print('test entry not found in dns zone')
else:
print('test entry found in dns zone: %s' % message)
if lint_ok:
whitelist_file = self.config.get(self.section, 'whitelist_file')
if whitelist_file.strip() == '':
print('No whitelist defined')
return lint_ok
