class CreativeTLD(ScannerPlugin): """ Reject clients with unofficial TLD in rdns """ def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.requiredvars={ 'action':{ 'default':'REJECT', 'description':'Action if sender uses invalid TLD', }, 'message':{ 'default':'forged rDNS TLD', }, 'tldfile':{ 'default':'/etc/mail/tlds-alpha-by-domain.txt', }, } self.filelist=FileList(filename=None,strip=True, skip_empty=True, skip_comments=True,lowercase=True,minimum_time_between_reloads=86400) def examine(self,suspect): retaction,retmessage = DUNNO,'' revclient=suspect.get_value('reverse_client_name') self.filelist.filename=self.config.get(self.section,'domainsfile') tlds = self.filelist.get_list() if revclient is None or revclient.strip()=='unknown' or '.' not in revclient: return DUNNO,'' tld=revclient.split('.')[-1].lower() if tld not in tlds: retaction=self.config.get(self.section,'action').strip() retmessage=self.config.get(self.section,'message').strip() return retaction,retmessage def lint(self): lint_ok=True retaction=self.config.get(self.section,'action').strip().lower() reasonable_actions=[REJECT,DEFER,DEFER_IF_PERMIT,FILTER,HOLD,PREPEND,WARN] if retaction not in reasonable_actions: print "are you sure about action '%s' ?"%retaction print "I'd expect one of %s"%(",".join(reasonable_actions)) lint_ok=False if not self.checkConfig(): print 'Error checking config' lint_ok = False return lint_ok def __str__(self): return "Creative TLD"
class HELOTLDPlugin(ScannerPlugin): """ This plugin rejects messages if the HELO uses an invalid TLD """ def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.requiredvars={ 'tldfile':{ 'default':'/etc/mail/tlds-alpha-by-domain.txt', 'description':'filename containing official TLDs. Add a cronjob to dowload this.', }, 'exceptionfile':{ 'default':'/etc/mail/tlds-exceptions.txt', 'description':'additional tld file with local exceptions', }, 'on_fail':{ 'default':'REJECT', 'description':'Action to take if the TLD is invalid', }, 'messagetemplate':{ 'default':"""HELO ${helo_name} contains forged/unresolvable TLD '.${helo_tld}'""" } } self.tld_loader=None self.exception_loader=None def examine(self,suspect): helo_name=suspect.get_value('helo_name') if helo_name is None : self.logger.error('missing helo') return DUNNO helo_tld=helo_name.split('.')[-1].lower() #initialize loaders tld_file=self.config.get(self.section,'tldfile') if self.tld_loader is None: self.tld_loader=FileList(tld_file,lowercase=True,minimum_time_between_reloads=3600) if helo_tld in self.tld_loader.get_list(): return DUNNO,'' exceptionfile=self.config.get(self.section,'exceptionfile') if self.exception_loader is None: self.exception_loader=FileList(exceptionfile,lowercase=True,minimum_time_between_reloads=10) if helo_tld in self.exception_loader.get_list(): return DUNNO,'' message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(helo_tld=helo_tld)) action=self.config.get(self.section,"on_fail") return action, message def lint(self): lint_ok = True tld_file=self.config.get(self.section,'tldfile') exceptionfile=self.config.get(self.section,'exceptionfile') if not os.path.exists(tld_file): print("TLD file %s not found"%tld_file) lint_ok = False if not os.path.exists(exceptionfile): print("TLD exception file %s not found"%exceptionfile) lint_ok = False if not self.checkConfig(): print('Error checking config') lint_ok = False return lint_ok def __str__(self): return "HeloTLD"
class HELOTLDPlugin(ScannerPlugin): """ This plugin rejects messages if the HELO uses an invalid TLD """ def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.requiredvars = { 'tldfile': { 'default': '/etc/mail/tlds-alpha-by-domain.txt', 'description': 'filename containing official TLDs. Add a cronjob to dowload this.', }, 'exceptionfile': { 'default': '/etc/mail/tlds-exceptions.txt', 'description': 'additional tld file with local exceptions', }, 'on_fail': { 'default': 'REJECT', 'description': 'Action to take if the TLD is invalid', }, 'messagetemplate': { 'default': """HELO ${helo_name} contains forged/unresolvable TLD '.${helo_tld}'""" } } self.tld_loader = None self.exception_loader = None def examine(self, suspect): helo_name = suspect.get_value('helo_name') if helo_name is None: self.logger.error('missing helo') return DUNNO helo_tld = helo_name.split('.')[-1].lower() #initialize loaders tld_file = self.config.get(self.section, 'tldfile') if self.tld_loader is None: self.tld_loader = FileList(tld_file, lowercase=True, minimum_time_between_reloads=3600) if helo_tld in self.tld_loader.get_list(): return DUNNO, '' exceptionfile = self.config.get(self.section, 'exceptionfile') if self.exception_loader is None: self.exception_loader = FileList(exceptionfile, lowercase=True, minimum_time_between_reloads=10) if helo_tld in self.exception_loader.get_list(): return DUNNO, '' message = apply_template( self.config.get(self.section, 'messagetemplate'), suspect, dict(helo_tld=helo_tld)) action = self.config.get(self.section, "on_fail") return action, message def lint(self): lint_ok = True tld_file = self.config.get(self.section, 'tldfile') exceptionfile = self.config.get(self.section, 'exceptionfile') if not os.path.exists(tld_file): print("TLD file %s not found" % tld_file) lint_ok = False if not os.path.exists(exceptionfile): print("TLD exception file %s not found" % exceptionfile) lint_ok = False if not self.checkConfig(): print('Error checking config') lint_ok = False return lint_ok def __str__(self): return "HeloTLD"
class CreativeTLD(ScannerPlugin): """ Reject clients with unofficial TLD in rdns """ def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.requiredvars = { 'action': { 'default': 'REJECT', 'description': 'Action if sender uses invalid TLD', }, 'message': { 'default': 'forged rDNS TLD', }, 'tldfile': { 'default': '/etc/mail/tlds-alpha-by-domain.txt', }, } self.filelist = FileList(filename=None, strip=True, skip_empty=True, skip_comments=True, lowercase=True, minimum_time_between_reloads=86400) def examine(self, suspect): retaction, retmessage = DUNNO, '' revclient = suspect.get_value('reverse_client_name') self.filelist.filename = self.config.get(self.section, 'domainsfile') tlds = self.filelist.get_list() if revclient is None or revclient.strip( ) == 'unknown' or '.' not in revclient: return DUNNO, '' tld = revclient.split('.')[-1].lower() if tld not in tlds: retaction = self.config.get(self.section, 'action').strip() retmessage = self.config.get(self.section, 'message').strip() return retaction, retmessage def lint(self): lint_ok = True retaction = self.config.get(self.section, 'action').strip().lower() reasonable_actions = [ REJECT, DEFER, DEFER_IF_PERMIT, FILTER, HOLD, PREPEND, WARN ] if retaction not in reasonable_actions: print("are you sure about action '%s' ?" % retaction) print("I'd expect one of %s" % (",".join(reasonable_actions))) lint_ok = False if not self.checkConfig(): print('Error checking config') lint_ok = False return lint_ok def __str__(self): return "Creative TLD"
class SPFPlugin(ScannerPlugin): """This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/ by default, it just logs the result (test mode) to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg: on_fail = REJECT valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral' """ def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.requiredvars = { 'ip_whitelist_file': { 'default': '', 'description': 'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted', }, 'domain_selective_spf_file': { 'default': '', 'description': 'if this is non-empty, only sender domains in this file will be checked for SPF', }, 'dbconnection': { 'default': "mysql://root@localhost/spfcheck?charset=utf8", 'description': 'SQLAlchemy Connection string. Leave empty to disable SQL lookups', }, 'domain_sql_query': { 'default': "SELECT check_spf from domain where domain_name=:domain", 'description': 'get from sql database :domain will be replaced with the actual domain name. must return field check_spf', }, 'on_fail': { 'default': 'DUNNO', 'description': 'Action for SPF fail.', }, 'on_softfail': { 'default': 'DUNNO', 'description': 'Action for SPF softfail.', }, 'messagetemplate': { 'default': 'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}' } } self.ip_whitelist_loader = None self.ip_whitelist = [ ] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available self.selective_domain_loader = None def check_this_domain(self, from_domain): global SETTINGSCACHE do_check = False selective_sender_domain_file = self.config.get( self.section, 'domain_selective_spf_file', '').strip() if selective_sender_domain_file != '' and os.path.exists( selective_sender_domain_file): if self.selective_domain_loader is None: self.selective_domain_loader = FileList( selective_sender_domain_file, lowercase=True) if from_domain.lower() in self.selective_domain_loader.get_list(): do_check = True if not do_check: dbconnection = self.config.get(self.section, 'dbconnection', '').strip() sqlquery = self.config.get(self.section, 'domain_sql_query') if dbconnection != '' and SQLALCHEMY_AVAILABLE: if SETTINGSCACHE is None: SETTINGSCACHE = SettingsCache() do_check = get_domain_setting(from_domain, dbconnection, sqlquery, SETTINGSCACHE, False, self.logger) elif dbconnection != '' and not SQLALCHEMY_AVAILABLE: self.logger.error( 'dbconnection specified but sqlalchemy not available - skipping db lookup' ) return do_check def is_private_address(self, addr): if addr == '127.0.0.1' or addr == '::1' or addr.startswith( '10.') or addr.startswith('192.168.') or addr.startswith( 'fe80:'): return True if not addr.startswith('172.'): return False for i in range(16, 32): if addr.startswith('172.%s' % i): return True return False def ip_whitelisted(self, addr): if self.is_private_address(addr): return True #check ip whitelist ip_whitelist_file = self.config.get(self.section, 'ip_whitelist_file', '').strip() if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file): plainlist = [] if self.ip_whitelist_loader is None: self.ip_whitelist_loader = FileList(ip_whitelist_file, lowercase=True) if self.ip_whitelist_loader.file_changed(): plainlist = self.ip_whitelist_loader.get_list() if HAVE_NETADDR: self.ip_whitelist = [IPNetwork(x) for x in plainlist] else: self.ip_whitelist = plainlist if HAVE_NETADDR: checkaddr = IPAddress(addr) for net in self.ip_whitelist: if checkaddr in net: return True else: if addr in plainlist: return True return False def examine(self, suspect): if not HAVE_SPF: return DUNNO client_address = suspect.get_value('client_address') helo_name = suspect.get_value('helo_name') sender = suspect.get_value('sender') if client_address is None or helo_name is None or sender is None: self.logger.error('missing client_address or helo or sender') return DUNNO if self.ip_whitelisted(client_address): self.logger.info("Client %s is whitelisted - no SPF check" % client_address) return DUNNO sender_email = strip_address(sender) if sender_email == '' or sender_email is None: return DUNNO sender_domain = extract_domain(sender_email) if sender_domain is None: self.logger.error('no domain found in sender address %s' % sender_email) return DUNNO if not self.check_this_domain(sender_domain): self.logger.debug('skipping SPF check for %s' % sender_domain) return DUNNO result, explanation = spf.check2(client_address, sender_email, helo_name) suspect.tags['spf'] = result if result != 'none': self.logger.info( 'SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result, explanation)) action = DUNNO message = apply_template( self.config.get(self.section, 'messagetemplate'), suspect, dict(result=result, explanation=explanation)) configopt = 'on_%s' % result if self.config.has_option(self.section, configopt): action = string_to_actioncode( self.config.get(self.section, configopt)) return action, message def lint(self): lint_ok = True if not HAVE_SPF: print 'pyspf or pydns module not installed - this plugin will do nothing' lint_ok = False if not HAVE_NETADDR: print 'WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation' if not self.checkConfig(): print 'Error checking config' lint_ok = False selective_sender_domain_file = self.config.get( self.section, 'domain_selective_spf_file', '').strip() if selective_sender_domain_file != '' and not os.path.exists( selective_sender_domain_file): print "domain_selective_spf_file %s does not exist" % selective_sender_domain_file lint_ok = False ip_whitelist_file = self.config.get(self.section, 'ip_whitelist_file', '').strip() if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file): print "ip_whitelist_file %s does not exist - IP whitelist is disabled" % ip_whitelist_file lint_ok = False sqlquery = self.config.get(self.section, 'domain_sql_query') dbconnection = self.config.get(self.section, 'dbconnection', '').strip() if not SQLALCHEMY_AVAILABLE and dbconnection != '': print 'SQLAlchemy not available, cannot use SQL backend' lint_ok = False elif dbconnection == '': print 'No DB connection defined. Disabling SQL backend' else: if not sqlquery.lower().startswith('select '): lint_ok = False print 'SQL statement must be a SELECT query' if lint_ok: try: conn = get_session(dbconnection) conn.execute(sqlquery, {'domain': 'example.com'}) except Exception as e: lint_ok = False print str(e) return lint_ok def __str__(self): return "SPF"
class EBLLookup(ScannerPlugin): def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.whitelist = None self.requiredvars = { 'whitelist_file': { 'default': '/etc/postomaat/conf.d/ebl-whitelist.txt', 'description': 'path to file containing whitelisted sender domains', }, 'dnszone': { 'default': 'ebl.msbl.org', 'description': 'the DNS zone to query. defaults to ebl.msbl.org', }, 'response': { 'default': '127.0.0.2', 'description': 'expected response of zone query', }, 'messagetemplate': { 'default': '${sender} listed by ${dnszone} for ${message}' } } def _is_whitelisted(self, from_domain): whitelist_file = self.config.get(self.section, 'whitelist_file', '').strip() if whitelist_file == '': return False whitelisted = False self.whitelist = FileList(whitelist_file, lowercase=True) if from_domain in self.whitelist.get_list(): whitelisted = True return whitelisted def _email_normalise(self, address): if not '@' in address: self.logger.error('Not an email address: %s' % address) return address address = address.lower() lhs, domain = address.split('@', 1) domainparts = domain.split('.') if 'googlemail' in domainparts: # replace googlemail with gmail tld = domainparts.split('.', 1) domainparts = 'gmail.%s' % tld if '+' in lhs: # strip all + tags lhs = lhs.split('+')[0] if 'gmail' in domainparts: # discard periods in gmail lhs = lhs.replace('.', '') if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo lhs = lhs.split('-')[0] lhs = re.sub('^(envelope-from|id|r|receiver)=', '', lhs) # strip mail log prefixes return '%s@%s' % (lhs, domain) def _create_hash(self, value): myhash = sha1(value).hexdigest() return myhash def _ebl_lookup(self, addr_hash): listed = False message = None dnszone = self.config.get(self.section, 'dnszone', '').strip() response = self.config.get(self.section, 'response', '').strip() query = '%s.%s' % (addr_hash, dnszone) result = lookup(query) if result is not None: for rec in result: if rec == response: listed = True result = lookup(query, qtype='TXT') if result: message = result[0] break return listed, message def examine(self, suspect): if not HAVE_DNS: return DUNNO from_address = suspect.get_value('sender') if from_address is None: self.logger.warning('No FROM address found') return DEFER_IF_PERMIT, 'internal policy error (no from address)' from_address = strip_address(from_address) from_domain = extract_domain(from_address) if self._is_whitelisted(from_domain): return DUNNO from_address = self._email_normalise(from_address) addr_hash = self._create_hash(from_address) listed, message = self._ebl_lookup(addr_hash) if listed: values = { 'dnszone': self.config.get(self.section, 'dnszone', '').strip(), 'message': message, } message = apply_template( self.config.get(self.section, 'messagetemplate'), suspect, values) return REJECT, message else: return DUNNO def lint(self): lint_ok = True if not self.checkConfig(): print 'Error checking config' lint_ok = False if not HAVE_DNS: print "no DNS resolver library available - this plugin will do nothing" lint_ok = False dnszone = self.config.get(self.section, 'dnszone', '').strip() print 'querying zone %s' % dnszone addr_hash = self._create_hash('*****@*****.**') listed, message = self._ebl_lookup(addr_hash) if not listed: lint_ok = False print 'test entry not found in dns zone' else: print 'test entry found in dns zone: %s' % message if lint_ok: whitelist_file = self.config.get(self.section, 'whitelist_file') if whitelist_file.strip() == '': print 'No whitelist defined' return lint_ok
class EBLLookup(ScannerPlugin): def __init__(self, config, section=None): ScannerPlugin.__init__(self, config, section) self.logger = self._logger() self.whitelist = None self.requiredvars = { 'whitelist_file': { 'default': '/etc/postomaat/conf.d/ebl-whitelist.txt', 'description': 'path to file containing whitelisted sender domains', }, 'dnszone': { 'default': 'ebl.msbl.org', 'description': 'the DNS zone to query. defaults to ebl.msbl.org', }, 'hash': { 'default': 'sha1', 'description': 'hash function used by DNS zone. Use one of sha1, md5' }, 'response': { 'default': '127.0.0.2', 'description': 'expected response of zone query', }, 'messagetemplate': { 'default': '${sender} listed by ${dnszone} for ${message}' }, 'normalisation': { 'default': 'ebl', 'description': 'type of normalisation to be applied to email addresses before hashing. choose one of ebl (full normalisation according to ebl.msbl.org standard), low (lowercase only)' }, 'decode_srs': { 'default': '0', 'description': 'decode SRS encoded sender addresses before lookup' }, 'check_srs_only': { 'default': '0', 'description': 'only check decoded SRS sender addresses against the blacklist zone' }, } def _is_whitelisted(self, from_domain): whitelist_file = self.config.get(self.section, 'whitelist_file').strip() if whitelist_file == '': return False whitelisted = False self.whitelist = FileList(whitelist_file, lowercase=True) if from_domain in self.whitelist.get_list(): whitelisted = True return whitelisted def _email_normalise_ebl(self, address): if not '@' in address: self.logger.error('Not an email address: %s' % address) return address address = address.lower() lhs, domain = address.rsplit('@', 1) domainparts = domain.split('.') if 'googlemail' in domainparts: # replace googlemail with gmail tld = '.'.join(domainparts[1:]) domain = 'gmail.%s' % tld domainparts = ['gmail', tld] if '+' in lhs: # strip all + tags lhs = lhs.split('+')[0] if 'gmail' in domainparts: # discard periods in gmail lhs = lhs.replace('.', '') if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo lhs = lhs.split('-')[0] lhs = re.sub('^(envelope-from|id|r|receiver)=', '', lhs) # strip mail log prefixes return '%s@%s' % (lhs, domain) def _email_normalise_low(self, address): address = address.lower() return address def _email_normalise(self, address): n = self.config.get(self.section, 'normalisation') if n == 'ebl': address = self._email_normalise_ebl(address) elif n == 'low': address = self._email_normalise_low(address) return address def _create_hash(self, value): hashtype = self.config.get(self.section, 'hash').lower() if hashtype == 'sha1': myhash = sha1(value.encode('utf-8')).hexdigest() elif hashtype == 'md5': myhash = md5(value).hexdigest() else: myhash = '' return myhash def _ebl_lookup(self, addr_hash): listed = False message = None dnszone = self.config.get(self.section, 'dnszone').strip() response = self.config.get(self.section, 'response').strip() query = '%s.%s' % (addr_hash, dnszone) result = lookup(query) if result is not None: for rec in result: if rec == response: listed = True result = lookup(query, qtype=QTYPE_TXT) if result: message = result[0] break return listed, message def _is_srs(self, addr): if addr.startswith('SRS0=') or addr.startswith('SRS1='): return True return False def _decode_srs(self, addr): srs = SRSDecode() return srs.reverse(addr) def examine(self, suspect): if not DNSQUERY_EXTENSION_ENABLED: return DUNNO from_address = suspect.get_value('sender') if from_address is None: self.logger.warning('No FROM address found') return DEFER_IF_PERMIT, 'internal policy error (no from address)' from_address = strip_address(from_address) if self.config.getboolean( self.section, 'check_srs_only') and not self._is_srs(from_address): self.logger.info('skipping non SRS address %s' % from_address) return DUNNO if HAVE_SRS and self.config.getboolean(self.section, 'decode_srs'): from_address = self._decode_srs(from_address) from_domain = extract_domain(from_address) if self._is_whitelisted(from_domain): return DUNNO from_address = self._email_normalise(from_address) addr_hash = self._create_hash(from_address) listed, message = self._ebl_lookup(addr_hash) if listed: values = { 'dnszone': self.config.get(self.section, 'dnszone', '').strip(), 'message': message, } message = apply_template( self.config.get(self.section, 'messagetemplate'), suspect, values) return REJECT, message else: return DUNNO def lint(self): dnszone = self.config.get(self.section, 'dnszone', '').strip() print('querying zone %s' % dnszone) lint_ok = True if not self.checkConfig(): print('Error checking config') lint_ok = False if not DNSQUERY_EXTENSION_ENABLED: print( "no DNS resolver library available - this plugin will do nothing" ) lint_ok = False if self.config.getboolean(self.section, 'decode_srs') and not HAVE_SRS: print('decode_srs enabled but SRS library is not available') lint_ok = False hashtype = self.config.get(self.section, 'hash').lower() if hashtype not in ['sha1', 'md5']: lint_ok = False print('unsupported hash type %s' % hashtype) normalisation = self.config.get(self.section, 'normalisation') if normalisation not in ['ebl', 'low']: lint_ok = False print('unsupported normalisation type %s' % normalisation) addr_hash = self._create_hash('*****@*****.**') listed, message = self._ebl_lookup(addr_hash) if not listed: lint_ok = False print('test entry not found in dns zone') else: print('test entry found in dns zone: %s' % message) if lint_ok: whitelist_file = self.config.get(self.section, 'whitelist_file') if whitelist_file.strip() == '': print('No whitelist defined') return lint_ok
class SPFPlugin(ScannerPlugin): """This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/ by default, it just logs the result (test mode) to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg: on_fail = REJECT valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral' """ def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.requiredvars={ 'ip_whitelist_file':{ 'default':'', 'description':'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted', }, 'domain_selective_spf_file':{ 'default':'', 'description':'if this is non-empty, only sender domains in this file will be checked for SPF', }, 'dbconnection':{ 'default':"mysql://root@localhost/spfcheck?charset=utf8", 'description':'SQLAlchemy Connection string. Leave empty to disable SQL lookups', }, 'domain_sql_query':{ 'default':"SELECT check_spf from domain where domain_name=:domain", 'description':'get from sql database :domain will be replaced with the actual domain name. must return field check_spf', }, 'on_fail':{ 'default':'DUNNO', 'description':'Action for SPF fail.', }, 'on_softfail':{ 'default':'DUNNO', 'description':'Action for SPF softfail.', }, 'messagetemplate':{ 'default':'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}' } } if HAVE_NETADDR: self.private_nets = [ IPNetwork('10.0.0.0/8'), # private network IPNetwork('127.0.0.0/8'), # localhost IPNetwork('169.254.0.0/16'), # link local IPNetwork('172.16.0.0/12'), # private network IPNetwork('192.168.0.0/16'), # private network IPNetwork('fe80::/10'), # ipv6 link local IPNetwork('::1/128'), # localhost ] else: self.private_nets = None self.ip_whitelist_loader=None self.ip_whitelist=[] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available self.selective_domain_loader=None def check_this_domain(self, from_domain): do_check = False selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file','').strip() if selective_sender_domain_file != '' and os.path.exists(selective_sender_domain_file): if self.selective_domain_loader is None: self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True) if from_domain.lower() in self.selective_domain_loader.get_list(): do_check = True if not do_check: dbconnection = self.config.get(self.section, 'dbconnection', '').strip() sqlquery = self.config.get(self.section, 'domain_sql_query') if dbconnection!='' and SQL_EXTENSION_ENABLED: cache = get_default_cache() do_check = get_domain_setting(from_domain, dbconnection, sqlquery, cache, self.section, False, self.logger) elif dbconnection!='' and not SQL_EXTENSION_ENABLED: self.logger.error('dbconnection specified but sqlalchemy not available - skipping db lookup') return do_check def is_private_address(self,addr): if HAVE_NETADDR: ipaddr = IPAddress(addr) private = False for net in self.private_nets: if ipaddr in net: private = True break return private else: if addr=='127.0.0.1' or addr=='::1' or addr.startswith('10.') or addr.startswith('192.168.') or addr.startswith('fe80:'): return True if not addr.startswith('172.'): return False for i in range(16,32): if addr.startswith('172.%s'%i): return True return False def ip_whitelisted(self,addr): if self.is_private_address(addr): return True #check ip whitelist ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file', '').strip() if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file): plainlist = [] if self.ip_whitelist_loader is None: self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True) if self.ip_whitelist_loader.file_changed(): plainlist=self.ip_whitelist_loader.get_list() if HAVE_NETADDR: self.ip_whitelist=[IPNetwork(x) for x in plainlist] else: self.ip_whitelist=plainlist if HAVE_NETADDR: checkaddr=IPAddress(addr) for net in self.ip_whitelist: if checkaddr in net: return True else: if addr in plainlist: return True return False def examine(self,suspect): if not HAVE_SPF: return DUNNO client_address=suspect.get_value('client_address') helo_name=suspect.get_value('helo_name') sender=suspect.get_value('sender') if client_address is None or helo_name is None or sender is None: self.logger.error('missing client_address or helo or sender') return DUNNO if self.ip_whitelisted(client_address): self.logger.info("Client %s is whitelisted - no SPF check"%client_address) return DUNNO sender_email = strip_address(sender) if sender_email=='' or sender_email is None: return DUNNO sender_domain = extract_domain(sender_email) if sender_domain is None: self.logger.error('no domain found in sender address %s' % sender_email) return DUNNO if not self.check_this_domain(sender_domain): self.logger.debug('skipping SPF check for %s' % sender_domain) return DUNNO result, explanation = spf.check2(client_address, sender_email, helo_name) suspect.tags['spf'] = result if result != 'none': self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation)) action = DUNNO message = apply_template(self.config.get(self.section, 'messagetemplate'), suspect, dict(result=result, explanation=explanation)) configopt = 'on_%s' % result if self.config.has_option(self.section, configopt): action=string_to_actioncode(self.config.get(self.section, configopt)) return action, message def lint(self): lint_ok = True if not HAVE_SPF: print('pyspf or pydns module not installed - this plugin will do nothing') lint_ok = False if not HAVE_NETADDR: print('WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation') if not self.checkConfig(): print('Error checking config') lint_ok = False selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file','').strip() if selective_sender_domain_file != '' and not os.path.exists(selective_sender_domain_file): print("domain_selective_spf_file %s does not exist" % selective_sender_domain_file) lint_ok = False ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file', '').strip() if ip_whitelist_file != '' and os.path.exists(ip_whitelist_file): print("ip_whitelist_file %s does not exist - IP whitelist is disabled" % ip_whitelist_file) lint_ok = False sqlquery = self.config.get(self.section, 'domain_sql_query') dbconnection = self.config.get(self.section, 'dbconnection', '').strip() if not SQL_EXTENSION_ENABLED and dbconnection != '': print('SQLAlchemy not available, cannot use SQL backend') lint_ok = False elif dbconnection == '': print('No DB connection defined. Disabling SQL backend') else: if not sqlquery.lower().startswith('select '): lint_ok = False print('SQL statement must be a SELECT query') if lint_ok: try: conn=get_session(dbconnection) conn.execute(sqlquery, {'domain':'example.com'}) except Exception as e: lint_ok = False print(str(e)) return lint_ok def __str__(self): return "SPF"
class EBLLookup(ScannerPlugin): def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.whitelist = None self.requiredvars={ 'whitelist_file':{ 'default':'/etc/postomaat/conf.d/ebl-whitelist.txt', 'description':'path to file containing whitelisted sender domains', }, 'dnszone':{ 'default':'ebl.msbl.org', 'description':'the DNS zone to query. defaults to ebl.msbl.org', }, 'hash': { 'default':'sha1', 'description':'hash function used by DNS zone. Use one of sha1, md5' }, 'response':{ 'default':'127.0.0.2', 'description':'expected response of zone query', }, 'messagetemplate':{ 'default':'${sender} listed by ${dnszone} for ${message}' }, 'normalisation':{ 'default':'ebl', 'description':'type of normalisation to be applied to email addresses before hashing. choose one of ebl (full normalisation according to ebl.msbl.org standard), low (lowercase only)' }, 'decode_srs':{ 'default':'0', 'description':'decode SRS encoded sender addresses before lookup' }, 'check_srs_only':{ 'default':'0', 'description':'only check decoded SRS sender addresses against the blacklist zone' }, } def _is_whitelisted(self, from_domain): whitelist_file = self.config.get(self.section,'whitelist_file').strip() if whitelist_file == '': return False whitelisted = False self.whitelist = FileList(whitelist_file,lowercase=True) if from_domain in self.whitelist.get_list(): whitelisted = True return whitelisted def _email_normalise_ebl(self, address): if not '@' in address: self.logger.error('Not an email address: %s' % address) return address address = address.lower() lhs, domain = address.rsplit('@',1) domainparts = domain.split('.') if 'googlemail' in domainparts: # replace googlemail with gmail tld = '.'.join(domainparts[1:]) domain = 'gmail.%s' % tld domainparts = ['gmail', tld] if '+' in lhs: # strip all + tags lhs = lhs.split('+')[0] if 'gmail' in domainparts: # discard periods in gmail lhs = lhs.replace('.', '') if 'yahoo' in domainparts or 'ymail' in domainparts: # strip - tags from yahoo lhs = lhs.split('-')[0] lhs = re.sub('^(envelope-from|id|r|receiver)=', '', lhs) # strip mail log prefixes return '%s@%s' % (lhs, domain) def _email_normalise_low(self, address): address = address.lower() return address def _email_normalise(self, address): n = self.config.get(self.section,'normalisation') if n == 'ebl': address = self._email_normalise_ebl(address) elif n == 'low': address = self._email_normalise_low(address) return address def _create_hash(self, value): hashtype = self.config.get(self.section,'hash').lower() if hashtype == 'sha1': myhash = sha1(value.encode('utf-8')).hexdigest() elif hashtype == 'md5': myhash = md5(value).hexdigest() else: myhash = '' return myhash def _ebl_lookup(self, addr_hash): listed = False message = None dnszone = self.config.get(self.section,'dnszone').strip() response = self.config.get(self.section,'response').strip() query = '%s.%s' % (addr_hash, dnszone) result = lookup(query) if result is not None: for rec in result: if rec == response: listed = True result = lookup(query, qtype=QTYPE_TXT) if result: message = result[0] break return listed, message def _is_srs(self, addr): if addr.startswith('SRS0=') or addr.startswith('SRS1='): return True return False def _decode_srs(self, addr): srs = SRSDecode() return srs.reverse(addr) def examine(self, suspect): if not DNSQUERY_EXTENSION_ENABLED: return DUNNO from_address=suspect.get_value('sender') if from_address is None: self.logger.warning('No FROM address found') return DEFER_IF_PERMIT,'internal policy error (no from address)' from_address=strip_address(from_address) if self.config.getboolean(self.section,'check_srs_only') and not self._is_srs(from_address): self.logger.info('skipping non SRS address %s' % from_address) return DUNNO if HAVE_SRS and self.config.getboolean(self.section,'decode_srs'): from_address = self._decode_srs(from_address) from_domain=extract_domain(from_address) if self._is_whitelisted(from_domain): return DUNNO from_address = self._email_normalise(from_address) addr_hash = self._create_hash(from_address) listed, message = self._ebl_lookup(addr_hash) if listed: values = { 'dnszone': self.config.get(self.section,'dnszone','').strip(), 'message': message, } message = apply_template(self.config.get(self.section,'messagetemplate'),suspect, values) return REJECT, message else: return DUNNO def lint(self): dnszone = self.config.get(self.section,'dnszone','').strip() print('querying zone %s' % dnszone) lint_ok = True if not self.checkConfig(): print('Error checking config') lint_ok = False if not DNSQUERY_EXTENSION_ENABLED: print("no DNS resolver library available - this plugin will do nothing") lint_ok = False if self.config.getboolean(self.section,'decode_srs') and not HAVE_SRS: print('decode_srs enabled but SRS library is not available') lint_ok = False hashtype = self.config.get(self.section,'hash').lower() if hashtype not in ['sha1', 'md5']: lint_ok = False print('unsupported hash type %s' % hashtype) normalisation = self.config.get(self.section,'normalisation') if normalisation not in ['ebl', 'low']: lint_ok = False print('unsupported normalisation type %s' % normalisation) addr_hash = self._create_hash('*****@*****.**') listed, message = self._ebl_lookup(addr_hash) if not listed: lint_ok = False print('test entry not found in dns zone') else: print('test entry found in dns zone: %s' % message) if lint_ok: whitelist_file = self.config.get(self.section,'whitelist_file') if whitelist_file.strip() == '': print('No whitelist defined') return lint_ok
class EnforceTLS(ScannerPlugin): def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.selective_domain_loader = None self.requiredvars={ 'domainlist':{ 'default':'', 'description':""" if this is empty, all recipient domains will be forced to use TLS txt:<filename> - get from simple textfile which lists one domain per line sql:<statement> - get from sql database :domain will be replaced with the actual domain name. must return field enforce_inbound_tls """, }, 'dbconnection':{ 'default':"mysql://root@localhost/enforcetls?charset=utf8", 'description':'SQLAlchemy Connection string', }, 'action':{ 'default':'DEFER', 'description':'Action if connection is not TLS encrypted. set to DUNNO, DEFER, REJECT', }, 'messagetemplate':{ 'default':'Unencrypted connection. This recipient requires TLS' } } def enforce_domain(self, to_domain): dbconnection = self.config.get(self.section,'dbconnection', '').strip() domainlist = self.config.get(self.section,'domainlist') enforce = False if domainlist.strip() == '': enforce = True elif domainlist.startswith('txt:'): domainfile = domainlist[4:] if self.selective_domain_loader is None: self.selective_domain_loader=FileList(domainfile,lowercase=True) if to_domain in self.selective_domain_loader.get_list(): enforce = True elif domainlist.startswith('sql:') and dbconnection != '': cache = get_default_cache() sqlquery = domainlist[4:] enforce = get_domain_setting(to_domain, dbconnection, sqlquery, cache, self.section, False, self.logger) return enforce def examine(self, suspect): encryption_protocol = suspect.get_value('encryption_protocol') recipient=suspect.get_value('recipient') rcpt_email = strip_address(recipient) if rcpt_email=='' or rcpt_email is None: return DUNNO enforce = self.enforce_domain(extract_domain(rcpt_email)) action = DUNNO message = None if enforce and encryption_protocol == '': action=string_to_actioncode(self.config.get(self.section, 'action')) message = apply_template(self.config.get(self.section,'messagetemplate'),suspect) return action, message def lint(self): lint_ok = True if not self.checkConfig(): print('Error checking config') lint_ok = False if lint_ok: domainlist = self.config.get(self.section,'domainlist') if domainlist.strip() == '': print('Enforcing TLS for all domains') elif domainlist.startswith('txt:'): domainfile = domainlist[4:] if not os.path.exists(domainfile): print('Cannot find domain file %s' % domainfile) lint_ok = False elif domainlist.startswith('sql:'): sqlquery = domainlist[4:] if not sqlquery.lower().startswith('select '): lint_ok = False print('SQL statement must be a SELECT query') if not SQL_EXTENSION_ENABLED: print('SQLAlchemy not available, cannot use sql backend') if lint_ok: dbconnection = self.config.get(self.section, 'dbconnection') try: conn=get_session(dbconnection) conn.execute(sqlquery, {'domain':'example.com'}) except Exception as e: lint_ok = False print(str(e)) else: lint_ok = False print('Could not determine domain list backend type') return lint_ok def __str__(self): return "EnforceTLS"
class SPFPlugin(ScannerPlugin): """This plugin performs SPF validation using the pyspf module https://pypi.python.org/pypi/pyspf/ by default, it just logs the result (test mode) to enable actual rejection of messages, add a config option on_<resulttype> with a valid postfix action. eg: on_fail = REJECT valid result types are: 'pass', 'permerror', 'fail', 'temperror', 'softfail', 'none', and 'neutral' """ def __init__(self,config,section=None): ScannerPlugin.__init__(self,config,section) self.logger=self._logger() self.requiredvars={ 'ip_whitelist_file':{ 'default':'', 'description':'file containing a list of ip adresses to be exempted from SPF checks. Supports CIDR notation if the netaddr module is installed. 127.0.0.0/8 is always exempted', }, 'domain_selective_spf_file':{ 'default':'', 'description':'if this is non-empty, only sender domains in this file will be checked for SPF', }, 'on_fail':{ 'default':'DUNNO', 'description':'Action for SPF fail.', }, 'on_softfail':{ 'default':'DUNNO', 'description':'Action for SPF softfail.', }, 'messagetemplate':{ 'default':'SPF ${result} for domain ${from_domain} from ${client_address} : ${explanation}' } } self.ip_whitelist_loader=None self.ip_whitelist=[] # either a list of plain ip adress strings or a list of IPNetwork if netaddr is available self.selective_domain_loader=None def is_private_address(self,addr): if addr=='127.0.0.1' or addr=='::1' or addr.startswith('10.') or addr.startswith('192.168.') or addr.startswith('fe80:'): return True if not addr.startswith('172.'): return False for i in range(16,32): if addr.startswith('172.%s'%i): return True return False def ip_whitelisted(self,addr): if self.is_private_address(addr): return True #check ip whitelist ip_whitelist_file=self.config.get(self.section,'ip_whitelist_file') if ip_whitelist_file is not None: plainlist = [] if self.ip_whitelist_loader is None: self.ip_whitelist_loader=FileList(ip_whitelist_file,lowercase=True) if self.ip_whitelist_loader.file_changed(): plainlist=self.ip_whitelist_loader.get_list() if have_netaddr: self.ip_whitelist=[IPNetwork(x) for x in plainlist] else: self.ip_whitelist=plainlist if have_netaddr: checkaddr=IPAddress(addr) for net in self.ip_whitelist: if checkaddr in net: return True else: if addr in plainlist: return True return False def examine(self,suspect): if not have_spf: return DUNNO client_address=suspect.get_value('client_address') helo_name=suspect.get_value('helo_name') sender=suspect.get_value('sender') if client_address is None or helo_name is None or sender is None: self.logger.error('missing client_address or helo or sender') return DUNNO if self.ip_whitelisted(client_address): self.logger.info("Client %s is whitelisted - no SPF check"%client_address) return DUNNO sender_email = strip_address(sender) if sender_email=='' or sender_email is None: return DUNNO selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file') if selective_sender_domain_file!='': if self.selective_domain_loader is None: self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True) try: sender_domain = extract_domain(sender_email) if sender_domain is None: return DUNNO except ValueError as e: self.logger.warning(str(e)) return DUNNO if not sender_domain.lower() in self.selective_domain_loader.get_list(): return DUNNO result, explanation = spf.check2(client_address, sender_email, helo_name) suspect.tags['spf']=result if result!='none': self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation)) action = DUNNO message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(result=result,explanation=explanation)) configopt='on_%s'%result if self.config.has_option(self.section,configopt): action=self.config.get(self.section,configopt) return action, message def lint(self): lint_ok = True if not have_spf: print 'pyspf or pydns module not installed - this plugin will do nothing' lint_ok = False if not have_netaddr: print 'WARNING: netaddr python module not installed - IP whitelist will not support CIDR notation' if not self.checkConfig(): print 'Error checking config' lint_ok = False return lint_ok def __str__(self): return "SPF"