def _problemcode(self):
retcode=string_to_actioncode(self.config.get(self.section,'problemaction'), self.config)
if retcode!=None:
return retcode
else:
#in case of invalid problem action
return DEFER
def examine(self, suspect):
if not self.should_we_check_this_domain(suspect):
return DUNNO
envelope_recipient_domain = suspect.to_domain.lower()
envelope_sender_domain = suspect.from_domain.lower()
if envelope_sender_domain == envelope_recipient_domain:
return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF)
header_from_domain = extract_from_domain(suspect)
if header_from_domain is None:
self._logger().warn("%s: Could not extract header from domain for spearphish check" % suspect.id)
return DUNNO
if header_from_domain == envelope_recipient_domain:
virusname = self.config.get(self.section, 'virusname')
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
logmsg = '%s: spear phish pattern detected, recipient=%s env_sender_domain=%s header_from_domain=%s' % (
suspect.id, suspect.to_address, envelope_sender_domain, header_from_domain)
self._logger().info(logmsg)
self.flag_as_phish(suspect, virusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname})
return actioncode, message
return DUNNO
def examine(self, suspect):
if suspect.get_tag('Dspam.skip') is True:
self.logger.debug(
'%s Skipping Dspam Plugin (requested by previous plugin)' %
suspect.id)
suspect.set_tag('Dspam.skipreason', 'requested by previous plugin')
return DUNNO
if self.config.getboolean(self.section, 'scanoriginal'):
content = suspect.get_original_source()
else:
content = suspect.get_source()
recipient = suspect.to_address
try:
results = self._check_dspam(content, recipient)
except DspamClientError as e:
self.logger.error('%s failed to process by dspam: %s' %
(suspect.id, str(e)))
return self._problemcode()
action = DUNNO
message = None
isspam = self._is_spam(results)
suspect.tags['spam']['Dspam'] = isspam
suspect.tags['DSpam.report'] = self._report(results)
if isspam:
action = string_to_actioncode(
self.config.get(self.section, 'spamaction'), self.config)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
results)
return action, message
def examine(self, suspect):
self.filelist.filename = self.config.get(self.section, 'domainsfile')
checkdomains = self.filelist.get_list()
envelope_sender_domain = suspect.from_domain.lower()
header_from_domain = extract_from_domain(suspect)
if header_from_domain is None:
return
if header_from_domain not in checkdomains:
return
# TODO: do we need a tag from dkim to check if the verified dkim domain
# actually matches the header from domain?
dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False)
if dkimresult == True:
return DUNNO
# DKIM failed, check SPF if envelope senderdomain belongs to header
# from domain
spfresult = suspect.get_tag('SPF.status', 'unknown')
if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass':
return DUNNO
failaction = self.config.get(self.section, 'failaction')
actioncode = string_to_actioncode(failaction, self.config)
values = dict(
header_from_domain=header_from_domain)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
def examine(self, suspect):
self.filelist.filename = self.config.get(self.section, 'domainsfile')
checkdomains = self.filelist.get_list()
envelope_sender_domain = suspect.from_domain.lower()
header_from_domain = self.extract_from_domain(suspect)
if header_from_domain == None:
return
if header_from_domain not in checkdomains:
return
# TODO: do we need a tag from dkim to check if the verified dkim domain
# actually matches the header from domain?
dkimresult = suspect.get_tag('DKIMVerify.sigvalid', False)
if dkimresult == True:
return DUNNO
# DKIM failed, check SPF if envelope senderdomain belongs to header
# from domain
spfresult = suspect.get_tag('SPF.status', 'unknown')
if (envelope_sender_domain == header_from_domain or envelope_sender_domain.endswith('.%s' % header_from_domain)) and spfresult == 'pass':
return DUNNO
failaction = self.config.get(self.section, 'failaction')
actioncode = string_to_actioncode(failaction, self.config)
values = dict(
header_from_domain=header_from_domain)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
def examine(self, suspect):
urls = suspect.get_tag('black.uris', defaultvalue=[])
add_header_links = self.config.getboolean(self.section,
'addheaderlinks')
add_header_count = self.config.getboolean(self.section,
'addheadercount')
if len(urls) == 0:
return DUNNO
elif add_header_count is True and add_header_links is False:
suspect.add_header('X-Black-Host-Count',
str(len(urls)),
immediate=True)
elif add_header_count is True and add_header_links is True:
suspect.add_header('X-Black-Host',
"\t" + "\r\n\t\t\t ".join(urls),
immediate=True)
suspect.add_header('X-Black-Host-Count',
str(len(urls)),
immediate=True)
elif add_header_count is False and add_header_links is True:
suspect.add_header('X-Black-Host',
"\t" + "\r\n\t\t\t ".join(urls),
immediate=True)
return string_to_actioncode(
self.config.get('URIExtractPlugin', 'action'),
self.config), apply_template(
self.config.get('URIExtractPlugin', 'message'), suspect,
dict(domain=urls[0], blacklist='tbd'))
def load_limiter_config(self, text):
patt = re.compile(
r'^limit\s+name=(?P<name>[^\s]+)\s+rate=(?P<max>\-?\d{1,10})\/(?P<time>\d{1,10})\s+fields=(?P<fieldlist>[^\s]+)(\s+match=\/(?P<matchregex>.+)\/(\s+skip=(?P<skiplist>[^\s]+))?)?\s+action=(?P<action>[^\s]+)\s+message=(?P<message>.*)$'
)
limiters = []
lineno = 0
for line in text.split('\n'):
lineno += 1
line = line.strip()
if line.startswith('#') or line.strip() == '':
continue
match = patt.match(line)
if match == None:
self.logger.error('cannot parse limiter config line %s' %
lineno)
continue
gdict = match.groupdict()
limiter = Limiter()
limiter.name = gdict['name']
limiter.max = int(gdict['max'])
limiter.timespan = int(gdict['time'])
limiter.fields = gdict['fieldlist'].split(',')
limiter.regex = gdict['matchregex']
if gdict['skiplist'] != None:
limiter.skip = gdict['skiplist'].split(',')
action = string_to_actioncode(gdict['action'])
if action == None:
self.logger.error(
"Limiter config line %s : invalid action %s" %
(lineno, gdict['action']))
limiter.action = action
limiter.message = gdict['message']
limiters.append(limiter)
return limiters
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self._logger().info('Not scanning - message too big (message %s bytes > config %s bytes )' %
(suspect.size, self.config.getint(self.section, 'maxsize')))
return DUNNO
try:
viruses = self.scan_file(suspect.tempfile)
if viruses is not None:
self._logger().info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus'][self.config.get(self.section,'identifier')] = True
suspect.tags['%s.virus'%self.config.get(self.section,'identifier')] = viruses
suspect.debug('Viruses found in message : %s' % viruses)
else:
suspect.tags['virus'][self.config.get(self.section,'identifier')] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = viruses.items()[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
else:
return DUNNO
except Exception, e:
self._logger().warning("Error encountered while running cmdline av scan: %s" % str(e))
def examine(self, suspect):
enginename = self.config.get(self.section, 'enginename')
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info('Not scanning - message too big')
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
if viruses != None:
self.logger.info(
"Virus found in message from %s : %s" % (suspect.from_address, viruses))
suspect.tags['virus'][enginename] = True
suspect.tags['%s.virus' % enginename] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus'][enginename] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = viruses.items()[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
return DUNNO
except Exception, e:
self.logger.warning("Error encountered while contacting ICAP server (try %s of %s): %s" % (
i + 1, self.config.getint(self.section, 'retries'), str(e)))
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info('Not scanning - message too big')
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
if viruses != None:
self.logger.info(
"Virus found in message from %s : %s" % (suspect.from_address, viruses))
suspect.tags['virus']['ClamAV'] = True
suspect.tags['ClamavPlugin.virus'] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['ClamAV'] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = viruses.items()[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
return DUNNO
except Exception, e:
self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % (
i + 1, self.config.getint(self.section, 'retries'), str(e)))
def examine(self, suspect):
if not self.should_we_check_this_domain(suspect):
return DUNNO
envelope_recipient_domain = suspect.to_domain.lower()
envelope_sender_domain = suspect.from_domain.lower()
if envelope_sender_domain == envelope_recipient_domain:
return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF)
header_from_domain = extract_from_domain(suspect)
if header_from_domain is None:
self._logger().warn(
"%s: Could not extract header from domain for spearphish check"
% suspect.id)
return DUNNO
if header_from_domain == envelope_recipient_domain:
virusname = self.config.get(self.section, 'virusname')
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
logmsg = '%s: spear phish pattern detected, recipient=%s env_sender_domain=%s header_from_domain=%s' % (
suspect.id, suspect.to_address, envelope_sender_domain,
header_from_domain)
self._logger().info(logmsg)
self.flag_as_phish(suspect, virusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
{'virusname': virusname})
return actioncode, message
return DUNNO
def check_sql_blacklist(self, suspect, runtimeconfig=None):
"""Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise"""
#work in progress
if not self.config.has_option(
self.section,
'check_sql_blacklist') or not self.config.getboolean(
self.section, 'check_sql_blacklist'):
return DUNNO
if not SQL_EXTENSION_ENABLED:
self.logger.error(
'Cannot check sql blacklist, SQLALCHEMY extension is not available'
)
return DUNNO
try:
dbsession = get_session(
self.config.get(self.section, 'sql_blacklist_dbconnectstring'))
conf_sql = self.config.get(self.section, 'sql_blacklist_sql')
sql, params = self._replace_sql_params(suspect, conf_sql)
resultproxy = dbsession.execute(sql, params)
except Exception as e:
self.logger.error('Could not read blacklist from DB: %s' % e)
suspect.debug('Blacklist check failed: %s' % e)
return DUNNO
for result in resultproxy:
dbvalue = result[0] # this value might have multiple words
allvalues = dbvalue.split()
for blvalue in allvalues:
self.logger.debug(blvalue)
# build regex
# translate glob to regexr
# http://stackoverflow.com/questions/445910/create-regex-from-glob-expression
regexp = re.escape(blvalue).replace(r'\?',
'.').replace(r'\*', '.*?')
self.logger.debug(regexp)
pattern = re.compile(regexp)
if pattern.search(suspect.from_address):
self.logger.debug(
'%s Blacklist match : %s for sa pref %s' %
(suspect.id, suspect.from_address, blvalue))
confcheck = self.config
if runtimeconfig is not None:
confcheck = runtimeconfig
configaction = string_to_actioncode(
confcheck.get(self.section, 'highspamaction'),
self.config)
suspect.tags['spam']['SpamAssassin'] = True
prependheader = self.config.get('main',
'prependaddedheaders')
suspect.addheader("%sBlacklisted" % prependheader, blvalue)
suspect.debug('Sender is Blacklisted: %s' % blvalue)
if configaction is None:
return DUNNO
return configaction
return DUNNO
def _problemcode(self):
retcode = string_to_actioncode(
self.config.get(self.section, 'problemaction'), self.config)
if retcode != None:
return retcode
else:
# in case of invalid problem action
return DEFER
def check_sql_blacklist(self, suspect, runtimeconfig=None):
"""Check this message against the SQL blacklist. returns highspamaction on hit, DUNNO otherwise"""
#work in progress
if not self.config.has_option(self.section, 'check_sql_blacklist') or not self.config.getboolean(self.section, 'check_sql_blacklist'):
return DUNNO
from fuglu.extensions.sql import ENABLED
if not ENABLED:
self.logger.error(
'Cannot check sql blacklist, SQLALCHEMY extension is not available')
return DUNNO
from fuglu.extensions.sql import get_session
try:
dbsession = get_session(
self.config.get(self.section, 'sql_blacklist_dbconnectstring'))
conf_sql = self.config.get(self.section, 'sql_blacklist_sql')
sql, params = self._replace_sql_params(suspect, conf_sql)
resultproxy = dbsession.execute(sql, params)
except Exception as e:
self.logger.error('Could not read blacklist from DB: %s' % e)
suspect.debug('Blacklist check failed: %s' % e)
return DUNNO
for result in resultproxy:
dbvalue = result[0] # this value might have multiple words
allvalues = dbvalue.split()
for blvalue in allvalues:
self.logger.debug(blvalue)
# build regex
# translate glob to regexr
# http://stackoverflow.com/questions/445910/create-regex-from-glob-expression
regexp = re.escape(blvalue).replace(
r'\?', '.').replace(r'\*', '.*?')
self.logger.debug(regexp)
pattern = re.compile(regexp)
if pattern.search(suspect.from_address):
self.logger.debug(
'Blacklist match : %s for sa pref %s' % (suspect.from_address, blvalue))
confcheck = self.config
if runtimeconfig != None:
confcheck = runtimeconfig
configaction = string_to_actioncode(
confcheck.get(self.section, 'highspamaction'), self.config)
suspect.tags['spam']['SpamAssassin'] = True
prependheader = self.config.get(
'main', 'prependaddedheaders')
suspect.addheader("%sBlacklisted" % prependheader, blvalue)
suspect.debug('Sender is Blacklisted: %s' % blvalue)
if configaction == None:
return DUNNO
return configaction
return DUNNO
def test_defaultcodes(self):
"""test actioncode<->string conversion"""
conf = ConfigParser()
conf.add_section('spam')
conf.add_section('virus')
conf.set('spam', 'defaultlowspamaction', 'REJEcT')
conf.set('spam', 'defaulthighspamaction', 'REjECT')
conf.set('virus', 'defaultvirusaction', 'rejeCt')
self.assertEqual(
string_to_actioncode('defaultlowspamaction', conf), REJECT)
self.assertEqual(
string_to_actioncode('defaulthighspamaction', conf), REJECT)
self.assertEqual(
string_to_actioncode('defaultvirusaction', conf), REJECT)
self.assertEqual(string_to_actioncode('nonexistingstuff'), None)
self.assertEqual(actioncode_to_string(REJECT), 'REJECT')
self.assertEqual(
actioncode_to_string(string_to_actioncode('discard')), 'DELETE')
def test_defaultcodes(self):
"""test actioncode<->string conversion"""
conf = ConfigParser()
conf.add_section('spam')
conf.add_section('virus')
conf.set('spam', 'defaultlowspamaction', 'REJEcT')
conf.set('spam', 'defaulthighspamaction', 'REjECT')
conf.set('virus', 'defaultvirusaction', 'rejeCt')
self.assertEqual(string_to_actioncode('defaultlowspamaction', conf),
REJECT)
self.assertEqual(string_to_actioncode('defaulthighspamaction', conf),
REJECT)
self.assertEqual(string_to_actioncode('defaultvirusaction', conf),
REJECT)
self.assertEqual(string_to_actioncode('nonexistingstuff'), None)
self.assertEqual(actioncode_to_string(REJECT), 'REJECT')
self.assertEqual(actioncode_to_string(string_to_actioncode('discard')),
'DELETE')
def examine(self, suspect):
starttime = time.time()
if suspect.size > self.config.getint(self.section, 'maxsize'):
self._logger().info(
'Not scanning - message too big (message %s bytes > config %s bytes )'
% (suspect.size, self.config.getint(self.section, 'maxsize')))
return DUNNO
content = suspect.get_message_rep().as_string()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
headerscannername = '-' + self.config.get(
self.section, 'headerscannername')
if headerscannername == '-':
headerscannername = ''
if self.config.getboolean(self.section, 'networkmode'):
viruses = self.scan_stream(content)
else:
viruses = self.scan_file(suspect.tempfile)
if viruses != None:
self._logger().info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus']['F-Prot'] = True
suspect.tags['FprotPlugin.virus'] = viruses
suspect.debug('Viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['F-Prot'] = False
addheaderclean = self.config.get(self.section,
'addheaderclean')
suspect.set_tag('fprot.clean', 'Clean')
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
addheaderinfected = self.config.get(
self.section, 'addheaderinfected')
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(infectedfile=firstinfected,
virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'),
suspect, values)
suspect.set_tag('fprot.virus', firstvirusname)
return actioncode, message
else:
return DUNNO
except Exception as e:
self._logger().warning(
"Error encountered while contacting fpscand (try %s of %s): %s"
% (i + 1, self.config.getint(self.section,
'retries'), str(e)))
self._logger().error("fpscand failed after %s retries" %
self.config.getint(self.section, 'retries'))
content = None
return self._problemcode()
def examine(self,suspect):
if not YARA_AVAILABLE:
return
self.reload_if_necessary()
if not self.compiled_rules:
return
yarahits = None
infectedfile = None
m = suspect.get_message_rep()
for i in m.walk():
if i.is_multipart():
continue
att_name = i.get_filename(None)
if att_name:
# some filenames are encoded, try to decode
try:
att_name = ''.join([x[0] for x in decode_header(att_name)])
except:
pass
else:
att_name='message'
payload = StringIO(i.get_payload(decode=True))
yarahits = self.check_file(suspect,att_name, payload)
if yarahits != None:
infectedfile = att_name
break
if infectedfile and yarahits:
self.logger.info(
"YARA hit(s) in message from %s : %s" % (suspect.from_address, yarahits))
suspect.tags['virus']['YARA'] = True
suspect.tags['YARAPlugin.virus'] = yarahits[0].rule
suspect.debug('YARA hit found in message : %s' % yarahits)
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
values = dict(
infectedfile=infectedfile, virusname=yarahits[0].rule)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
else:
suspect.tags['virus']['YARA'] = False
return DUNNO
def examine(self, suspect):
enginename = 'sophos'
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info('Not scanning - message too big')
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
headerscannername = '-' + self.config.get(
self.section, 'headerscannername')
if headerscannername == '-':
headerscannername = ''
if viruses is not None:
self.logger.info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus'][enginename] = True
suspect.tags['%s.virus' % enginename] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus'][enginename] = False
addheaderclean = self.config.get(self.section,
'addheaderclean')
suspect.set_tag('sssp.clean', 'Clean')
if viruses is not None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
addheaderinfected = self.config.get(
self.section, 'addheaderinfected')
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(infectedfile=firstinfected,
virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'),
suspect, values)
suspect.set_tag('sssp.virus', firstvirusname)
return actioncode, message
return DUNNO
except Exception as e:
self.logger.warning(
"Error encountered while contacting SSSP server (try %s of %s): %s"
% (i + 1, self.config.getint(self.section,
'retries'), str(e)))
self.logger.error("SSSP scan failed after %s retries" %
self.config.getint(self.section, 'retries'))
return self._problemcode()
def examine(self, suspect):
if not self.should_we_check_this_domain(suspect):
return DUNNO
envelope_recipient_domain = suspect.to_domain.lower()
envelope_sender_domain = suspect.from_domain.lower()
if envelope_sender_domain == envelope_recipient_domain:
return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF)
header_from_domains = []
header_from_domain = extract_from_domain(suspect)
if header_from_domain is None:
self.logger.warn(
"%s: Could not extract header from domain for spearphish check"
% suspect.id)
return DUNNO
else:
header_from_domains.append(header_from_domain)
self.logger.debug(
'%s: checking domain %s (source: From header address part)' %
(suspect.id, header_from_domain))
if self.config.getboolean(self.section, 'check_display_part'):
display_from_domain = extract_from_domain(suspect, False)
if display_from_domain is not None and display_from_domain not in header_from_domains:
header_from_domains.append(display_from_domain)
self.logger.debug(
'%s: checking domain %s (source: From header display part)'
% (suspect.id, display_from_domain))
actioncode = DUNNO
message = None
for header_from_domain in header_from_domains:
if header_from_domain == envelope_recipient_domain:
virusname = self.config.get(self.section, 'virusname')
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \
(suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain)
self.logger.info(logmsg)
self.flag_as_phish(suspect, virusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
{'virusname': virusname})
break
return actioncode, message
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info('Not scanning - message too big')
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
headerscannername = '-' + self.config.get(self.section, 'headerscannername')
if headerscannername == '-':
headerscannername = ''
if viruses != None:
self.logger.info(
"Virus found in message from %s : %s" % (suspect.from_address, viruses))
suspect.tags['virus']['ClamAV'] = True
suspect.tags['ClamavPlugin.virus'] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['ClamAV'] = False
addheaderclean = self.config.get(self.section, 'addheaderclean')
suspect.set_tag('clam.clean', 'Clean')
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
addheaderinfected = self.config.get(self.section, 'addheaderinfected')
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
suspect.set_tag('clam.virus', firstvirusname)
return actioncode, message
return DUNNO
except Exception as e:
self.__invalidate_socket()
# don't warn the first time if it's just a broken pipe which
# can happen with the new pipelining protocol
if not (i == 0 and isinstance(e, socket.error) and e.errno == errno.EPIPE):
self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % (
i + 1, self.config.getint(self.section, 'retries'), str(e)))
self.logger.error("Clamdscan failed after %s retries" %
self.config.getint(self.section, 'retries'))
content = None
return self._problemcode()
def lint(self):
viract = self.config.get(self.section, 'virusaction')
print("Virusaction: %s" %
actioncode_to_string(string_to_actioncode(viract, self.config)))
allok = self.checkConfig() and self.lint_ping() and self.lint_eicar()
if self.config.getboolean(self.section, 'clamscanfallback'):
print('WARNING: Fallback to clamscan enabled')
starttime = time.time()
allok = self.lint_eicar('shell')
if allok:
runtime = time.time() - starttime
print('clamscan scan time: %.2fs' % runtime)
return allok
def lint(self):
viract = self.config.get(self.section, 'virusaction')
print("Virusaction: %s" % actioncode_to_string(
string_to_actioncode(viract, self.config)))
allok = self.checkConfig() and self.lint_ping() and self.lint_eicar()
if self.config.getboolean(self.section, 'clamscanfallback'):
print('WARNING: Fallback to clamscan enabled')
starttime = time.time()
allok = self.lint_eicar('shell')
if allok:
runtime = time.time()-starttime
print('clamscan scan time: %.2fs' % runtime)
return allok
def examine(self, suspect):
if not YARA_AVAILABLE:
return
self.reload_if_necessary()
if not self.compiled_rules:
return
yarahits = None
infectedfile = None
m = suspect.get_message_rep()
for i in m.walk():
if i.is_multipart():
continue
att_name = i.get_filename(None)
if att_name:
# some filenames are encoded, try to decode
try:
att_name = "".join([x[0] for x in decode_header(att_name)])
except:
pass
else:
att_name = "message"
payload = StringIO(i.get_payload(decode=True))
yarahits = self.check_file(suspect, att_name, payload)
if yarahits != None:
infectedfile = att_name
break
if infectedfile and yarahits:
self.logger.info("YARA hit(s) in message from %s : %s" % (suspect.from_address, yarahits))
suspect.tags["virus"]["YARA"] = True
suspect.tags["YARAPlugin.virus"] = yarahits[0].rule
suspect.debug("YARA hit found in message : %s" % yarahits)
virusaction = self.config.get(self.section, "virusaction")
actioncode = string_to_actioncode(virusaction, self.config)
values = dict(infectedfile=infectedfile, virusname=yarahits[0].rule)
message = apply_template(self.config.get(self.section, "rejectmessage"), suspect, values)
return actioncode, message
else:
suspect.tags["virus"]["YARA"] = False
return DUNNO
def examine(self, suspect):
if not DOMAINMAGIC_AVAILABLE:
self.logger.info('Not scanning - Domainmagic not available')
return DUNNO
if self.rbllookup is None:
self.rbllookup = RBLLookup()
self.rbllookup.from_config(
self.config.get(self.section, 'blacklistconfig'))
self._init_tldmagic()
urls = suspect.get_tag('body.uris', defaultvalue=[])
#self.logger.info("Body URIs to check: %s"%urls)
domains = set(map(fqdn_from_uri, urls))
counter = 0
for domain in domains:
counter += 1
if counter > self.config.getint(self.section, 'maxdomains'):
self.logger.info("maximum number of domains reached")
break
tldcount = self.tldmagic.get_tld_count(domain)
parts = domain.split('.')
if self.config.getboolean(self.section, 'checksubdomains'):
subrange = range(tldcount + 1, len(parts) + 1)
else:
subrange = [tldcount + 1]
for subindex in subrange:
subdomain = '.'.join(parts[-subindex:])
listings = self.rbllookup.listings(subdomain)
for identifier, humanreadable in iter(listings.items()):
self.logger.info(
"%s : url host %s flagged as %s because %s" %
(suspect.id, domain, identifier, humanreadable))
return string_to_actioncode(
self.config.get(self.section, 'action'),
self.config), apply_template(
self.config.get(self.section, 'message'), suspect,
dict(domain=domain, blacklist=identifier))
return DUNNO
def examine(self, suspect):
if not DNSQUERY_EXTENSION_ENABLED:
return DUNNO
if not self.config.getboolean(self.section, 'check_always'):
# save the lookup if mail is already tagged as virus or spam
if suspect.is_virus() or suspect.is_spam() or suspect.is_blocked():
return DUNNO
maxlookups = self.config.getint(self.section, 'maxlookups')
emails = suspect.get_tag('emails', defaultvalue=[])[:maxlookups]
emails = [self._email_normalise(email) for email in emails]
emails = list(set(emails))
#if emails:
# self.logger.debug('%s EBL checking addresses %s' % (suspect.id, ', '.join(emails)))
listed = False
action = DUNNO
message = None
email = None
for email in emails:
addr_hash = self._create_hash(email)
listed, message = self._ebl_lookup(addr_hash)
if listed:
break
suspect.tags['spam']['EBL'] = listed
if listed:
self.logger.debug('%s EBL hit for %s' % (suspect.id, email))
action = string_to_actioncode(
self.config.get(self.section, 'action'))
suspect.tags['EBL.email'] = email
suspect.tags['EBL.reason'] = message
if action != DUNNO:
values = {
'dnszone': self.config.get(self.section, 'dnszone'),
'message': message,
}
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
values)
return action, message
def examine(self, suspect):
actionrules = self.config.get(self.section, 'actionrules')
if actionrules == None or actionrules == "":
return DUNNO
if not os.path.exists(actionrules):
self.logger.error('Action Rules file does not exist : %s' %
actionrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(actionrules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg == None or arg.strip() == '':
self.logger.error("Rule match but no action defined.")
return DUNNO
arg = arg.strip()
spl = arg.split(None, 1)
actionstring = spl[0]
message = None
if len(spl) == 2:
message = spl[1]
self.logger.debug("%s: Rule match! Action override: %s" %
(suspect.id, arg.upper()))
actioncode = string_to_actioncode(actionstring, self.config)
if actioncode != None:
return actioncode, message
elif actionstring.upper() == 'REDIRECT':
suspect.to_address = message.strip()
suspect.recipients = [
suspect.to_address,
]
# todo: should we override to_domain? probably not
# todo: check for invalid adress, multiple adressses
# todo: document redirect action
else:
self.logger.error("Invalid action: %s" % arg)
return DUNNO
return DUNNO
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info('Not scanning - message too big')
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
if viruses != None:
self.logger.info(
"Virus found in message from %s : %s" % (suspect.from_address, viruses))
suspect.tags['virus']['ClamAV'] = True
suspect.tags['ClamavPlugin.virus'] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['ClamAV'] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
return DUNNO
except Exception as e:
self.__invalidate_socket()
# don't warn the first time if it's just a broken pipe which
# can happen with the new pipelining protocol
if not (i == 0 and isinstance(e, socket.error) and e.errno == errno.EPIPE):
self.logger.warning("Error encountered while contacting clamd (try %s of %s): %s" % (
i + 1, self.config.getint(self.section, 'retries'), str(e)))
self.logger.error("Clamdscan failed after %s retries" %
self.config.getint(self.section, 'retries'))
content = None
return self._problemcode()
def examine(self, suspect):
starttime = time.time()
if suspect.size > self.config.getint(self.section, 'maxsize'):
self._logger().info('Not scanning - message too big (message %s bytes > config %s bytes )' %
(suspect.size, self.config.getint(self.section, 'maxsize')))
return DUNNO
content = suspect.get_message_rep().as_string()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
if self.config.getboolean(self.section, 'networkmode'):
viruses = self.scan_stream(content)
else:
viruses = self.scan_file(suspect.tempfile)
if viruses != None:
self._logger().info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus']['F-Prot'] = True
suspect.tags['FprotPlugin.virus'] = viruses
suspect.debug('Viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['F-Prot'] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
else:
return DUNNO
except Exception as e:
self._logger().warning("Error encountered while contacting fpscand (try %s of %s): %s" %
(i + 1, self.config.getint(self.section, 'retries'), str(e)))
self._logger().error("fpscand failed after %s retries" %
self.config.getint(self.section, 'retries'))
content = None
return self._problemcode()
def examine(self, suspect):
actionrules = self.config.get(self.section, 'actionrules')
if actionrules == None or actionrules == "":
return DUNNO
if not os.path.exists(actionrules):
self.logger.error(
'Action Rules file does not exist : %s' % actionrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(actionrules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg == None or arg.strip() == '':
self.logger.error("Rule match but no action defined.")
return DUNNO
arg = arg.strip()
spl = arg.split(None, 1)
actionstring = spl[0]
message = None
if len(spl) == 2:
message = spl[1]
self.logger.debug(
"%s: Rule match! Action override: %s" % (suspect.id, arg.upper()))
actioncode = string_to_actioncode(actionstring, self.config)
if actioncode != None:
return actioncode, message
elif actionstring.upper() == 'REDIRECT':
suspect.to_address = message.strip()
suspect.recipients = [suspect.to_address, ]
# todo: should we override to_domain? probably not
# todo: check for invalid adress, multiple adressses
# todo: document redirect action
else:
self.logger.error("Invalid action: %s" % arg)
return DUNNO
return DUNNO
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self.logger.info(
'Not scanning - message too big (message %s bytes > config %s bytes )'
% (suspect.size, self.config.getint(self.section, 'maxsize')))
return DUNNO
content = suspect.get_message_rep().as_string()
for i in range(0, self.config.getint(self.section, 'retries')):
try:
viruses = self.scan_stream(content)
if viruses != None:
self.logger.info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus']['drweb'] = True
suspect.tags['DrWebPlugin.virus'] = viruses
suspect.debug('Viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['drweb'] = False
if viruses != None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(infectedfile=firstinfected,
virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'),
suspect, values)
return actioncode, message
else:
return DUNNO
except Exception as e:
self.logger.warning(
"Error encountered while contacting drweb (try %s of %s): %s"
% (i + 1, self.config.getint(self.section,
'retries'), str(e)))
self.logger.error("drweb scan failed after %s retries" %
self.config.getint(self.section, 'retries'))
content = None
return self._problemcode()
def _virusreport(self, suspect, viruses):
actioncode = DUNNO
message = None
if viruses is not None:
self.logger.info("%s Virus found in message from %s : %s" % (suspect.id, suspect.from_address, viruses))
suspect.tags['virus']['ClamAV'] = True
suspect.tags['ClamavPlugin.virus'] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['ClamAV'] = False
if viruses is not None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(
infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
return actioncode, message
def examine(self, suspect):
if not self.should_we_check_this_domain(suspect):
return DUNNO
envelope_recipient_domain = suspect.to_domain.lower()
envelope_sender_domain = suspect.from_domain.lower()
if envelope_sender_domain == envelope_recipient_domain:
return DUNNO # we only check the message if the env_sender_domain differs. If it's the same it will be caught by other means (like SPF)
header_from_domains = []
header_from_domain = extract_from_domain(suspect)
if header_from_domain is None:
self.logger.warn("%s: Could not extract header from domain for spearphish check" % suspect.id)
return DUNNO
else:
header_from_domains.append(header_from_domain)
self.logger.debug('%s: checking domain %s (source: From header address part)' % (suspect.id, header_from_domain))
if self.config.getboolean(self.section, 'check_display_part'):
display_from_domain = extract_from_domain(suspect, False)
if display_from_domain is not None and display_from_domain not in header_from_domains:
header_from_domains.append(display_from_domain)
self.logger.debug('%s: checking domain %s (source: From header display part)' % (suspect.id, display_from_domain))
actioncode = DUNNO
message = None
for header_from_domain in header_from_domains:
if header_from_domain == envelope_recipient_domain:
virusname = self.config.get(self.section, 'virusname')
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
logmsg = '%s: spear phish pattern detected, env_rcpt_domain=%s env_sender_domain=%s header_from_domain=%s' % \
(suspect.id, envelope_recipient_domain, envelope_sender_domain, header_from_domain)
self.logger.info(logmsg)
self.flag_as_phish(suspect, virusname)
message = apply_template(self.config.get(self.section, 'rejectmessage'), suspect, {'virusname': virusname})
break
return actioncode, message
def examine(self, suspect):
if suspect.size > self.config.getint(self.section, 'maxsize'):
self._logger().info(
'Not scanning - message too big (message %s bytes > config %s bytes )'
% (suspect.size, self.config.getint(self.section, 'maxsize')))
return DUNNO
try:
viruses = self.scan_file(suspect.tempfile)
if viruses is not None:
self._logger().info("Virus found in message from %s : %s" %
(suspect.from_address, viruses))
suspect.tags['virus'][self.config.get(self.section,
'identifier')] = True
suspect.tags['%s.virus' % self.config.get(
self.section, 'identifier')] = viruses
suspect.debug('Viruses found in message : %s' % viruses)
else:
suspect.tags['virus'][self.config.get(self.section,
'identifier')] = False
if viruses is not None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = viruses.items()[0]
values = dict(infectedfile=firstinfected,
virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
values)
return actioncode, message
else:
return DUNNO
except Exception as e:
self._logger().warning(
"Error encountered while running cmdline av scan: %s" % str(e))
return self._problemcode()
def _virusreport(self, suspect, viruses):
actioncode = DUNNO
message = None
if viruses is not None:
self.logger.info("%s Virus found in message from %s : %s" %
(suspect.id, suspect.from_address, viruses))
suspect.tags['virus']['ClamAV'] = True
suspect.tags['ClamavPlugin.virus'] = viruses
suspect.debug('viruses found in message : %s' % viruses)
else:
suspect.tags['virus']['ClamAV'] = False
if viruses is not None:
virusaction = self.config.get(self.section, 'virusaction')
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = list(viruses.items())[0]
values = dict(infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
values)
return actioncode, message
def examine(self,suspect):
if self.rbllookup==None:
self.rbllookup=domainmagic.rbl.RBLLookup()
self.rbllookup.from_config(self.config.get(self.section,'blacklistconfig'))
if self.tldmagic==None:
self.tldmagic=domainmagic.tld.TLDMagic()
urls=suspect.get_tag('body.uris',defaultvalue=[])
#self.logger.info("Body URIs to check: %s"%urls)
domains=set(map(domainmagic.extractor.domain_from_uri,urls))
counter=0
for domain in domains:
counter+=1
if counter>self.config.getint(self.section,'maxdomains'):
self.logger.info("maximum number of domains reached")
break
tldcount=self.tldmagic.get_tld_count(domain)
parts=domain.split('.')
if self.config.getboolean(self.section,'checksubdomains'):
subrange=range(tldcount+1,len(parts)+1)
else:
subrange=[tldcount+1]
for subindex in subrange:
subdomain='.'.join(parts[-subindex:])
listings=self.rbllookup.listings(subdomain)
for identifier,humanreadable in listings.iteritems():
self.logger.info("%s : url host %s flagged as %s because %s"%(suspect.id,domain,identifier,humanreadable))
return string_to_actioncode(self.config.get(self.section,'action'), self.config),apply_template(self.config.get(self.section,'message'), suspect, dict(domain=domain,blacklist=identifier))
return DUNNO
def examine(self, suspect):
if suspect.get_tag('send_lmtp') is not True:
return DUNNO
lmtpfrom = self.config.get(self.section, 'lmtpfrom')
if not lmtpfrom:
lmtpfrom = suspect.from_address
if self.config.getboolean(self.section, 'sendoriginal'):
content = suspect.get_original_source()
else:
content = suspect.get_source()
try:
lmtp = self.__init_socket()
except Exception as e:
self.logger.error('%s could not connect to LMTP server: %s' % (suspect.id, str(e)))
return DEFER, 'could not connect to LMTP server'
try:
self.__auth(lmtp)
except Exception as e:
self.logger.error('%s could not authenticate to LMTP server: %s' % (suspect.id, str(e)))
return DEFER, 'could not authenticate to LMTP server'
try:
errors = lmtp.sendmail(lmtpfrom, suspect.to_address, content)
for rcpt in errors:
self.logger.error('%s could not deliver to LMTP server for %s: %s' % (suspect.id, rcpt, errors[rcpt]))
except Exception as e:
self.logger.error('%s could not deliver to LMTP server: %s' % (suspect.id, str(e)))
return DEFER, 'could not deliver to LMTP server'
finally:
lmtp.quit()
successaction = string_to_actioncode(self.config.get(self.section, 'successaction'))
return successaction
def examine(self, suspect):
enginename = "sophos"
if suspect.size > self.config.getint(self.section, "maxsize"):
self.logger.info("Not scanning - message too big")
return
content = suspect.get_source()
for i in range(0, self.config.getint(self.section, "retries")):
try:
viruses = self.scan_stream(content)
if viruses != None:
self.logger.info("Virus found in message from %s : %s" % (suspect.from_address, viruses))
suspect.tags["virus"][enginename] = True
suspect.tags["%s.virus" % enginename] = viruses
suspect.debug("viruses found in message : %s" % viruses)
else:
suspect.tags["virus"][enginename] = False
if viruses != None:
virusaction = self.config.get(self.section, "virusaction")
actioncode = string_to_actioncode(virusaction, self.config)
firstinfected, firstvirusname = viruses.items()[0]
values = dict(infectedfile=firstinfected, virusname=firstvirusname)
message = apply_template(self.config.get(self.section, "rejectmessage"), suspect, values)
return actioncode, message
return DUNNO
except Exception as e:
self.logger.warning(
"Error encountered while contacting SSSP server (try %s of %s): %s"
% (i + 1, self.config.getint(self.section, "retries"), str(e))
)
self.logger.error("SSSP scan failed after %s retries" % self.config.getint(self.section, "retries"))
content = None
return self._problemcode()
def walk(self, suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction = self.config.get(self.section, 'blockaction')
blockactioncode = string_to_actioncode(blockaction)
# try db rules first
self.rulescache.reloadifnecessary()
dbconn = ''
if self.config.has_option(self.section, 'dbconnectstring'):
dbconn = self.config.get(self.section, 'dbconnectstring')
if dbconn.strip() != '':
self.logger.debug('Loading attachment rules from database')
query = self.config.get(self.section, 'query')
dbfile = DBFile(dbconn, query)
user_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN}))
user_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT}))
user_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
user_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' %
(len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address))
domain_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN}))
domain_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT}))
domain_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
domain_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' %
(len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain))
else:
self.logger.debug('Loading attachment rules from filesystem')
user_names = self.rulescache.getNAMERules(suspect.to_address)
user_ctypes = self.rulescache.getCTYPERules(suspect.to_address)
user_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_address)
user_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_address)
domain_names = self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain)
domain_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_domain)
domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_domain)
# always get defaults from file
default_names = self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT)
default_archive_names = self.rulescache.getARCHIVENAMERules(
FUATT_DEFAULT)
default_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
FUATT_DEFAULT)
m = suspect.get_message_rep()
for i in m.walk():
if i.is_multipart():
continue
contenttype_mime = i.get_content_type()
att_name = i.get_filename(None)
if not att_name:
# workaround for mimetypes, it always takes .ksh for text/plain
if i.get_content_type() == 'text/plain':
ext = '.txt'
else:
ext = mimetypes.guess_extension(i.get_content_type())
if ext == None:
ext = ''
att_name = 'unnamed%s' % ext
res = self.matchMultipleSets(
[user_names, domain_names, default_names], att_name, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s : blocked by name)" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# go through content type rules
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE:
pl = i.get_payload(decode=True)
contenttype_magic = self.getBuffertype(pl)
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# archives
if self.config.getboolean(self.section, 'checkarchivenames') or self.config.getboolean(self.section, 'checkarchivecontent'):
archive_type=None
for arext in self.supported_archive_extensions:
if att_name.lower().endswith('.%s'%arext):
archive_type=arext
break
if archive_type!=None:
try:
pl = StringIO(i.get_payload(decode=True))
archive_handle = self._archive_handle(archive_type,pl)
namelist = self._archive_namelist(archive_type,archive_handle)
if self.config.getboolean(self.section, 'checkarchivenames'):
for name in namelist:
if type(name)==unicode: #rarfile returns unicode objects which mess up generated bounces
name=name.encode("utf-8","ignore")
res = self.matchMultipleSets(
[user_archive_names, domain_archive_names, default_archive_names], name, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Blocked filename in archive %s SILENT DELETE" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Blocked filename in archive %s" % att_name)
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE and self.config.getboolean(self.section, 'checkarchivecontent'):
for name in namelist:
safename = self.asciionly(name)
extracted = self._archive_extract(archive_type,archive_handle,name)
if extracted==None:
self._debuginfo(suspect,'%s not extracted - too large'%(safename))
contenttype_magic = self.getBuffertype(
extracted)
res = self.matchMultipleSets(
[user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
except Exception, e:
self.logger.warning(
"archive scanning failed in attachment %s: %s" % (att_name, str(e)))
def lint(self):
viract = self.config.get(self.section, 'virusaction')
print "Virusaction: %s" % actioncode_to_string(
string_to_actioncode(viract, self.config))
allok = (self.checkConfig() and self.lint_eicar())
return allok
def walk(self, suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction = self.config.get(self.section, 'blockaction')
blockactioncode = string_to_actioncode(blockaction)
# try db rules first
self.rulescache.reloadifnecessary()
dbconn = ''
if self.config.has_option(self.section, 'dbconnectstring'):
dbconn = self.config.get(self.section, 'dbconnectstring')
if dbconn.strip() != '':
self.logger.debug('Loading attachment rules from database')
query = self.config.get(self.section, 'query')
dbfile = DBFile(dbconn, query)
user_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN}))
user_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT}))
user_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
user_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' %
(len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address))
domain_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN}))
domain_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT}))
domain_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
domain_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' %
(len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain))
else:
self.logger.debug('Loading attachment rules from filesystem dir %s'%(self.config.get(self.section,'rulesdir')))
user_names = self.rulescache.getNAMERules(suspect.to_address)
user_ctypes = self.rulescache.getCTYPERules(suspect.to_address)
user_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_address)
user_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_address)
domain_names = self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain)
domain_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_domain)
domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_domain)
# always get defaults from file
default_names = self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT)
default_archive_names = self.rulescache.getARCHIVENAMERules(
FUATT_DEFAULT)
default_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
FUATT_DEFAULT)
m = suspect.get_message_rep()
for part in self.walk_all_parts(m):
if part.is_multipart():
continue
contenttype_mime = part.get_content_type()
att_name = part.get_filename(None)
if att_name:
# some filenames are encoded, try to decode
try:
att_name = ''.join([x[0] for x in decode_header(att_name)])
except Exception:
pass
else:
# workaround for mimetypes, it always takes .ksh for text/plain
if part.get_content_type() == 'text/plain':
ext = '.txt'
else:
ext = mimetypes.guess_extension(part.get_content_type())
if ext is None:
ext = ''
att_name = 'unnamed%s' % ext
att_name = self.asciionly(att_name)
res = self.matchMultipleSets(
[user_names, domain_names, default_names], att_name, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s : blocked by name)" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# go through content type rules
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
contenttype_magic = None
if MAGIC_AVAILABLE:
pl = part.get_payload(decode=True)
contenttype_magic = self.getBuffertype(pl)
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# archives
if self.checkarchivenames or self.checkarchivecontent:
# try guessing the archive type based on magic content type first
# we don't need to check for MAGIC_AVAILABLE here, if it is available the contenttype_magic is not None
archive_type = self.archive_type_from_content_type(contenttype_magic)
# if it didn't work, try to guess by the filename extension, if it is enabled
if archive_type is None:
# sort by length, so tar.gz is checked before .gz
for arext in sorted(self.supported_archive_extensions.keys(), key=lambda x: len(x), reverse=True):
if att_name.lower().endswith('.%s' % arext):
archive_type = self.supported_archive_extensions[arext]
break
if archive_type is not None:
self.logger.debug("Extracting {attname} as {artype}".format(attname=att_name,artype=archive_type))
try:
pl = BytesIO(part.get_payload(decode=True))
archive_handle = self._archive_handle(archive_type, pl)
namelist = self._archive_namelist(archive_type, archive_handle)
if self.checkarchivenames:
for name in namelist:
# rarfile returns unicode objects which mess up
# generated bounces
if sys.version_info[0] == 2:
# Py3 defaults to unicode
name = self.asciionly(name)
res = self.matchMultipleSets(
[user_archive_names, domain_archive_names, default_archive_names], name, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Blocked filename in archive %s SILENT DELETE" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Blocked filename in archive %s" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE and self.checkarchivecontent:
for name in namelist:
safename = self.asciionly(name)
extracted = self._archive_extract(archive_type, archive_handle, name)
if extracted is None:
self._debuginfo(
suspect, '%s not extracted - too large' % (safename))
contenttype_magic = self.getBuffertype(
extracted)
res = self.matchMultipleSets(
[user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
if hasattr(archive_handle, 'close'):
archive_handle.close()
pl.close()
except Exception:
self.logger.warning(
"archive scanning failed in attachment {attname}: {error}".format(attname=att_name, error=traceback.format_exc() ))
return DUNNO
def examine(self, suspect):
# check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip') is True:
self.logger.debug(
'%s Skipping SA Plugin (requested by previous plugin)' %
suspect.id)
suspect.set_tag('SAPlugin.skipreason',
'requested by previous plugin')
return DUNNO
runtimeconfig = DBConfig(self.config, suspect)
spamsize = suspect.size
maxsize = self.config.getint(self.section, 'maxsize')
spamheadername = self.config.get(self.section, 'spamheader')
if spamsize > maxsize:
self.logger.info('%s Size Skip, %s > %s' %
(suspect.id, spamsize, maxsize))
suspect.debug('Too big for spamchecks. %s > %s' %
(spamsize, maxsize))
prependheader = self.config.get('main', 'prependaddedheaders')
suspect.addheader(
"%sSA-SKIP" % prependheader,
'Too big for spamchecks. %s > %s' % (spamsize, maxsize))
suspect.set_tag('SAPlugin.skipreason', 'size skip')
return self.check_sql_blacklist(suspect)
forwardoriginal = self.config.getboolean(self.section,
'forwardoriginal')
if self.config.getboolean(self.section, 'scanoriginal'):
spam = suspect.get_original_source()
else:
spam = suspect.get_source()
# prepend temporary headers set by other plugins
tempheader = suspect.get_tag('SAPlugin.tempheader')
if tempheader is not None:
if type(tempheader) == list:
tempheader = "\r\n".join(tempheader)
tempheader = tempheader.strip()
if tempheader != '':
spam = tempheader + '\r\n' + spam
if forwardoriginal:
ret = self.safilter_report(spam, suspect.to_address)
if ret is None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
isspam, spamscore, report = ret
suspect.tags['SAPlugin.report'] = report
else:
filtered = self.safilter(spam, suspect.to_address)
if filtered is None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
else:
content = filtered
newmsgrep = email.message_from_string(content)
suspect.set_source(content)
isspam, spamscore = self._extract_spamstatus(
newmsgrep, spamheadername, suspect)
action = DUNNO
message = None
if isspam:
self.logger.debug('%s Message is spam' % suspect.id)
suspect.debug('Message is spam')
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'lowspamaction'), self.config)
if configaction is not None:
action = configaction
values = dict(spamscore=spamscore)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
values)
else:
self.logger.debug('%s Message is not spam' % suspect.id)
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin'] = isspam
suspect.tags['highspam']['SpamAssassin'] = False
if spamscore is not None:
suspect.tags['SAPlugin.spamscore'] = spamscore
highspamlevel = runtimeconfig.getfloat(self.section,
'highspamlevel')
if spamscore >= highspamlevel:
suspect.tags['highspam']['SpamAssassin'] = True
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'highspamaction'),
self.config)
if configaction is not None:
action = configaction
return action, message
def walk(self, suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction = self.config.get(self.section, 'blockaction')
blockactioncode = string_to_actioncode(blockaction)
# try db rules first
self.rulescache.reloadifnecessary()
dbconn = ''
if self.config.has_option(self.section, 'dbconnectstring'):
dbconn = self.config.get(self.section, 'dbconnectstring')
if dbconn.strip() != '':
self.logger.debug('Loading attachment rules from database')
query = self.config.get(self.section, 'query')
dbfile = DBFile(dbconn, query)
user_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_FN
}))
user_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_CT
}))
user_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_ARCHIVE_FN
}))
user_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_ARCHIVE_CT
}))
self.logger.debug(
'Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s'
% (len(user_names), len(user_ctypes), len(user_archive_names),
len(user_archive_ctypes), suspect.to_address))
domain_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_FN
}))
domain_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_CT
}))
domain_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_ARCHIVE_FN
}))
domain_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_ARCHIVE_CT
}))
self.logger.debug(
'Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s'
% (len(domain_names), len(domain_ctypes),
len(domain_archive_names), len(domain_archive_ctypes),
suspect.to_domain))
else:
self.logger.debug(
'Loading attachment rules from filesystem dir %s' %
(self.config.get(self.section, 'rulesdir')))
user_names = self.rulescache.getNAMERules(suspect.to_address)
user_ctypes = self.rulescache.getCTYPERules(suspect.to_address)
user_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_address)
user_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_address)
domain_names = self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain)
domain_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_domain)
domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_domain)
# always get defaults from file
default_names = self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT)
default_archive_names = self.rulescache.getARCHIVENAMERules(
FUATT_DEFAULT)
default_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
FUATT_DEFAULT)
m = suspect.get_message_rep()
for part in self.walk_all_parts(m):
if part.is_multipart():
continue
contenttype_mime = part.get_content_type()
att_name = part.get_filename(None)
if att_name:
# some filenames are encoded, try to decode
try:
att_name = ''.join([x[0] for x in decode_header(att_name)])
except Exception:
pass
else:
# workaround for mimetypes, it always takes .ksh for text/plain
if part.get_content_type() == 'text/plain':
ext = '.txt'
else:
ext = mimetypes.guess_extension(part.get_content_type())
if ext is None:
ext = ''
att_name = 'unnamed%s' % ext
att_name = self.asciionly(att_name)
res = self.matchMultipleSets(
[user_names, domain_names, default_names], att_name, suspect,
att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s SILENT DELETE : blocked by name" %
att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s : blocked by name)" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# go through content type rules
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_mime,
suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)"
% (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s : blocked by mime content type (message source)"
% (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
contenttype_magic = None
if MAGIC_AVAILABLE:
pl = part.get_payload(decode=True)
contenttype_magic = self.getBuffertype(pl)
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes],
contenttype_magic, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)"
% (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s : blocked by mime content type (magic)"
% (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# archives
if self.checkarchivenames or self.checkarchivecontent:
# try guessing the archive type based on magic content type first
# we don't need to check for MAGIC_AVAILABLE here, if it is available the contenttype_magic is not None
archive_type = self.archive_type_from_content_type(
contenttype_magic)
# if it didn't work, try to guess by the filename extension, if it is enabled
if archive_type is None:
# sort by length, so tar.gz is checked before .gz
for arext in sorted(
self.supported_archive_extensions.keys(),
key=lambda x: len(x),
reverse=True):
if att_name.lower().endswith('.%s' % arext):
archive_type = self.supported_archive_extensions[
arext]
break
if archive_type is not None:
self.logger.debug(
"Extracting {attname} as {artype}".format(
attname=att_name, artype=archive_type))
try:
pl = BytesIO(part.get_payload(decode=True))
archive_handle = self._archive_handle(archive_type, pl)
namelist = self._archive_namelist(
archive_type, archive_handle)
if self.checkarchivenames:
for name in namelist:
# rarfile returns unicode objects which mess up
# generated bounces
if sys.version_info[0] == 2:
# Py3 defaults to unicode
name = self.asciionly(name)
res = self.matchMultipleSets([
user_archive_names, domain_archive_names,
default_archive_names
], name, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Blocked filename in archive %s SILENT DELETE"
% att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Blocked filename in archive %s" %
att_name)
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE and self.checkarchivecontent:
for name in namelist:
safename = self.asciionly(name)
extracted = self._archive_extract(
archive_type, archive_handle, name)
if extracted is None:
self._debuginfo(
suspect,
'%s not extracted - too large' %
(safename))
contenttype_magic = self.getBuffertype(
extracted)
res = self.matchMultipleSets([
user_archive_ctypes, domain_archive_ctypes,
default_archive_ctypes
], contenttype_magic, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)"
% (safename, att_name,
contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)"
% (safename, att_name,
contenttype_magic))
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
if hasattr(archive_handle, 'close'):
archive_handle.close()
pl.close()
except Exception:
self.logger.warning(
"archive scanning failed in attachment {attname}: {error}"
.format(attname=att_name,
error=traceback.format_exc()))
return DUNNO
def walk(self,suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction=self.config.get(self.section,'blockaction')
blockactioncode=string_to_actioncode(blockaction)
#try db rules first
self.rulescache.reloadifnecessary()
dbconn=''
if self.config.has_option(self.section,'dbconnectstring'):
dbconn=self.config.get(self.section,'dbconnectstring')
if dbconn.strip()!='':
self.logger.debug('Loading attachment rules from database')
query=self.config.get(self.section,'query')
dbfile=DBFile(dbconn, query)
user_names=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_address,'checktype':FUATT_CHECKTYPE_FN}))
user_ctypes=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_address,'checktype':FUATT_CHECKTYPE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules for address %s'%(len(user_names),len(user_ctypes),suspect.to_address))
domain_names=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_domain,'checktype':FUATT_CHECKTYPE_FN}))
domain_ctypes=self.rulescache.get_rules_from_config_lines(dbfile.getContent({'scope':suspect.to_domain,'checktype':FUATT_CHECKTYPE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules for domain %s'%(len(domain_names),len(domain_ctypes),suspect.to_domain))
else:
self.logger.debug('Loading attachment rules from filesystem')
user_names=self.rulescache.getNAMERules(suspect.to_address)
user_ctypes=self.rulescache.getCTYPERules(suspect.to_address)
domain_names=self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes=self.rulescache.getCTYPERules(suspect.to_domain)
#always get defaults from file
default_names=self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes=self.rulescache.getCTYPERules(FUATT_DEFAULT)
m=suspect.get_message_rep()
for i in m.walk():
if i.is_multipart():
continue
contenttype_mime=i.get_content_type()
att_name = i.get_filename(None)
if not att_name:
#workaround for mimetypes, it always takes .ksh for text/plain
if i.get_content_type()=='text/plain':
ext='.txt'
else:
ext = mimetypes.guess_extension(i.get_content_type())
if ext==None:
ext=''
att_name = 'unnamed%s' % ext
res=self.matchMultipleSets([user_names,domain_names,default_names], att_name,suspect,att_name)
if res==ATTACHMENT_SILENTDELETE:
self._debuginfo(suspect,"Attachment name=%s SILENT DELETE : blocked by name"%att_name)
return DELETE
if res==ATTACHMENT_BLOCK:
self._debuginfo(suspect,"Attachment name=%s : blocked by name)"%att_name)
message=suspect.tags['FiletypePlugin.errormessage']
return blockactioncode,message
#go through content type rules
res=self.matchMultipleSets([user_ctypes,domain_ctypes,default_ctypes], contenttype_mime,suspect,att_name)
if res==ATTACHMENT_SILENTDELETE:
self._debuginfo(suspect,"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)"%(att_name,contenttype_mime))
return DELETE
if res==ATTACHMENT_BLOCK:
self._debuginfo(suspect,"Attachment name=%s content-type=%s : blocked by mime content type (message source)"%(att_name,contenttype_mime))
message=suspect.tags['FiletypePlugin.errormessage']
return blockactioncode,message
if MAGIC_AVAILABLE:
pl = i.get_payload(decode=True)
contenttype_magic=self.getBuffertype(pl)
res=self.matchMultipleSets([user_ctypes,domain_ctypes,default_ctypes], contenttype_magic,suspect,att_name)
if res==ATTACHMENT_SILENTDELETE:
self._debuginfo(suspect,"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)"%(att_name,contenttype_mime))
return DELETE
if res==ATTACHMENT_BLOCK:
self._debuginfo(suspect,"Attachment name=%s content-type=%s : blocked by mime content type (magic)"%(att_name,contenttype_mime))
message=suspect.tags['FiletypePlugin.errormessage']
return blockactioncode,message
return DUNNO
def walk(self, suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction = self.config.get(self.section, 'blockaction')
blockactioncode = string_to_actioncode(blockaction)
# try db rules first
self.rulescache.reloadifnecessary()
dbconn = ''
if self.config.has_option(self.section, 'dbconnectstring'):
dbconn = self.config.get(self.section, 'dbconnectstring')
if dbconn.strip() != '':
self.logger.debug('%s Loading attachment rules from database' %
suspect.id)
query = self.config.get(self.section, 'query')
dbfile = DBFile(dbconn, query)
user_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_FN
}))
user_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_CT
}))
user_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_ARCHIVE_FN
}))
user_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_address,
'checktype': FUATT_CHECKTYPE_ARCHIVE_CT
}))
self.logger.debug(
'%s Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s'
% (suspect.id, len(user_names), len(user_ctypes),
len(user_archive_names), len(user_archive_ctypes),
suspect.to_address))
domain_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_FN
}))
domain_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_CT
}))
domain_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_ARCHIVE_FN
}))
domain_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({
'scope': suspect.to_domain,
'checktype': FUATT_CHECKTYPE_ARCHIVE_CT
}))
self.logger.debug(
'%s Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s'
% (suspect.id, len(domain_names), len(domain_ctypes),
len(domain_archive_names), len(domain_archive_ctypes),
suspect.to_domain))
else:
self.logger.debug(
'%s Loading attachment rules from filesystem dir %s' %
(suspect.id, self.config.get(self.section, 'rulesdir')))
user_names = self.rulescache.getNAMERules(suspect.to_address)
user_ctypes = self.rulescache.getCTYPERules(suspect.to_address)
user_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_address)
user_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_address)
domain_names = self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain)
domain_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_domain)
domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_domain)
# always get defaults from file
default_names = self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT)
default_archive_names = self.rulescache.getARCHIVENAMERules(
FUATT_DEFAULT)
default_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
FUATT_DEFAULT)
# get mail attachment objects (only directly attached objects)
for attObj in suspect.att_mgr.get_objectlist():
contenttype_mime = attObj.contenttype_mime
att_name = attObj.filename
if attObj.is_inline or attObj.is_attachment or not attObj.filename_generated:
# process all attachments marked as "inline", "attachment" or parts
# with filenames that are not auto-generated
pass
else:
self.logger.debug(
"%s Skip message object: %s (attachment: %s, inline: %s, auto-name: %s)"
% (suspect.id, att_name, attObj.is_attachment,
attObj.is_inline, attObj.filename_generated))
continue
att_name = self.asciionly(att_name)
res = self.matchMultipleSets(
[user_names, domain_names, default_names], att_name, suspect,
att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s SILENT DELETE : blocked by name" %
att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s : blocked by name)" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# go through content type rules
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_mime,
suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)"
% (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s : blocked by mime content type (message source)"
% (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
contenttype_magic = attObj.contenttype
if contenttype_magic is not None:
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes],
contenttype_magic, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)"
% (att_name, contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Attachment name=%s content-type=%s : blocked by mime content type (magic)"
% (att_name, contenttype_magic))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# archives
if self.checkarchivenames or self.checkarchivecontent:
#if archive_type is not None:
if attObj.is_archive:
# check if extension was used to determine archive type and
# if yes, check if extension is enabled. This code
# is here to remain backward compatible in the behavior. It
# is recommended to define inactive archive-types and -extensions
# differently
if attObj.atype_fromext() is not None:
if not attObj.atype_fromext(
) in self.active_archive_extensions.keys():
# skip if extension is not in active list
continue
self.logger.debug(
"%s Extracting %s as %s" %
(suspect.id, att_name, attObj.archive_type))
archivecontentmaxsize = self.config.getint(
self.section, 'archivecontentmaxsize')
try:
archiveextractlevel = self.config.getint(
self.section, 'archiveextractlevel')
if archiveextractlevel < 0: # value must be greater or equals 0
archiveextractlevel = None
except Exception:
archiveextractlevel = None
try:
if self.checkarchivenames:
if self.checkarchivecontent:
namelist = attObj.get_fileslist(
0, archiveextractlevel,
archivecontentmaxsize)
else:
namelist = attObj.fileslist_archive
for name in namelist:
res = self.matchMultipleSets([
user_archive_names, domain_archive_names,
default_archive_names
], name, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Blocked filename in archive %s SILENT DELETE"
% att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Blocked filename in archive %s" %
att_name)
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
if filetype_handler.available(
) and self.checkarchivecontent:
nocheckinfo = NoExtractInfo()
for archObj in attObj.get_objectlist(
0,
archiveextractlevel,
archivecontentmaxsize,
noextractinfo=nocheckinfo):
safename = self.asciionly(archObj.filename)
contenttype_magic = archObj.contenttype
# Keeping this check for backward compatibility
# This could easily be removed since memory is used anyway
if archivecontentmaxsize is not None and archObj.filesize > archivecontentmaxsize:
nocheckinfo.append(
archObj.filename, u"toolarge",
u"already extracted but too large for check: %u > %u"
% (archObj.filesize,
archivecontentmaxsize))
continue
res = self.matchMultipleSets([
user_archive_ctypes, domain_archive_ctypes,
default_archive_ctypes
], contenttype_magic, suspect, safename)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect,
"Extracted file %s from archive %s content-type=%s "
"SILENT DELETE: blocked by mime content type (magic)"
% (safename, att_name,
contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect,
"Extracted file %s from archive %s content-type=%s : "
"blocked by mime content type (magic)"
% (safename, att_name,
contenttype_magic))
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
for item in nocheckinfo.get_filtered():
try:
self._debuginfo(
suspect,
'Archive File not checked: reason: %s -> %s'
% (item[0], item[1]))
except Exception as e:
self._debuginfo(
suspect,
'Archive File not checked: %s' %
str(e))
except Exception as e:
self.logger.error(
"%s archive scanning failed in attachment %s: %s" %
(suspect.id, att_name, str(e)))
return DUNNO
self.logger.debug(blvalue)
# build regex
# translate glob to regexr
# http://stackoverflow.com/questions/445910/create-regex-from-glob-expression
regexp = re.escape(blvalue).replace(
r'\?', '.').replace(r'\*', '.*?')
self.logger.debug(regexp)
pattern = re.compile(regexp)
if pattern.search(suspect.from_address):
self.logger.debug(
'Blacklist match : %s for sa pref %s' % (suspect.from_address, blvalue))
confcheck = self.config
if runtimeconfig != None:
confcheck = runtimeconfig
configaction = string_to_actioncode(
confcheck.get(self.section, 'highspamaction'), self.config)
suspect.tags['spam']['SpamAssassin'] = True
prependheader = self.config.get(
'main', 'prependaddedheaders')
suspect.addheader("%sBlacklisted" % prependheader, blvalue)
suspect.debug('Sender is Blacklisted: %s' % blvalue)
if configaction == None:
return DUNNO
return configaction
return DUNNO
def _problemcode(self):
retcode = string_to_actioncode(
self.config.get(self.section, 'problemaction'), self.config)
if retcode != None:
def examine(self, suspect):
# check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip') == True:
self.logger.debug(
'Skipping SA Plugin (requested by previous plugin)')
suspect.set_tag(
'SAPlugin.skipreason', 'requested by previous plugin')
return
runtimeconfig = DBConfig(self.config, suspect)
spamsize = suspect.size
maxsize = self.config.getint(self.section, 'maxsize')
spamheadername = self.config.get(self.section, 'spamheader')
if spamsize > maxsize:
self.logger.info('%s Size Skip, %s > %s' %
(suspect.id, spamsize, maxsize))
suspect.debug('Too big for spamchecks. %s > %s' %
(spamsize, maxsize))
prependheader = self.config.get('main', 'prependaddedheaders')
suspect.addheader(
"%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize))
suspect.set_tag('SAPlugin.skipreason', 'size skip')
return self.check_sql_blacklist(suspect)
forwardoriginal = self.config.getboolean(
self.section, 'forwardoriginal')
if self.config.getboolean(self.section, 'scanoriginal'):
spam = suspect.get_original_source()
else:
spam = suspect.get_source()
# prepend temporary headers set by other plugins
tempheader = suspect.get_tag('SAPlugin.tempheader')
if tempheader != None:
if type(tempheader) == list:
tempheader = "\r\n".join(tempheader)
tempheader = tempheader.strip()
if tempheader != '':
spam = tempheader + '\r\n' + spam
if forwardoriginal:
ret = self.safilter_report(spam, suspect.to_address)
if ret == None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
isspam, spamscore, report = ret
suspect.tags['SAPlugin.report'] = report
else:
filtered = self.safilter(spam, suspect.to_address)
content = None
if filtered == None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
else:
content = filtered
newmsgrep = email.message_from_string(content)
suspect.set_source(content)
isspam, spamscore = self._extract_spamstatus(
newmsgrep, spamheadername, suspect)
action = DUNNO
message = None
if isspam:
self.logger.debug('Message is spam')
suspect.debug('Message is spam')
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'lowspamaction'), self.config)
if configaction != None:
action = configaction
values = dict(spamscore=spamscore)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
else:
self.logger.debug('Message is not spam')
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin'] = isspam
suspect.tags['highspam']['SpamAssassin'] = False
if spamscore != None:
suspect.tags['SAPlugin.spamscore'] = spamscore
highspamlevel = runtimeconfig.getint(self.section, 'highspamlevel')
if spamscore >= highspamlevel:
suspect.tags['highspam']['SpamAssassin'] = True
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'highspamaction'), self.config)
if configaction != None:
action = configaction
return action, message
def lint(self):
viract = self.config.get(self.section, 'virusaction')
print("Virusaction: %s" % actioncode_to_string(string_to_actioncode(viract, self.config)))
allok = (self.checkConfig() and self.lint_eicar())
return allok
def examine(self,suspect):
#check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip')==True:
self.logger.debug('Skipping SA Plugin (requested by previous plugin)')
suspect.set_tag('SAPlugin.skipreason','requested by previous plugin')
return
runtimeconfig=DBConfig(self.config, suspect)
spamsize=suspect.size
maxsize=self.config.getint(self.section, 'maxsize')
spamheadername=self.config.get(self.section,'spamheader')
if spamsize>maxsize:
self.logger.info('%s Size Skip, %s > %s'%(suspect.id,spamsize,maxsize))
suspect.debug('Too big for spamchecks. %s > %s'%(spamsize,maxsize))
prependheader=self.config.get('main','prependaddedheaders')
suspect.addheader("%sSA-SKIP"%prependheader, 'Too big for spamchecks. %s > %s'%(spamsize,maxsize))
suspect.set_tag('SAPlugin.skipreason','size skip')
return self.check_sql_blacklist(suspect)
starttime=time.time()
forwardoriginal=self.config.getboolean(self.section,'forwardoriginal')
if self.config.getboolean(self.section,'scanoriginal'):
spam=suspect.getOriginalSource()
else:
spam=suspect.getSource()
#prepend temporary headers set by other plugins
tempheader=suspect.get_tag('SAPlugin.tempheader')
if tempheader!=None:
if type(tempheader)==list:
tempheader="\r\n".join(tempheader)
tempheader=tempheader.strip()
if tempheader!='':
spam=tempheader+'\r\n'+spam
if forwardoriginal:
ret=self.safilter_report(spam, suspect.to_address)
if ret==None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED'%suspect.id)
suspect.addheader('%sSA-SKIP'%self.config.get('main','prependaddedheaders'),'SA scan failed')
suspect.set_tag('SAPlugin.skipreason','scan failed')
return self._problemcode()
isspam,spamscore,report=ret
suspect.tags['SAPlugin.report']=report
else:
filtered=self.safilter(spam,suspect.to_address)
content=None
if filtered==None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED'%suspect.id)
suspect.addheader('%sSA-SKIP'%self.config.get('main','prependaddedheaders'),'SA scan failed')
suspect.set_tag('SAPlugin.skipreason','scan failed')
return self._problemcode()
else:
content=filtered
newmsgrep=email.message_from_string(content)
suspect.setSource(content)
isspam=False
spamheader=newmsgrep[spamheadername]
spamscore=None
if spamheader==None:
self.logger.warning('Did not find Header %s in returned message from SA'%spamheadername)
else:
if len(spamheader)>2 and 'yes' in spamheader.lower():
isspam=True
patt=re.compile('Score=([\-\d\.]+)',re.IGNORECASE)
m=patt.search(spamheader)
if m !=None:
spamscore=float(m.group(1))
self.logger.debug('Spamscore: %s'%spamscore)
suspect.debug('Spamscore: %s'%spamscore)
else:
self.logger.warning('Could not extract spam score from header: %s'%spamheader)
suspect.debug('Could not read spam score from header %s'%spamheader)
action=DUNNO
message=None
if isspam:
self.logger.debug('Message is spam')
suspect.debug('Message is spam')
configaction=string_to_actioncode(runtimeconfig.get(self.section,'lowspamaction'),self.config)
if configaction!=None:
action=configaction
values=dict(spamscore=spamscore)
message=apply_template(self.config.get(self.section,'rejectmessage'), suspect, values)
else:
self.logger.debug('Message is not spam')
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin']=isspam
suspect.tags['highspam']['SpamAssassin']=False
if spamscore != None:
suspect.tags['SAPlugin.spamscore']=spamscore
highspamlevel=runtimeconfig.getint(self.section,'highspamlevel')
if spamscore>=highspamlevel:
suspect.tags['highspam']['SpamAssassin']=True
configaction=string_to_actioncode(runtimeconfig.get(self.section,'highspamaction'),self.config)
if configaction!=None:
action=configaction
endtime=time.time()
difftime=endtime-starttime
suspect.tags['SAPlugin.time']="%.4f"%difftime
return action,message
def walk(self, suspect):
"""walks through a message and checks each attachment according to the rulefile specified in the config"""
blockaction = self.config.get(self.section, 'blockaction')
blockactioncode = string_to_actioncode(blockaction)
# try db rules first
self.rulescache.reloadifnecessary()
dbconn = ''
if self.config.has_option(self.section, 'dbconnectstring'):
dbconn = self.config.get(self.section, 'dbconnectstring')
if dbconn.strip() != '':
self.logger.debug('Loading attachment rules from database')
query = self.config.get(self.section, 'query')
dbfile = DBFile(dbconn, query)
user_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_FN}))
user_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_CT}))
user_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
user_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_address, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for address %s' %
(len(user_names), len(user_ctypes), len(user_archive_names), len(user_archive_ctypes), suspect.to_address))
domain_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_FN}))
domain_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_CT}))
domain_archive_names = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_FN}))
domain_archive_ctypes = self.rulescache.get_rules_from_config_lines(
dbfile.getContent({'scope': suspect.to_domain, 'checktype': FUATT_CHECKTYPE_ARCHIVE_CT}))
self.logger.debug('Found %s filename rules, %s content-type rules, %s archive filename rules, %s archive content rules for domain %s' %
(len(domain_names), len(domain_ctypes), len(domain_archive_names), len(domain_archive_ctypes), suspect.to_domain))
else:
self.logger.debug('Loading attachment rules from filesystem')
user_names = self.rulescache.getNAMERules(suspect.to_address)
user_ctypes = self.rulescache.getCTYPERules(suspect.to_address)
user_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_address)
user_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_address)
domain_names = self.rulescache.getNAMERules(suspect.to_domain)
domain_ctypes = self.rulescache.getCTYPERules(suspect.to_domain)
domain_archive_names = self.rulescache.getARCHIVENAMERules(
suspect.to_domain)
domain_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
suspect.to_domain)
# always get defaults from file
default_names = self.rulescache.getNAMERules(FUATT_DEFAULT)
default_ctypes = self.rulescache.getCTYPERules(FUATT_DEFAULT)
default_archive_names = self.rulescache.getARCHIVENAMERules(
FUATT_DEFAULT)
default_archive_ctypes = self.rulescache.getARCHIVECTYPERules(
FUATT_DEFAULT)
m = suspect.get_message_rep()
for i in m.walk():
if i.is_multipart():
continue
contenttype_mime = i.get_content_type()
att_name = i.get_filename(None)
if att_name:
# some filenames are encoded, try to decode
try:
att_name = ''.join([x[0] for x in decode_header(att_name)])
except:
pass
else:
# workaround for mimetypes, it always takes .ksh for text/plain
if i.get_content_type() == 'text/plain':
ext = '.txt'
else:
ext = mimetypes.guess_extension(i.get_content_type())
if ext == None:
ext = ''
att_name = 'unnamed%s' % ext
res = self.matchMultipleSets(
[user_names, domain_names, default_names], att_name, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s SILENT DELETE : blocked by name" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s : blocked by name)" % att_name)
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# go through content type rules
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_mime, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (message source)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (message source)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE:
pl = i.get_payload(decode=True)
contenttype_magic = self.getBuffertype(pl)
res = self.matchMultipleSets(
[user_ctypes, domain_ctypes, default_ctypes], contenttype_magic, suspect, att_name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (att_name, contenttype_mime))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Attachment name=%s content-type=%s : blocked by mime content type (magic)" % (att_name, contenttype_mime))
message = suspect.tags['FiletypePlugin.errormessage']
return blockactioncode, message
# archives
if self.config.getboolean(self.section, 'checkarchivenames') or self.config.getboolean(self.section, 'checkarchivecontent'):
archive_type = None
for arext in self.supported_archive_extensions:
if att_name.lower().endswith('.%s' % arext):
archive_type = arext
break
if archive_type != None:
try:
pl = StringIO(i.get_payload(decode=True))
archive_handle = self._archive_handle(archive_type, pl)
namelist = self._archive_namelist(
archive_type, archive_handle)
if self.config.getboolean(self.section, 'checkarchivenames'):
for name in namelist:
# rarfile returns unicode objects which mess up
# generated bounces
if type(name) == unicode:
name = name.encode("utf-8", "ignore")
res = self.matchMultipleSets(
[user_archive_names, domain_archive_names, default_archive_names], name, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Blocked filename in archive %s SILENT DELETE" % att_name)
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Blocked filename in archive %s" % att_name)
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
if MAGIC_AVAILABLE and self.config.getboolean(self.section, 'checkarchivecontent'):
for name in namelist:
safename = self.asciionly(name)
extracted = self._archive_extract(
archive_type, archive_handle, name)
if extracted == None:
self._debuginfo(
suspect, '%s not extracted - too large' % (safename))
contenttype_magic = self.getBuffertype(
extracted)
res = self.matchMultipleSets(
[user_archive_ctypes, domain_archive_ctypes, default_archive_ctypes], contenttype_magic, suspect, name)
if res == ATTACHMENT_SILENTDELETE:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s SILENT DELETE: blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
return DELETE
if res == ATTACHMENT_BLOCK:
self._debuginfo(
suspect, "Extracted file %s from archive %s content-type=%s : blocked by mime content type (magic)" % (safename, att_name, contenttype_magic))
message = suspect.tags[
'FiletypePlugin.errormessage']
return blockactioncode, message
except Exception as e:
self.logger.warning(
"archive scanning failed in attachment %s: %s" % (att_name, str(e)))
return DUNNO
def examine(self, suspect):
# check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip') is True:
self.logger.debug('%s Skipping SA Plugin (requested by previous plugin)' % suspect.id)
suspect.set_tag('SAPlugin.skipreason', 'requested by previous plugin')
return DUNNO
runtimeconfig = DBConfig(self.config, suspect)
spamsize = suspect.size
maxsize = self.config.getint(self.section, 'maxsize')
strip_oversize = self.config.getboolean(self.section, 'strip_oversize')
if spamsize > maxsize and not strip_oversize:
self.logger.info('%s Size Skip, %s > %s' % (suspect.id, spamsize, maxsize))
suspect.debug('Too big for spamchecks. %s > %s' % (spamsize, maxsize))
prependheader = self.config.get('main', 'prependaddedheaders')
suspect.addheader("%sSA-SKIP" % prependheader, 'Too big for spamchecks. %s > %s' % (spamsize, maxsize))
suspect.set_tag('SAPlugin.skipreason', 'size skip')
return self.check_sql_blacklist(suspect)
if self.config.getboolean(self.section, 'scanoriginal'):
content = suspect.get_original_source()
else:
content = suspect.get_source()
stripped = False
if spamsize > maxsize:
stripped = True
# keep copy of original content before stripping
content_orig = content
# send maxsize-1 to be consistent with previous implementation
content = suspect.source_stripped_attachments(content=content, maxsize=maxsize-1)
self.logger.info('%s stripped attachments, body size reduced from %s to %s bytes' % (suspect.id, len(content_orig), len(content)))
# stick to bytes
content = force_bString(content)
# prepend temporary headers set by other plugins
tempheader = suspect.get_tag('SAPlugin.tempheader')
if tempheader is not None:
if isinstance(tempheader, list):
tempheader = "\r\n".join(tempheader)
tempheader = tempheader.strip()
if tempheader != '':
content = force_bString(tempheader + '\r\n') + content
# add envelope sender information
msgrep = suspect.get_message_rep()
if not 'Return-Path' in msgrep.keys():
content = force_bString('Return-Path: %s' % suspect.from_address + '\r\n') + content
forwardoriginal = self.config.getboolean(self.section, 'forwardoriginal')
if forwardoriginal:
ret = self.safilter_report(content, suspect.to_address)
if ret is None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED' % suspect.id)
suspect.addheader('%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
isspam, spamscore, report = ret
suspect.tags['SAPlugin.report'] = report
else:
filtered = self.safilter(content, suspect.to_address)
if filtered is None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED' % suspect.id)
suspect.addheader('%sSA-SKIP' % self.config.get('main', 'prependaddedheaders'), 'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
else:
if stripped:
# create msgrep of filtered msg
if isinstance(content,str):
msgrep_filtered = email.message_from_string(filtered, _class=PatchedMessage)
else:
msgrep_filtered = email.message_from_bytes(filtered, _class=PatchedMessage)
header_new = []
for h,v in msgrep_filtered.items():
header_new.append(force_uString(h).strip() + ': ' + force_uString(v).strip())
# add headers to msg
sa_prepend = self.config.get(self.section, 'spamheader_prepend')
for i in header_new:
if sa_prepend == '' or sa_prepend is None:
break
if re.match('^' + sa_prepend + '[^:]+: ', i, re.I):
# in case of stripped msg add header to original content
content_orig = force_bString(i) + b'\r\n' + force_bString(content_orig)
else:
continue
content = content_orig
else:
content = filtered
if isinstance(content,str):
newmsgrep = email.message_from_string(content, _class=PatchedMessage)
else:
newmsgrep = email.message_from_bytes(content, _class=PatchedMessage)
# if original content is forwarded there's no need to reset the attachmant
# manager. Only header have been changed.
suspect.set_source(content,att_mgr_reset=(not forwardoriginal))
spamheadername = self.config.get(self.section, 'spamheader')
isspam, spamscore, report = self._extract_spamstatus(newmsgrep, spamheadername, suspect)
suspect.tags['SAPlugin.report'] = report
self.logger.debug('suspect %s %s %s %s' % (suspect.id, isspam, spamscore, suspect.get_tag('SAPlugin.report')))
action = DUNNO
message = None
if isspam:
self.logger.debug('%s Message is spam' % suspect.id)
suspect.debug('Message is spam')
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'lowspamaction'), self.config)
if configaction is not None:
action = configaction
values = dict(spamscore=spamscore)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect, values)
else:
self.logger.debug('%s Message is not spam' % suspect.id)
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin'] = isspam
suspect.tags['highspam']['SpamAssassin'] = False
if spamscore is not None:
suspect.tags['SAPlugin.spamscore'] = spamscore
highspamlevel = runtimeconfig.getfloat(self.section, 'highspamlevel')
if spamscore >= highspamlevel:
suspect.tags['highspam']['SpamAssassin'] = True
configaction = string_to_actioncode(runtimeconfig.get(self.section, 'highspamaction'), self.config)
if configaction is not None:
action = configaction
return action, message
# build regex
# translate glob to regexr
# http://stackoverflow.com/questions/445910/create-regex-from-glob-expression
regexp = re.escape(blvalue).replace(r'\?',
'.').replace(r'\*', '.*?')
self.logger.debug(regexp)
pattern = re.compile(regexp)
if pattern.search(suspect.from_address):
self.logger.debug('Blacklist match : %s for sa pref %s' %
(suspect.from_address, blvalue))
confcheck = self.config
if runtimeconfig != None:
confcheck = runtimeconfig
configaction = string_to_actioncode(
confcheck.get(self.section, 'highspamaction'),
self.config)
suspect.tags['spam']['SpamAssassin'] = True
prependheader = self.config.get('main',
'prependaddedheaders')
suspect.addheader("%sBlacklisted" % prependheader, blvalue)
suspect.debug('Sender is Blacklisted: %s' % blvalue)
if configaction == None:
return DUNNO
return configaction
return DUNNO
def _problemcode(self):
retcode = string_to_actioncode(
self.config.get(self.section, 'problemaction'), self.config)
def examine(self, suspect):
# check if someone wants to skip sa checks
if suspect.get_tag('SAPlugin.skip') is True:
self.logger.debug(
'%s Skipping SA Plugin (requested by previous plugin)' %
suspect.id)
suspect.set_tag('SAPlugin.skipreason',
'requested by previous plugin')
return DUNNO
runtimeconfig = DBConfig(self.config, suspect)
spamsize = suspect.size
maxsize = self.config.getint(self.section, 'maxsize')
strip_oversize = self.config.getboolean(self.section, 'strip_oversize')
if spamsize > maxsize and not strip_oversize:
self.logger.info('%s Size Skip, %s > %s' %
(suspect.id, spamsize, maxsize))
suspect.debug('Too big for spamchecks. %s > %s' %
(spamsize, maxsize))
prependheader = self.config.get('main', 'prependaddedheaders')
suspect.addheader(
"%sSA-SKIP" % prependheader,
'Too big for spamchecks. %s > %s' % (spamsize, maxsize))
suspect.set_tag('SAPlugin.skipreason', 'size skip')
return self.check_sql_blacklist(suspect)
if self.config.getboolean(self.section, 'scanoriginal'):
content = suspect.get_original_source()
else:
content = suspect.get_source()
stripped = False
if spamsize > maxsize:
stripped = True
# keep copy of original content before stripping
content_orig = content
content = self._strip_attachments(content, maxsize)
self.logger.info(
'%s stripped attachments, body size reduced from %s to %s bytes'
% (suspect.id, len(content_orig), len(content)))
# stick to bytes
content = force_bString(content)
# prepend temporary headers set by other plugins
tempheader = suspect.get_tag('SAPlugin.tempheader')
if tempheader is not None:
if isinstance(tempheader, list):
tempheader = "\r\n".join(tempheader)
tempheader = tempheader.strip()
if tempheader != '':
content = force_bString(tempheader + '\r\n') + content
forwardoriginal = self.config.getboolean(self.section,
'forwardoriginal')
if forwardoriginal:
ret = self.safilter_report(content, suspect.to_address)
if ret is None:
suspect.debug('SA report Scan failed - please check error log')
self.logger.error('%s SA report scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
isspam, spamscore, report = ret
suspect.tags['SAPlugin.report'] = report
else:
filtered = self.safilter(content, suspect.to_address)
if filtered is None:
suspect.debug('SA Scan failed - please check error log')
self.logger.error('%s SA scan FAILED' % suspect.id)
suspect.addheader(
'%sSA-SKIP' %
self.config.get('main', 'prependaddedheaders'),
'SA scan failed')
suspect.set_tag('SAPlugin.skipreason', 'scan failed')
return self._problemcode()
else:
if stripped:
# create msgrep of filtered msg
msgrep_filtered = email.message_from_string(filtered)
header_new = []
header_old = []
# create a msgrep from original msg
msgrep_orig = email.message_from_string(content_orig)
# read all headers from after-scan and before-scan
for h, v in msgrep_filtered.items():
header_new.append(h.strip() + ': ' + v.strip())
for h, v in msgrep_orig.items():
header_old.append(h.strip() + ': ' + v.strip())
# create a list of headers added by spamd
# header diff between before-scan and after-scan msg
header_new = reversed(self.diff(header_new, header_old))
# add headers to msg
for i in header_new:
if re.match('^Received: ', i, re.I):
continue
# in case of stripped msg add header to original content
content_orig = i + '\r\n' + content_orig
content = content_orig
else:
content = filtered
if sys.version_info > (3, ):
# Python 3 and larger
# the basic "str" type is unicode
if isinstance(content, str):
newmsgrep = email.message_from_string(content)
else:
newmsgrep = email.message_from_bytes(content)
else:
# Python 2.x
newmsgrep = email.message_from_string(content)
suspect.set_source(content)
spamheadername = self.config.get(self.section, 'spamheader')
isspam, spamscore, report = self._extract_spamstatus(
newmsgrep, spamheadername, suspect)
suspect.tags['SAPlugin.report'] = report
self.logger.debug('suspect %s %s %s %s' %
(suspect.id, isspam, spamscore,
suspect.get_tag('SAPlugin.report')))
action = DUNNO
message = None
if isspam:
self.logger.debug('%s Message is spam' % suspect.id)
suspect.debug('Message is spam')
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'lowspamaction'), self.config)
if configaction is not None:
action = configaction
values = dict(spamscore=spamscore)
message = apply_template(
self.config.get(self.section, 'rejectmessage'), suspect,
values)
else:
self.logger.debug('%s Message is not spam' % suspect.id)
suspect.debug('Message is not spam')
suspect.tags['spam']['SpamAssassin'] = isspam
suspect.tags['highspam']['SpamAssassin'] = False
if spamscore is not None:
suspect.tags['SAPlugin.spamscore'] = spamscore
highspamlevel = runtimeconfig.getfloat(self.section,
'highspamlevel')
if spamscore >= highspamlevel:
suspect.tags['highspam']['SpamAssassin'] = True
configaction = string_to_actioncode(
runtimeconfig.get(self.section, 'highspamaction'),
self.config)
if configaction is not None:
action = configaction
return action, message
