def examine(self, suspect):
starttime = time.time()
if self.filter == None:
self.filter = SuspectFilter(
self.config.get(self.section, 'filterfile'))
hits = self.filter.get_args(suspect, extended=True)
if len(hits) == 0:
return DUNNO
#open file
ofile = self.config.get(self.section, 'outputfile')
if ofile.strip() == '':
self._logger().error("No output file specified for headerwriter")
return DUNNO
fh = open(ofile, 'a')
for hit in hits:
(fieldname, matchedvalue, arg, regex) = hit
if arg == None or arg == '':
arg = self.config.get(self.section, 'defaultlinetemplate')
addvalues = dict(fieldname=fieldname,
matchedvalue=matchedvalue,
regex=regex)
outputline = apply_template(arg, suspect, addvalues)
fh.write(outputline)
fh.write('\n')
fh.close()
def lint_imap(self):
#read file, check for all imap accounts
imapcopyrules = self.config.get(self.section, 'imapcopyrules')
if imapcopyrules != '' and not os.path.exists(imapcopyrules):
print "Imap copy rules file does not exist : %s" % imapcopyrules
return False
filter = SuspectFilter(imapcopyrules)
accounts = []
for tup in filter.patterns:
(headername, pattern, arg) = tup
if arg not in accounts:
if arg == None:
print "Rule %s %s has no imap copy target" % (
headername, pattern.pattern)
return False
if arg.lower() == 'no':
continue
accounts.append(arg)
for acc in accounts:
p = urlparse(acc)
host = p.hostname
username = p.username
folder = p.path[1:]
print "Checking %s@%s/%s" % (username, host, folder)
imap = self.imapconnect(acc, lintmode=True)
if not imap:
print "Lint failed for this account"
return False
return True
def examine(self, suspect):
archiverules = self.config.get(self.section, 'archiverules')
if archiverules == None or archiverules == "":
return DUNNO
if not os.path.exists(archiverules):
self.logger.error('Archive Rules file does not exist : %s' %
archiverules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(archiverules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg != None and arg.lower() == 'no':
suspect.debug("Suspect matches archive exception rule")
self.logger.debug(
"""Header matches archive exception rule - not archiving"""
)
else:
if arg != None and arg.lower() != 'yes':
self.logger.warning(
"Unknown archive action '%s' assuming 'yes'" % arg)
self.logger.debug("""Header matches archive rule""")
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches archiving rule (i would archive it if we weren't in debug mode)"
)
else:
self.archive(suspect)
else:
suspect.debug(
"No archive rule/exception rule applies to this message")
def __init__(self, msg):
self.debug = []
self.digest = None
self.predigest = None
self.bodytext_size = 0
self.filter = SuspectFilter(None)
self.logger = logging.getLogger('fuglu.plugins.fuzor.Digest')
# digest config
self.LONG_WORD_THRESHOLD = 10 # what is considered a long word
self.REPLACE_LONG_WORD = '[LONG]' # Replace long words in pre-digest with... None to disable
self.REPLACE_EMAIL = '[EMAIL]' # Replace email addrs in pre-digest with... None to disable
self.REPLACE_URL = '[LINK]' # Replace urls in pre-digest with... None to disable
self.INCLUDE_ATTACHMENT_CONTENT = False # should non-text attachment contents be included in digest (not recommended, there are better attachment hash systems)
self.INCLUDE_ATTACHMENT_COUNT = True # should the number of non-text-attachments be included in the digest
self.MINIMUM_PREDIGEST_SIZE = 27 # if the predigest is smaller than this, ignore this message
self.MINIMUM_UNMODIFIED_CONTENT = 27 # minimum unmodified content after stripping, eg. [SOMETHING] removed from the predigest (27>'von meinem Iphone gesendet')
self.MINIMUM_BODYTEXT_SIZE = 27 # if the body text content is smaller than this, ignore this message
self.STRIP_WHITESPACE = True # remove all whitespace from the pre-digest
self.STRIP_HTML_MARKUP = True # remove html tags (but keep content)
self.REMOVE_HTML_TAGS = [
'script',
'style'] # strip tags (including content)
self.predigest = self._make_predigest(msg)
self.digest = self._make_hash(self.predigest)
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
'limiterfile': {
'default': '/etc/fuglu/ratelimit.conf',
'description': 'file based rate limits',
},
'backendtype': {
'default':
'memory',
'description':
'type of backend where the events are stored. memory is only recommended for low traffic standalone systems. alternatives are: redis, sqlalchemy'
},
'backendconfig': {
'default':
'',
'description':
'backend specific configuration. sqlalchemy: the database url, redis: hostname:port:db'
}
}
self.logger = self._logger()
self.backend_instance = None
self.limiters = None
self.filter = SuspectFilter(None)
def _initfilter(self):
if self.filter is not None:
return True
filename = self.config.get(self.section, 'filterfile')
if filename is None or filename == "":
return False
if not os.path.exists(filename):
self.logger.error('Filterfile not found for skipper: %s' % filename)
return False
self.filter = SuspectFilter(filename)
return True
def examine(self, suspect):
actionrules = self.config.get(self.section, 'actionrules')
if actionrules == None or actionrules == "":
return DUNNO
if not os.path.exists(actionrules):
self.logger.error('Action Rules file does not exist : %s' %
actionrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(actionrules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg == None or arg.strip() == '':
self.logger.error("Rule match but no action defined.")
return DUNNO
arg = arg.strip()
spl = arg.split(None, 1)
actionstring = spl[0]
message = None
if len(spl) == 2:
message = spl[1]
self.logger.debug("%s: Rule match! Action override: %s" %
(suspect.id, arg.upper()))
actioncode = string_to_actioncode(actionstring, self.config)
if actioncode != None:
return actioncode, message
elif actionstring.upper() == 'REDIRECT':
suspect.to_address = message.strip()
suspect.recipients = [
suspect.to_address,
]
# todo: should we override to_domain? probably not
# todo: check for invalid adress, multiple adressses
# todo: document redirect action
else:
self.logger.error("Invalid action: %s" % arg)
return DUNNO
return DUNNO
def __init__(self):
self.requiredvars = {
'backendtype': {
'default':
'redis',
'description':
'Token store backend type. Allowed values are: sqlalchemy , redis',
},
'backendconfig': {
'default':
'',
'description':
'Backend configuration. Depends on backendtype, eg. sqlalchemy url, redis host:port:db',
},
'spambias': {
'default':
'0.5',
'description':
'overall spam bias. 0.5=no bias. 0.8=around 80% of scanned mail traffic is spam',
},
'minimum-token-occurence': {
'default':
'3',
'description':
"don't make assumptions on tokens seen less than this amount",
},
'maximum-tokens-per-message': {
'default': '5000',
'description': 'stop tokenizing after x tokens',
},
'minimum-ham': {
'default': '10',
'description': "minimum known hams for classification",
},
'minimum-spam': {
'default': '10',
'description': "minimum known spams for classification",
},
}
self.tokenstore = None
self.calc_minimum = 0.00000001 # work around division by zero etc
self.logger = self._logger()
self.filter = SuspectFilter(None)
def examine(self, suspect):
imapcopyrules = self.config.get(self.section, 'imapcopyrules')
if imapcopyrules == None or imapcopyrules == "":
return DUNNO
if not os.path.exists(imapcopyrules):
self._logger().error('IMAP copy rules file does not exist : %s' %
imapcopyrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(imapcopyrules)
(match, info) = self.filter.matches(suspect, extended=True)
if match:
field, matchedvalue, arg, regex = info
if arg != None and arg.lower() == 'no':
suspect.debug("Suspect matches imap copy exception rule")
self.logger.info(
"""%s: Header %s matches imap copy exception rule '%s' """
% (suspect.id, field, regex))
else:
if arg == None or (not arg.lower().startswith('imap')):
self.logger.error(
"Unknown target format '%s' should be 'imap(s)://user:pass@host/folder'"
% arg)
else:
self.logger.info(
"""%s: Header %s matches imap copy rule '%s' """ %
(suspect.id, field, regex))
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches imap copy rule (I would copy it if we weren't in debug mode)"
)
else:
self.storeimap(suspect, arg)
else:
suspect.debug(
"No imap copy rule/exception rule applies to this message")
def examine(self,suspect):
starttime=time.time()
filterfile=self.config.get(self.section, 'filterfile','').strip()
if self.filter==None:
if filterfile!='':
if not os.path.exists(filterfile):
self._logger().warning('LDA filter rules file does not exist : %s'%filterfile)
return DEFER
self.filter=SuspectFilter(filterfile)
if self.filter!=None:
match=self.filter.matches(suspect)
if not match:
return DUNNO
self.boxtypemap[self.config.get(self.section, 'boxtype')](suspect)
#For debugging, its good to know how long each plugin took
endtime=time.time()
difftime=endtime-starttime
suspect.tags['LDAPlugin.time']="%.4f"%difftime
def setUp(self):
self.candidate = SuspectFilter(TESTDATADIR + '/headertest.regex')
def lint_filter(self):
filterfile = self.config.get(self.section, 'filterfile')
filter = SuspectFilter(filterfile)
return filter.lint()
def lint_filter(self):
filterfile = self.config.get(self.section, 'actionrules')
sfilter = SuspectFilter(filterfile)
return sfilter.lint()
