class LDAPlugin(ScannerPlugin):
"""Deliver message to maildir / mbox"""
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.requiredvars={
'path':{
'default':'/usr/local/fuglu/deliver/${to_address}',
'description':'Path to maildir / mbox file, supports templates',
},
#maybe we need to support our own locking later, for now we use python's built-ins
#'locktype':{
# 'default':'',
# 'description':"flock, ...",
#},
'boxtype':{
'default':'mbox',
'description':"mbox, maildir",
},
#maybe we need to support various mbox types later, for now we use python's built-in module
#'subtype':{
# 'default':'',
# 'description':"what type of mbox... ",
#},
'filterfile':{
'default':'',
'description':"only store messages which use filter...",
},
}
self.logger=self._logger()
self.filter=None
self.boxtypemap={
'mbox':self.deliver_mbox,
'maildir':self.deliver_maildir,
}
def lint(self):
allok=self.checkConfig()
filterfile=self.config.get(self.section, 'filterfile','').strip()
if filterfile!='' and not os.path.exists(filterfile):
print 'LDA filter rules file does not exist : %s'%filterfile
allok=False
boxtype=self.config.get(self.section, 'boxtype')
if boxtype not in self.boxtypemap:
print "Unsupported boxtype: %s"%boxtype
allok=False
return allok
def examine(self,suspect):
starttime=time.time()
filterfile=self.config.get(self.section, 'filterfile','').strip()
if self.filter==None:
if filterfile!='':
if not os.path.exists(filterfile):
self._logger().warning('LDA filter rules file does not exist : %s'%filterfile)
return DEFER
self.filter=SuspectFilter(filterfile)
if self.filter!=None:
match=self.filter.matches(suspect)
if not match:
return DUNNO
self.boxtypemap[self.config.get(self.section, 'boxtype')](suspect)
#For debugging, its good to know how long each plugin took
endtime=time.time()
difftime=endtime-starttime
suspect.tags['LDAPlugin.time']="%.4f"%difftime
def deliver_mbox(self,suspect):
mbox_msg=mailbox.mboxMessage(suspect.get_message_rep())
mbox_path=apply_template(self.config.get(self.section,'path'), suspect)
mbox=mailbox.mbox( mbox_path)
try:
mbox.lock()
mbox.add(mbox_msg)
mbox.flush()
except Exception,e:
self.logger.error("Could not store message %s to %s: %s"%(suspect.id,mbox_path,str(e)))
finally:
class ActionOverridePlugin(ScannerPlugin):
""" Override actions based on a Suspect Filter file. For example, delete all messages from a specific sender domain. """
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'actionrules': {
'default': '/etc/fuglu/actionrules.regex',
'description': 'Rules file',
}
}
self.filter = None
def __str__(self):
return "Action Override"
def lint(self):
allok = (self.checkConfig() and self.lint_filter())
return allok
def lint_filter(self):
filterfile = self.config.get(self.section, 'actionrules')
filter = SuspectFilter(filterfile)
return filter.lint()
def examine(self, suspect):
actionrules = self.config.get(self.section, 'actionrules')
if actionrules == None or actionrules == "":
return DUNNO
if not os.path.exists(actionrules):
self.logger.error(
'Action Rules file does not exist : %s' % actionrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(actionrules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg == None or arg.strip() == '':
self.logger.error("Rule match but no action defined.")
return DUNNO
arg = arg.strip()
spl = arg.split(None, 1)
actionstring = spl[0]
message = None
if len(spl) == 2:
message = spl[1]
self.logger.debug(
"%s: Rule match! Action override: %s" % (suspect.id, arg.upper()))
actioncode = string_to_actioncode(actionstring, self.config)
if actioncode != None:
return actioncode, message
elif actionstring.upper() == 'REDIRECT':
suspect.to_address = message.strip()
suspect.recipients = [suspect.to_address, ]
# todo: should we override to_domain? probably not
# todo: check for invalid adress, multiple adressses
# todo: document redirect action
else:
self.logger.error("Invalid action: %s" % arg)
return DUNNO
return DUNNO
class SuspectFilterTestCase(unittest.TestCase):
"""Test Suspectfilter"""
def setUp(self):
self.candidate = SuspectFilter(TESTDATADIR + '/headertest.regex')
def tearDown(self):
pass
def test_sf_get_args(self):
"""Test SuspectFilter files"""
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
suspect.tags['testtag'] = 'testvalue'
headermatches = self.candidate.get_args(suspect)
self.assertTrue('Sent to unittest domain!' in headermatches,
"To_domain not found in headercheck")
self.assertTrue(
'Envelope sender is [email protected]' in headermatches,
"Envelope Sender not matched in header chekc")
self.assertTrue('Mime Version is 1.0' in headermatches,
"Standard header Mime Version not found")
self.assertTrue('A tag match' in headermatches,
"Tag match did not work")
self.assertTrue('Globbing works' in headermatches,
"header globbing failed")
self.assertTrue('body rule works' in headermatches,
"decoded body rule failed")
self.assertTrue('full body rule works' in headermatches,
"full body failed")
self.assertTrue('mime rule works' in headermatches, "mime rule failed")
self.assertFalse(
'this should not match in a body rule' in headermatches,
'decoded body rule matched raw body')
# perl style advanced rules
self.assertTrue('perl-style /-notation works!' in headermatches,
"new rule format failed: %s" % headermatches)
self.assertTrue(
'perl-style recipient match' in headermatches,
"new rule format failed for to_domain: %s" % headermatches)
self.assertFalse('this should not match' in headermatches,
"rule flag ignorecase was not detected")
# TODO: raw body rules
def test_sf_matches(self):
"""Test SuspectFilter extended matches"""
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
(match, info) = self.candidate.matches(suspect, extended=True)
self.assertTrue(match, 'Match should return True')
field, matchedvalue, arg, regex = info
self.assertTrue(field == 'to_domain')
self.assertTrue(matchedvalue == 'unittests.fuglu.org')
self.assertTrue(arg == 'Sent to unittest domain!')
self.assertTrue(regex == 'unittests\.fuglu\.org')
def test_sf_get_field(self):
"""Test SuspectFilter field extract"""
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
# additional field tests
self.assertEqual(
self.candidate.get_field(suspect, 'clienthelo')[0], 'helo1')
self.assertEqual(
self.candidate.get_field(suspect, 'clientip')[0], '10.0.0.1')
self.assertEqual(
self.candidate.get_field(suspect, 'clienthostname')[0], 'rdns1')
def test_strip(self):
html = """foo<a href="bar">bar</a><script language="JavaScript">echo('hello world');</script>baz"""
declarationtest = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de">
<head>
<title>greetings</title>
</head>
<body>
<font color="red">well met!</font>
</body>
</html>
"""
# word generated empty message
wordhtml = """<html xmlns:v=3D"urn:schemas-microsoft-com:vml"
xmlns:o=3D"urn:schemas-microsoft-com:office:office"
xmlns:w=3D"urn:schemas-microsoft-com:office:word"
xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml"
xmlns=3D"http://www.w3.org/TR/REC-html40"><head><META
HTTP-EQUIV=3D"Content-Type" CONTENT=3D"text/html;
charset=3Dus-ascii"><meta name=3DGenerator content=3D"Microsoft Word 15
(filtered medium)"><style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
span.E-MailFormatvorlage17
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:70.85pt 70.85pt 2.0cm 70.85pt;}
div.WordSection1
{page:WordSection1;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext=3D"edit">
<o:idmap v:ext=3D"edit" data=3D"1" />
</o:shapelayout></xml><![endif]--></head><body lang=3DDE-CH
link=3D"#0563C1" vlink=3D"#954F72"><div class=3DWordSection1><p
class=3DMsoNormal><o:p> </o:p></p></div></body></html>"""
for use_bfs in [True, False]:
stripped = self.candidate.strip_text(html, use_bfs=use_bfs)
self.assertEqual(stripped, 'foobarbaz')
docstripped = self.candidate.strip_text(declarationtest,
use_bfs=use_bfs)
self.assertEqual(docstripped.split(),
['greetings', 'well', 'met!'])
wordhtmstripped = self.candidate.strip_text(wordhtml,
use_bfs=use_bfs)
self.assertEqual(wordhtmstripped.strip(), '')
class IMAPCopyPlugin(ScannerPlugin):
"""This plugins stores a copy of the message to an IMAP mailbox if it matches certain criteria (Suspect Filter).
The rulefile works similar to the archive plugin. As third column you have to provide imap account data in the form:
<protocol>://<username>:<password>@<servernameorip>[:port]/<mailbox>
<protocol> is either imap or imaps
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
'imapcopyrules': {
'default': '/etc/fuglu/imapcopy.regex',
'description': 'IMAP copy suspectFilter File',
},
'storeoriginal': {
'default':
'1',
'description':
"if true/1/yes: store original message\nif false/0/no: store message probably altered by previous plugins, eg with spamassassin headers",
}
}
self.filter = None
self.logger = self._logger()
def examine(self, suspect):
imapcopyrules = self.config.get(self.section, 'imapcopyrules')
if imapcopyrules == None or imapcopyrules == "":
return DUNNO
if not os.path.exists(imapcopyrules):
self._logger().error('IMAP copy rules file does not exist : %s' %
imapcopyrules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(imapcopyrules)
(match, info) = self.filter.matches(suspect, extended=True)
if match:
field, matchedvalue, arg, regex = info
if arg != None and arg.lower() == 'no':
suspect.debug("Suspect matches imap copy exception rule")
self.logger.info(
"""%s: Header %s matches imap copy exception rule '%s' """
% (suspect.id, field, regex))
else:
if arg == None or (not arg.lower().startswith('imap')):
self.logger.error(
"Unknown target format '%s' should be 'imap(s)://user:pass@host/folder'"
% arg)
else:
self.logger.info(
"""%s: Header %s matches imap copy rule '%s' """ %
(suspect.id, field, regex))
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches imap copy rule (I would copy it if we weren't in debug mode)"
)
else:
self.storeimap(suspect, arg)
else:
suspect.debug(
"No imap copy rule/exception rule applies to this message")
def imapconnect(self, imapurl, lintmode=False):
p = urlparse(imapurl)
scheme = p.scheme.lower()
host = p.hostname
port = p.port
username = p.username
password = p.password
folder = p.path[1:]
if scheme == 'imaps':
ssl = True
else:
ssl = False
if port == None:
if ssl:
port = imaplib.IMAP4_SSL_PORT
else:
port = imaplib.IMAP4_PORT
try:
if ssl:
imap = imaplib.IMAP4_SSL(host=host, port=port)
else:
imap = imaplib.IMAP4(host=host, port=port)
except Exception, e:
ltype = 'IMAP'
if ssl:
ltype = 'IMAP-SSL'
msg = "%s Connection to server %s failed: %s" % (ltype, host,
str(e))
if lintmode:
print msg
else:
self.logger.error(msg)
return None
try:
imap.login(username, password)
except Exception, e:
msg = "Login to server %s failed: %s" % (host, str(e))
if lintmode:
print msg
else:
self.logger.error(msg)
return None
class ActionOverridePlugin(ScannerPlugin):
""" Override actions based on a Suspect Filter file. For example, delete all messages from a specific sender domain. """
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.logger = self._logger()
self.requiredvars = {
'actionrules': {
'default': '/etc/fuglu/actionrules.regex',
'description': 'Rules file',
}
}
self.filter = None
def __str__(self):
return "Action Override"
def lint(self):
allok = self.check_config() and self.lint_filter()
return allok
def lint_filter(self):
filterfile = self.config.get(self.section, 'actionrules')
sfilter = SuspectFilter(filterfile)
return sfilter.lint()
def examine(self, suspect):
actionrules = self.config.get(self.section, 'actionrules')
if actionrules is None or actionrules == "":
return DUNNO
if not os.path.exists(actionrules):
self.logger.error('Action Rules file does not exist : %s' %
actionrules)
return DUNNO
if self.filter is None:
self.filter = SuspectFilter(actionrules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg is None or arg.strip() == '':
self.logger.error("Rule match but no action defined.")
return DUNNO
arg = arg.strip()
spl = arg.split(None, 1)
actionstring = spl[0]
message = None
if len(spl) == 2:
message = spl[1]
self.logger.debug("%s: Rule match! Action override: %s" %
(suspect.id, arg.upper()))
actioncode = string_to_actioncode(actionstring, self.config)
if actioncode is not None:
return actioncode, message
elif actionstring.upper() == 'REDIRECT':
suspect.to_address = message.strip()
# todo: check for invalid adress, multiple adressses, set suspect.recipients instead of to_address
# todo: document redirect action
else:
self.logger.error("Invalid action: %s" % arg)
return DUNNO
return DUNNO
class ArchivePlugin(ScannerPlugin):
"""This plugins stores a copy of the message if it matches certain criteria (Suspect Filter).
You can use this if you want message archives for your domains or to debug problems occuring only for certain recipients.
Examples for the archive.regex filter file:
Archive messages to domain ''test.com'':
``to_domain test\.com``
Archive messages from [email protected]:
``envelope_from oli@fuglu\.org``
you can also append "yes" and "no" to the rules to create a more advanced configuration. Lets say we want to archive all messages to [email protected] and all regular messages [email protected] except the ones created by automated scripts like logwatch or daily backup messages etc.
envelope_from logwatch@.*fuglu.org no
envelope_to sales@fuglu\.org yes
from [email protected] no
envelope_to support@fuglu\.org yes
Note: The first rule to match in a message is the only rule that will be applied. Exclusion rules should therefore be put above generic/catch-all rules.
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
'archiverules': {
'default': '/etc/fuglu/archive.regex',
'description': 'Archiving SuspectFilter File',
},
'archivedir': {
'default': '/tmp',
'description': 'storage for archived messages',
},
'subdirtemplate': {
'default': '${to_domain}',
'description': 'subdirectory within archivedir',
},
'filenametemplate': {
'default': '${id}.eml',
'description': 'filename template for the archived messages',
},
'storeoriginal': {
'default': '1',
'description': "if true/1/yes: store original message\nif false/0/no: store message probably altered by previous plugins, eg with spamassassin headers",
},
'chown': {
'default': '',
'description': "change owner of saved messages (username or numeric id) - this only works if fuglu is running as root (which is NOT recommended)",
},
'chgrp': {
'default': '',
'description': "change group of saved messages (groupname or numeric id) - the user running fuglu must be a member of the target group for this to work",
},
'chmod': {
'default': '',
'description': "set file permissions of saved messages",
},
}
self.filter = None
self.logger = self._logger()
def __str__(self):
return "Archive"
def lint(self):
allok = (
self.checkConfig() and self.check_deprecated() and self.lint_dirs() and self.lint_filter())
return allok
def check_deprecated(self):
if self.config.has_option(self.section, 'makedomainsubdir'):
print(
"the config option 'makedomainsubdir' has been replaced with 'subdirtemplate' ")
print("please update your config")
print("makedomainsubdir=1 -> subdirtemplate=${to_domain}")
print("makedomainsubdir=0 -> subdirtemplate=")
return False
return True
def lint_filter(self):
filterfile = self.config.get(self.section, 'archiverules')
filter = SuspectFilter(filterfile)
return filter.lint()
def lint_dirs(self):
archivedir = self.config.get(self.section, 'archivedir')
if archivedir == "":
print('Archivedir is not specified')
return False
if not os.path.isdir(archivedir):
print("Archivedir '%s' does not exist or is not a directory" %
(archivedir))
return False
return True
def examine(self, suspect):
archiverules = self.config.get(self.section, 'archiverules')
if archiverules == None or archiverules == "":
return DUNNO
if not os.path.exists(archiverules):
self.logger.error(
'Archive Rules file does not exist : %s' % archiverules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(archiverules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg != None and arg.lower() == 'no':
suspect.debug("Suspect matches archive exception rule")
self.logger.debug(
"""Header matches archive exception rule - not archiving""")
else:
if arg != None and arg.lower() != 'yes':
self.logger.warning(
"Unknown archive action '%s' assuming 'yes'" % arg)
self.logger.debug("""Header matches archive rule""")
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches archiving rule (i would archive it if we weren't in debug mode)")
else:
self.archive(suspect)
else:
suspect.debug(
"No archive rule/exception rule applies to this message")
def archive(self, suspect):
archivedir = self.config.get(self.section, 'archivedir')
if archivedir == "":
self.logger.error('Archivedir is not specified')
return
subdirtemplate = self.config.get(self.section, 'subdirtemplate')
if self.config.has_option(self.section, 'makedomainsubdir') and subdirtemplate == self.requiredvars['subdirtemplate']['default']:
self.logger.warning(
"Archive config is using deprecated 'makedomainsubdir' config option. Emulating old behaviour. Update your config(subdirtemplate)")
if self.config.getboolean(self.section, 'makedomainsubdir'):
subdirtemplate = "${to_domain}"
else:
subdirtemplate = ""
# the archive root dir
startdir = os.path.abspath(archivedir)
# relative dir within archive root
subdir = apply_template(subdirtemplate, suspect)
if subdir.endswith('/'):
subdir = subdir[:-1]
# filename without dir
filenametemplate = self.config.get(self.section, 'filenametemplate')
filename = apply_template(filenametemplate, suspect)
# make sure filename can't create new folders
filename = filename.replace('/', '_')
# full relative filepath within archive dir
fpath = "%s/%s" % (subdir, filename)
# absolute final filepath
requested_path = os.path.abspath("%s/%s" % (startdir, fpath))
if not os.path.commonprefix([requested_path, startdir]).startswith(startdir):
self.logger.error(
"file path '%s' seems to be outside archivedir '%s' - storing to archivedir" % (requested_path, startdir))
requested_path = "%s/%s" % (startdir, filename)
finaldir = os.path.dirname(requested_path)
if not os.path.isdir(finaldir):
os.makedirs(finaldir, 0o755)
if self.config.getboolean(self.section, 'storeoriginal'):
shutil.copy(suspect.tempfile, requested_path)
else:
with open(requested_path, 'w') as fp:
fp.write(suspect.get_source())
chmod = self.config.get(self.section, 'chmod')
chgrp = self.config.get(self.section, 'chgrp')
chown = self.config.get(self.section, 'chown')
if chmod or chgrp or chown:
self.setperms(requested_path, chmod, chgrp, chown)
self.logger.info('Message from %s to %s archived as %s' % (
suspect.from_address, suspect.to_address, requested_path))
return requested_path
def setperms(self, filename, chmod, chgrp, chown):
"""Set file permissions and ownership
:param filename The target file
:param chmod string representing the permissions (example '640')
:param chgrp groupname or group id of the target group. the user running fuglu must be a member of this group for this to work
:param chown username or user id of the target user. fuglu must run as root for this to work (which is not recommended for security reasons)
"""
# chmod
if chmod:
perm = int(chmod, 8)
try:
os.chmod(filename, perm)
except:
self.logger.error(
'could not set permission on file %s' % filename)
# chgrp
changetogroup = -1
if chgrp:
group = None
try:
group = grp.getgrnam(chgrp)
except KeyError:
pass
try:
group = grp.getgrgid(int(chgrp))
except KeyError:
pass
except ValueError:
pass
if group != None:
changetogroup = group.gr_gid
else:
self.logger.warn("Group %s not found" % chgrp)
# chown
changetouser = -1
if chown:
user = None
try:
user = pwd.getpwnam(chown)
except KeyError:
pass
try:
user = pwd.getpwuid(int(chown))
except KeyError:
pass
except ValueError:
pass
if user != None:
changetouser = user.pw_uid
else:
self.logger.warn("User %s not found" % chown)
if changetogroup != -1 or changetouser != -1:
try:
os.chown(filename, changetouser, changetogroup)
except Exception as e:
self.logger.error(
"Could not change user/group of file %s : %s" % (filename, str(e)))
class LDAPlugin(ScannerPlugin):
"""Deliver message to maildir / mbox"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
"path": {
"default": "/usr/local/fuglu/deliver/${to_address}",
"description": "Path to maildir / mbox file, supports templates",
},
# maybe we need to support our own locking later, for now we use python's built-ins
#'locktype':{
# 'default':'',
# 'description':"flock, ...",
# },
"boxtype": {"default": "mbox", "description": "mbox, maildir"},
# maybe we need to support various mbox types later, for now we use python's built-in module
#'subtype':{
# 'default':'',
# 'description':"what type of mbox... ",
# },
"filterfile": {"default": "", "description": "only store messages which use filter..."},
}
self.logger = self._logger()
self.filter = None
self.boxtypemap = {"mbox": self.deliver_mbox, "maildir": self.deliver_maildir}
def lint(self):
allok = self.checkConfig()
filterfile = self.config.get(self.section, "filterfile", "").strip()
if filterfile != "" and not os.path.exists(filterfile):
print "LDA filter rules file does not exist : %s" % filterfile
allok = False
boxtype = self.config.get(self.section, "boxtype")
if boxtype not in self.boxtypemap:
print "Unsupported boxtype: %s" % boxtype
allok = False
return allok
def examine(self, suspect):
starttime = time.time()
filterfile = self.config.get(self.section, "filterfile", "").strip()
if self.filter == None:
if filterfile != "":
if not os.path.exists(filterfile):
self._logger().warning("LDA filter rules file does not exist : %s" % filterfile)
return DEFER
self.filter = SuspectFilter(filterfile)
if self.filter != None:
match = self.filter.matches(suspect)
if not match:
return DUNNO
self.boxtypemap[self.config.get(self.section, "boxtype")](suspect)
# For debugging, its good to know how long each plugin took
endtime = time.time()
difftime = endtime - starttime
suspect.tags["LDAPlugin.time"] = "%.4f" % difftime
def deliver_mbox(self, suspect):
mbox_msg = mailbox.mboxMessage(suspect.get_message_rep())
mbox_path = apply_template(self.config.get(self.section, "path"), suspect)
mbox = mailbox.mbox(mbox_path)
try:
mbox.lock()
mbox.add(mbox_msg)
mbox.flush()
except Exception, e:
self.logger.error("Could not store message %s to %s: %s" % (suspect.id, mbox_path, str(e)))
finally:
class ArchivePlugin(ScannerPlugin):
"""This plugins stores a copy of the message if it matches certain criteria (Suspect Filter).
You can use this if you want message archives for your domains or to debug problems occuring only for certain recipients.
Examples for the archive.regex filter file:
Archive messages to domain ''test.com'':
``to_domain test\.com``
Archive messages from [email protected]:
``envelope_from oli@fuglu\.org``
you can also append "yes" and "no" to the rules to create a more advanced configuration. Lets say we want to archive all messages to [email protected] and all regular messages [email protected] except the ones created by automated scripts like logwatch or daily backup messages etc.
envelope_from logwatch@.*fuglu.org no
envelope_to sales@fuglu\.org yes
from [email protected] no
envelope_to support@fuglu\.org yes
Note: The first rule to match in a message is the only rule that will be applied. Exclusion rules should therefore be put above generic/catch-all rules.
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
'archiverules': {
'default': '/etc/fuglu/archive.regex',
'description': 'Archiving SuspectFilter File',
},
'archivedir': {
'default': '/tmp',
'description': 'storage for archived messages',
},
'subdirtemplate': {
'default': '${to_domain}',
'description': 'subdirectory within archivedir',
},
'filenametemplate': {
'default': '${id}.eml',
'description': 'filename template for the archived messages',
},
'storeoriginal': {
'default':
'1',
'description':
"if true/1/yes: store original message\nif false/0/no: store message probably altered by previous plugins, eg with spamassassin headers",
},
'chown': {
'default':
'',
'description':
"change owner of saved messages (username or numeric id) - this only works if fuglu is running as root (which is NOT recommended)",
},
'chgrp': {
'default':
'',
'description':
"change group of saved messages (groupname or numeric id) - the user running fuglu must be a member of the target group for this to work",
},
'chmod': {
'default': '',
'description': "set file permissions of saved messages",
},
}
self.filter = None
self.logger = self._logger()
def __str__(self):
return "Archive"
def lint(self):
allok = (self.checkConfig() and self.check_deprecated()
and self.lint_dirs() and self.lint_filter())
return allok
def check_deprecated(self):
if self.config.has_option(self.section, 'makedomainsubdir'):
print(
"the config option 'makedomainsubdir' has been replaced with 'subdirtemplate' "
)
print("please update your config")
print("makedomainsubdir=1 -> subdirtemplate=${to_domain}")
print("makedomainsubdir=0 -> subdirtemplate=")
return False
return True
def lint_filter(self):
filterfile = self.config.get(self.section, 'archiverules')
filter = SuspectFilter(filterfile)
return filter.lint()
def lint_dirs(self):
archivedir = self.config.get(self.section, 'archivedir')
if archivedir == "":
print('Archivedir is not specified')
return False
if not os.path.isdir(archivedir):
print("Archivedir '%s' does not exist or is not a directory" %
(archivedir))
return False
return True
def examine(self, suspect):
archiverules = self.config.get(self.section, 'archiverules')
if archiverules == None or archiverules == "":
return DUNNO
if not os.path.exists(archiverules):
self.logger.error('Archive Rules file does not exist : %s' %
archiverules)
return DUNNO
if self.filter == None:
self.filter = SuspectFilter(archiverules)
(match, arg) = self.filter.matches(suspect)
if match:
if arg != None and arg.lower() == 'no':
suspect.debug("Suspect matches archive exception rule")
self.logger.debug(
"""Header matches archive exception rule - not archiving"""
)
else:
if arg != None and arg.lower() != 'yes':
self.logger.warning(
"Unknown archive action '%s' assuming 'yes'" % arg)
self.logger.debug("""Header matches archive rule""")
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches archiving rule (i would archive it if we weren't in debug mode)"
)
else:
self.archive(suspect)
else:
suspect.debug(
"No archive rule/exception rule applies to this message")
def archive(self, suspect):
archivedir = self.config.get(self.section, 'archivedir')
if archivedir == "":
self.logger.error('Archivedir is not specified')
return
subdirtemplate = self.config.get(self.section, 'subdirtemplate')
if self.config.has_option(
self.section, 'makedomainsubdir'
) and subdirtemplate == self.requiredvars['subdirtemplate']['default']:
self.logger.warning(
"Archive config is using deprecated 'makedomainsubdir' config option. Emulating old behaviour. Update your config(subdirtemplate)"
)
if self.config.getboolean(self.section, 'makedomainsubdir'):
subdirtemplate = "${to_domain}"
else:
subdirtemplate = ""
# the archive root dir
startdir = os.path.abspath(archivedir)
# relative dir within archive root
subdir = apply_template(subdirtemplate, suspect)
if subdir.endswith('/'):
subdir = subdir[:-1]
# filename without dir
filenametemplate = self.config.get(self.section, 'filenametemplate')
filename = apply_template(filenametemplate, suspect)
# make sure filename can't create new folders
filename = filename.replace('/', '_')
# full relative filepath within archive dir
fpath = "%s/%s" % (subdir, filename)
# absolute final filepath
requested_path = os.path.abspath("%s/%s" % (startdir, fpath))
if not os.path.commonprefix([requested_path, startdir
]).startswith(startdir):
self.logger.error(
"file path '%s' seems to be outside archivedir '%s' - storing to archivedir"
% (requested_path, startdir))
requested_path = "%s/%s" % (startdir, filename)
finaldir = os.path.dirname(requested_path)
if not os.path.isdir(finaldir):
os.makedirs(finaldir, 0o755)
if self.config.getboolean(self.section, 'storeoriginal'):
shutil.copy(suspect.tempfile, requested_path)
else:
with open(requested_path, 'w') as fp:
fp.write(suspect.get_source())
chmod = self.config.get(self.section, 'chmod')
chgrp = self.config.get(self.section, 'chgrp')
chown = self.config.get(self.section, 'chown')
if chmod or chgrp or chown:
self.setperms(requested_path, chmod, chgrp, chown)
self.logger.info(
'Message from %s to %s archived as %s' %
(suspect.from_address, suspect.to_address, requested_path))
return requested_path
def setperms(self, filename, chmod, chgrp, chown):
"""Set file permissions and ownership
:param filename The target file
:param chmod string representing the permissions (example '640')
:param chgrp groupname or group id of the target group. the user running fuglu must be a member of this group for this to work
:param chown username or user id of the target user. fuglu must run as root for this to work (which is not recommended for security reasons)
"""
# chmod
if chmod:
perm = int(chmod, 8)
try:
os.chmod(filename, perm)
except:
self.logger.error('could not set permission on file %s' %
filename)
# chgrp
changetogroup = -1
if chgrp:
group = None
try:
group = grp.getgrnam(chgrp)
except KeyError:
pass
try:
group = grp.getgrgid(int(chgrp))
except KeyError:
pass
except ValueError:
pass
if group != None:
changetogroup = group.gr_gid
else:
self.logger.warn("Group %s not found" % chgrp)
# chown
changetouser = -1
if chown:
user = None
try:
user = pwd.getpwnam(chown)
except KeyError:
pass
try:
user = pwd.getpwuid(int(chown))
except KeyError:
pass
except ValueError:
pass
if user != None:
changetouser = user.pw_uid
else:
self.logger.warn("User %s not found" % chown)
if changetogroup != -1 or changetouser != -1:
try:
os.chown(filename, changetouser, changetogroup)
except Exception as e:
self.logger.error(
"Could not change user/group of file %s : %s" %
(filename, str(e)))
class IMAPCopyPlugin(ScannerPlugin):
"""This plugins stores a copy of the message to an IMAP mailbox if it matches certain criteria (Suspect Filter).
The rulefile works similar to the archive plugin. As third column you have to provide imap account data in the form:
<protocol>://<username>:<password>@<servernameorip>[:port]/<mailbox>
<protocol> is either imap or imaps
"""
def __init__(self, config, section=None):
ScannerPlugin.__init__(self, config, section)
self.requiredvars = {
'imapcopyrules': {
'default': '/etc/fuglu/imapcopy.regex',
'description': 'IMAP copy suspectFilter File',
},
'storeoriginal': {
'default':
'1',
'description':
"if true/1/yes: store original message\nif false/0/no: store message probably altered by previous plugins, eg with spamassassin headers",
}
}
self.filter = None
self.logger = self._logger()
def examine(self, suspect):
imapcopyrules = self.config.get(self.section, 'imapcopyrules')
if imapcopyrules is None or imapcopyrules == "":
return DUNNO
if not os.path.exists(imapcopyrules):
self._logger().error('IMAP copy rules file does not exist : %s' %
imapcopyrules)
return DUNNO
if self.filter is None:
self.filter = SuspectFilter(imapcopyrules)
(match, info) = self.filter.matches(suspect, extended=True)
if match:
field, matchedvalue, arg, regex = info
if arg is not None and arg.lower() == 'no':
suspect.debug("Suspect matches imap copy exception rule")
self.logger.info(
"""%s: Header %s matches imap copy exception rule '%s' """
% (suspect.id, field, regex))
else:
if arg is None or (not arg.lower().startswith('imap')):
self.logger.error(
"Unknown target format '%s' should be 'imap(s)://user:pass@host/folder'"
% arg)
else:
self.logger.info(
"""%s: Header %s matches imap copy rule '%s' """ %
(suspect.id, field, regex))
if suspect.get_tag('debug'):
suspect.debug(
"Suspect matches imap copy rule (I would copy it if we weren't in debug mode)"
)
else:
self.storeimap(suspect, arg)
else:
suspect.debug(
"No imap copy rule/exception rule applies to this message")
def imapconnect(self, imapurl, lintmode=False):
p = urlparse(imapurl)
scheme = p.scheme.lower()
host = p.hostname
port = p.port
username = p.username
password = p.password
folder = p.path[1:]
if scheme == 'imaps':
ssl = True
else:
ssl = False
if port is None:
if ssl:
port = imaplib.IMAP4_SSL_PORT
else:
port = imaplib.IMAP4_PORT
try:
if ssl:
imap = imaplib.IMAP4_SSL(host=host, port=port)
else:
imap = imaplib.IMAP4(host=host, port=port)
except Exception as e:
ltype = 'IMAP'
if ssl:
ltype = 'IMAP-SSL'
msg = "%s Connection to server %s failed: %s" % (ltype, host,
str(e))
if lintmode:
print(msg)
else:
self.logger.error(msg)
return None
try:
imap.login(username, password)
except Exception as e:
msg = "Login to server %s failed: %s" % (host, str(e))
if lintmode:
print(msg)
else:
self.logger.error(msg)
return None
mtype, count = imap.select(folder)
if mtype == 'NO':
msg = "Could not select folder %s" % folder
if lintmode:
print(msg)
else:
self.logger.error(msg)
return None
return imap
def storeimap(self, suspect, imapurl):
imap = self.imapconnect(imapurl)
if not imap:
return
#imap.debug=4
p = urlparse(imapurl)
folder = p.path[1:]
if self.config.getboolean(self.section, 'storeoriginal'):
src = suspect.get_original_source()
else:
src = suspect.get_source()
mtype, data = imap.append(folder, None, None, src)
if mtype != 'OK':
self.logger.error(
'Could put store in IMAP. APPEND command failed: %s' % data)
imap.logout()
def lint(self):
allok = (self.check_config() and self.lint_imap())
return allok
def lint_imap(self):
#read file, check for all imap accounts
imapcopyrules = self.config.get(self.section, 'imapcopyrules')
if imapcopyrules != '' and not os.path.exists(imapcopyrules):
print("Imap copy rules file does not exist : %s" % imapcopyrules)
return False
sfilter = SuspectFilter(imapcopyrules)
accounts = []
for tup in sfilter.patterns:
headername, pattern, arg = tup
if arg not in accounts:
if arg is None:
print("Rule %s %s has no imap copy target" %
(headername, pattern.pattern))
return False
if arg.lower() == 'no':
continue
accounts.append(arg)
for acc in accounts:
p = urlparse(acc)
host = p.hostname
username = p.username
folder = p.path[1:]
print("Checking %s@%s/%s" % (username, host, folder))
imap = self.imapconnect(acc, lintmode=True)
if not imap:
print("Lint failed for this account")
return False
return True
class SuspectFilterTestCase(unittest.TestCase):
"""Test Header Filter"""
def setUp(self):
self.candidate = SuspectFilter(TESTDATADIR + '/headertest.regex')
def tearDown(self):
pass
def test_sf_get_args(self):
"""Test SuspectFilter files"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
suspect.tags['testtag'] = 'testvalue'
headermatches = self.candidate.get_args(suspect)
self.assertTrue(
'Sent to unittest domain!' in headermatches, "To_domain not found in headercheck")
self.assertTrue('Envelope sender is [email protected]' in headermatches,
"Envelope Sender not matched in header chekc")
self.assertTrue('Mime Version is 1.0' in headermatches,
"Standard header Mime Version not found")
self.assertTrue(
'A tag match' in headermatches, "Tag match did not work")
self.assertTrue(
'Globbing works' in headermatches, "header globbing failed")
self.assertTrue(
'body rule works' in headermatches, "decoded body rule failed")
self.assertTrue(
'full body rule works' in headermatches, "full body failed")
self.assertTrue('mime rule works' in headermatches, "mime rule failed")
self.assertFalse('this should not match in a body rule' in headermatches,
'decoded body rule matched raw body')
# perl style advanced rules
self.assertTrue('perl-style /-notation works!' in headermatches,
"new rule format failed: %s" % headermatches)
self.assertTrue('perl-style recipient match' in headermatches,
"new rule format failed for to_domain: %s" % headermatches)
self.assertFalse('this should not match' in headermatches,
"rule flag ignorecase was not detected")
# TODO: raw body rules
def test_sf_matches(self):
"""Test SuspectFilter extended matches"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
(match, info) = self.candidate.matches(suspect, extended=True)
self.assertTrue(match, 'Match should return True')
field, matchedvalue, arg, regex = info
self.assertTrue(field == 'to_domain')
self.assertTrue(matchedvalue == 'unittests.fuglu.org')
self.assertTrue(arg == 'Sent to unittest domain!')
self.assertTrue(regex == 'unittests\.fuglu\.org')
def test_sf_get_field(self):
"""Test SuspectFilter field extract"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
# additional field tests
self.assertEqual(self.candidate.get_field(
suspect, 'clienthelo')[0], 'helo1')
self.assertEqual(self.candidate.get_field(
suspect, 'clientip')[0], '10.0.0.1')
self.assertEqual(self.candidate.get_field(
suspect, 'clienthostname')[0], 'rdns1')
def test_strip(self):
html = """foo<a href="bar">bar</a><script language="JavaScript">echo('hello world');</script>baz"""
declarationtest = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de">
<head>
<title>greetings</title>
</head>
<body>
<font color="red">well met!</font>
</body>
</html>
"""
# word generated empty message
wordhtml = """<html xmlns:v=3D"urn:schemas-microsoft-com:vml"
xmlns:o=3D"urn:schemas-microsoft-com:office:office"
xmlns:w=3D"urn:schemas-microsoft-com:office:word"
xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml"
xmlns=3D"http://www.w3.org/TR/REC-html40"><head><META
HTTP-EQUIV=3D"Content-Type" CONTENT=3D"text/html;
charset=3Dus-ascii"><meta name=3DGenerator content=3D"Microsoft Word 15
(filtered medium)"><style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
span.E-MailFormatvorlage17
{mso-style-type:personal-compose;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-family:"Calibri",sans-serif;
mso-fareast-language:EN-US;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:70.85pt 70.85pt 2.0cm 70.85pt;}
div.WordSection1
{page:WordSection1;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext=3D"edit">
<o:idmap v:ext=3D"edit" data=3D"1" />
</o:shapelayout></xml><![endif]--></head><body lang=3DDE-CH
link=3D"#0563C1" vlink=3D"#954F72"><div class=3DWordSection1><p
class=3DMsoNormal><o:p> </o:p></p></div></body></html>"""
for use_bfs in [True, False]:
stripped = self.candidate.strip_text(html, use_bfs=use_bfs)
self.assertEqual(stripped, 'foobarbaz')
docstripped = self.candidate.strip_text(
declarationtest, use_bfs=use_bfs)
self.assertEqual(
docstripped.split(), ['greetings', 'well', 'met!'])
wordhtmstripped = self.candidate.strip_text(
wordhtml, use_bfs=use_bfs)
self.assertEqual(wordhtmstripped.strip(), '')
class IMAPCopyPlugin(ScannerPlugin):
"""This plugins stores a copy of the message to an IMAP mailbox if it matches certain criteria (Suspect Filter).
The rulefile works similar to the archive plugin. As third column you have to provide imap account data in the form:
<protocol>://<username>:<password>@<servernameorip>[:port]/<mailbox>
<protocol> is either imap or imaps
"""
def __init__(self,config,section=None):
ScannerPlugin.__init__(self,config,section)
self.requiredvars={
'imapcopyrules':{
'default':'/etc/fuglu/imapcopy.regex',
'description':'IMAP copy suspectFilter File',
},
'storeoriginal':{
'default':'1',
'description':"if true/1/yes: store original message\nif false/0/no: store message probably altered by previous plugins, eg with spamassassin headers",
}
}
self.filter=None
self.logger=self._logger()
def examine(self,suspect):
imapcopyrules=self.config.get(self.section, 'imapcopyrules')
if imapcopyrules==None or imapcopyrules=="":
return DUNNO
if not os.path.exists(imapcopyrules):
self._logger().error('IMAP copy rules file does not exist : %s'%imapcopyrules)
return DUNNO
if self.filter==None:
self.filter=SuspectFilter(imapcopyrules)
(match,info)=self.filter.matches(suspect,extended=True)
if match:
field,matchedvalue,arg,regex=info
if arg!=None and arg.lower()=='no':
suspect.debug("Suspect matches imap copy exception rule")
self.logger.info("""%s: Header %s matches imap copy exception rule '%s' """%(suspect.id,field,regex))
else:
if arg==None or (not arg.lower().startswith('imap')):
self.logger.error("Unknown target format '%s' should be 'imap(s)://user:pass@host/folder'"%arg)
else:
self.logger.info("""%s: Header %s matches imap copy rule '%s' """%(suspect.id,field,regex))
if suspect.get_tag('debug'):
suspect.debug("Suspect matches imap copy rule (I would copy it if we weren't in debug mode)")
else:
self.storeimap(suspect,arg)
else:
suspect.debug("No imap copy rule/exception rule applies to this message")
def imapconnect(self,imapurl,lintmode=False):
p=urlparse(imapurl)
scheme=p.scheme.lower()
host=p.hostname
port=p.port
username=p.username
password=p.password
folder=p.path[1:]
if scheme=='imaps':
ssl=True
else:
ssl=False
if port==None:
if ssl:
port=imaplib.IMAP4_SSL_PORT
else:
port=imaplib.IMAP4_PORT
try:
if ssl:
imap=imaplib.IMAP4_SSL(host=host,port=port)
else:
imap=imaplib.IMAP4(host=host,port=port)
except Exception,e:
ltype='IMAP'
if ssl:
ltype='IMAP-SSL'
msg="%s Connection to server %s failed: %s"%(ltype,host,str(e))
if lintmode:
print msg
else:
self.logger.error(msg)
return None
try:
imap.login(username,password)
except Exception,e:
msg="Login to server %s failed: %s"%(host,str(e))
if lintmode:
print msg
else:
self.logger.error(msg)
return None
