def test_dbrules(self):
"""Test if db rules correctly override defaults"""
testdata = u"""
INSERT INTO attachmentrules(scope,checktype,action,regex,description,prio) VALUES
('*****@*****.**','contenttype','allow','application/x-executable','this user likes exe',1)
"""
self.session.execute(testdata)
# copy file rules
tempfilename = tempfile.mktemp(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy(TESTDATADIR + '/binaryattachment.eml', tempfilename)
suspect = Suspect('*****@*****.**',
'*****@*****.**', tempfilename)
result = self.candidate.examine(suspect)
resstr = actioncode_to_string(result)
self.assertEquals(resstr, "DUNNO")
# another recipient should still get the block
suspect = Suspect('*****@*****.**',
'*****@*****.**', tempfilename)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
resstr = actioncode_to_string(result)
self.assertEquals(resstr, "DELETE")
os.remove(tempfilename)
def test_result(self):
"""Test if EICAR virus is detected and message deleted"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test eicar attachment
X-Mailer: swaks v20061116.0 jetmore.org/john/code/#swaks
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="----=_MIME_BOUNDARY_000_12140"
------=_MIME_BOUNDARY_000_12140
Content-Type: text/plain
Eicar test
------=_MIME_BOUNDARY_000_12140
Content-Type: application/octet-stream
Content-Transfer-Encoding: BASE64
Content-Disposition: attachment
UEsDBAoAAAAAAGQ7WyUjS4psRgAAAEYAAAAJAAAAZWljYXIuY29tWDVPIVAlQEFQWzRcUFpYNTQo
UF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNULUZJTEUhJEgrSCoNClBLAQIU
AAoAAAAAAGQ7WyUjS4psRgAAAEYAAAAJAAAAAAAAAAEAIAD/gQAAAABlaWNhci5jb21QSwUGAAAA
AAEAAQA3AAAAbQAAAAAA
------=_MIME_BOUNDARY_000_12140--"""
suspect.setMessageRep(email.message_from_string(stream))
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
strresult = actioncode_to_string(result)
self.assertEqual(strresult, "DELETE")
def test_ignore_sender(self):
from fuglu.shared import Suspect
v = Vacation()
v.ignoresender = u"unittests.fuglu.org [email protected]"
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'gone for good'
v.body = u'outta here'
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
suspect.set_tag('nobounce', True)
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(candidatevacation != None,
"Vacation object not found in database")
# TODO had to disable due to sqlalchemy error
# Instance <Vacation at 0x2938890> is not bound to a Session; attribute refresh operation cannot proceed
#self.assertEqual(v.ignoresender,candidatevacation.ignoresender,"Vacation object did not get ignore list")
self.assertTrue(
self.candidate.ignore_sender(candidatevacation, suspect),
"Test Message should generate vacation reply(ignored sender)")
self.assertFalse(
self.candidate.should_send_vacation_message(suspect),
"Sender on ignorelist, still wants to send message?!")
def test_generated_message(self):
from fuglu.shared import Suspect
suspect = Suspect(u'*****@*****.**',
'*****@*****.**', '/dev/null')
suspect.tags['nobounce'] = True
v = Vacation()
v.ignoresender = u""
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'Döner mit schärf!'
v.body = u"""Je m'envole pour l'Allemagne, où je suis né."""
# v.body="Away!"
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(candidatevacation != None,
"Vacation object not found in database")
message = self.candidate.send_vacation_reply(suspect,
candidatevacation)
def test_sf_get_args(self):
"""Test SuspectFilter files"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
suspect.tags['testtag'] = 'testvalue'
headermatches = self.candidate.get_args(suspect)
self.assertTrue(
'Sent to unittest domain!' in headermatches, "To_domain not found in headercheck")
self.assertTrue('Envelope sender is [email protected]' in headermatches,
"Envelope Sender not matched in header chekc")
self.assertTrue('Mime Version is 1.0' in headermatches,
"Standard header Mime Version not found")
self.assertTrue(
'A tag match' in headermatches, "Tag match did not work")
self.assertTrue(
'Globbing works' in headermatches, "header globbing failed")
self.assertTrue(
'body rule works' in headermatches, "decoded body rule failed")
self.assertTrue(
'full body rule works' in headermatches, "full body failed")
self.assertTrue('mime rule works' in headermatches, "mime rule failed")
self.assertFalse('this should not match in a body rule' in headermatches,
'decoded body rule matched raw body')
# perl style advanced rules
self.assertTrue('perl-style /-notation works!' in headermatches,
"new rule format failed: %s" % headermatches)
self.assertTrue('perl-style recipient match' in headermatches,
"new rule format failed for to_domain: %s" % headermatches)
self.assertFalse('this should not match' in headermatches,
"rule flag ignorecase was not detected")
def test_modified_message(self):
"""Test if the modified message gets archived correctly"""
from fuglu.shared import Suspect
import shutil
import tempfile
tempfilename = tempfile.mktemp(
suffix='archive', prefix='fuglu-unittest', dir='/tmp')
shutil.copy(TESTDATADIR + '/helloworld.eml', tempfilename)
self.tempfiles.append(tempfilename)
self.config.set('ArchivePlugin', 'storeoriginal', '0')
candidate = ArchivePlugin(self.config)
suspect = Suspect(
'*****@*****.**', '*****@*****.**', tempfilename)
origmessage = suspect.get_source()
# modify the mesg
msgrep = suspect.get_message_rep()
msgrep['X-Changed-Something'] = 'Yes'
suspect.setMessageRep(msgrep)
filename = candidate.archive(suspect)
self.assertTrue(filename != None and filename)
self.tempfiles.append(filename)
archivedmessage = open(filename, 'r').read()
self.assertNotEqual(origmessage.strip(), archivedmessage.strip(
)), 'Archived message should have stored modified message'
def test_generated_message(self):
from fuglu.shared import Suspect
suspect = Suspect(
u'*****@*****.**', '*****@*****.**', '/dev/null')
suspect.tags['nobounce'] = True
v = Vacation()
v.ignoresender = u""
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'Döner mit schärf!'
v.body = u"""Je m'envole pour l'Allemagne, où je suis né."""
# v.body="Away!"
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(
candidatevacation != None, "Vacation object not found in database")
message = self.candidate.send_vacation_reply(
suspect, candidatevacation)
def test_localpartblacklist(self):
"""test messages from mailer-daemon"""
from fuglu.shared import Suspect
import email
v = Vacation()
v.ignoresender = u""
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'awaaay'
v.body = u'cya'
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
botmsg = """From: [email protected]
Subject: mailinglist membership reminder...
"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
suspect.set_tag('nobounce', True)
suspect.set_source(botmsg)
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(
candidatevacation != None, "Vacation object not found in database")
self.assertFalse(self.candidate.should_send_vacation_message(
suspect), "Test Message should NOT generate vacation reply(automated)")
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming esmtp transfer did not finish')
return None
sess = self.sess
fromaddr = sess.from_address
tempfilename = sess.tempfilename
try:
suspect = Suspect(fromaddr, sess.recipients, tempfilename)
except ValueError as e:
if len(sess.recipients) > 0:
toaddr = sess.recipients[0]
else:
toaddr = ''
self.logger.error(
'failed to initialise suspect with from=<%s> to=<%s> : %s' %
(fromaddr, toaddr, str(e)))
raise
if sess.xforward_helo is not None and sess.xforward_addr is not None and sess.xforward_rdns is not None:
suspect.clientinfo = sess.xforward_helo, sess.xforward_addr, sess.xforward_rdns
return suspect
def test_ignore_sender(self):
from fuglu.shared import Suspect
v = Vacation()
v.ignoresender = u"unittests.fuglu.org [email protected]"
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'gone for good'
v.body = u'outta here'
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
suspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null')
suspect.set_tag('nobounce', True)
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(
candidatevacation != None, "Vacation object not found in database")
# TODO had to disable due to sqlalchemy error
# Instance <Vacation at 0x2938890> is not bound to a Session; attribute refresh operation cannot proceed
#self.assertEqual(v.ignoresender,candidatevacation.ignoresender,"Vacation object did not get ignore list")
self.assertTrue(self.candidate.ignore_sender(
candidatevacation, suspect), "Test Message should generate vacation reply(ignored sender)")
self.assertFalse(self.candidate.should_send_vacation_message(
suspect), "Sender on ignorelist, still wants to send message?!")
def test_virus(self):
"""Test if eicar is detected as virus"""
from fuglu.shared import Suspect
import shutil
self.mc.load_plugins()
if len(self.mc.plugins) == 0:
raise Exception("plugins not loaded")
sesshandler = SessionHandler(None, self.mc.config, self.mc.prependers,
self.mc.plugins, self.mc.appenders)
tempfilename = tempfile.mktemp(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy(TESTDATADIR + '/eicar.eml', tempfilename)
self.tempfiles.append(tempfilename)
suspect = Suspect('*****@*****.**',
'*****@*****.**', tempfilename)
pluglist = sesshandler.run_prependers(suspect)
self.assertFalse(
len(pluglist) == 0,
"Viruscheck will fail, pluginlist empty after run_prependers")
sesshandler.run_plugins(suspect, pluglist)
self.assertTrue(suspect.is_virus(),
"Eicar message was not detected as virus")
def test_something(self):
"""Test if examine runs through"""
from fuglu.shared import Suspect
suspect=Suspect('*****@*****.**','*****@*****.**','/dev/null')
self.candidate.examine(suspect)
self.failIf(suspect.get_tag('ExamplePlugin.time')==None, "Examine didn't run through")
def test_result(self):
"""Test if EICAR virus is detected and message deleted"""
suspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null')
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test eicar attachment
X-Mailer: swaks v20061116.0 jetmore.org/john/code/#swaks
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="----=_MIME_BOUNDARY_000_12140"
------=_MIME_BOUNDARY_000_12140
Content-Type: text/plain
Eicar test
------=_MIME_BOUNDARY_000_12140
Content-Type: application/octet-stream
Content-Transfer-Encoding: BASE64
Content-Disposition: attachment
UEsDBAoAAAAAAGQ7WyUjS4psRgAAAEYAAAAJAAAAZWljYXIuY29tWDVPIVAlQEFQWzRcUFpYNTQo
UF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNULUZJTEUhJEgrSCoNClBLAQIU
AAoAAAAAAGQ7WyUjS4psRgAAAEYAAAAJAAAAAAAAAAEAIAD/gQAAAABlaWNhci5jb21QSwUGAAAA
AAEAAQA3AAAAbQAAAAAA
------=_MIME_BOUNDARY_000_12140--"""
suspect.setMessageRep(email.message_from_string(stream))
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
strresult = actioncode_to_string(result)
self.assertEqual(strresult, "DELETE")
def _make_dummy_suspect(self,
senderdomain,
clientip,
helo='foo.example.com'):
s = Suspect('sender@%s' % senderdomain, '*****@*****.**',
'/dev/null')
s.clientinfo = (helo, clientip, 'ptr.example.com')
return s
def test_parse_rcvd_header(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
self.assertEqual(suspect._parse_rcvd_header(
"from helo1 (rdns1 [10.0.0.1])"), ('helo1', 'rdns1', '10.0.0.1'))
self.assertEqual(suspect._parse_rcvd_header("from mx0.slc.paypal.com (mx1.slc.paypal.com [173.0.84.226])"), (
'mx0.slc.paypal.com', 'mx1.slc.paypal.com', '173.0.84.226'))
self.assertEqual(suspect._parse_rcvd_header("from mail1a.tilaa.nl (mail1a.tilaa.nl [IPv6:2a02:2770:6:0:21a:4aff:fea8:1328])"), (
'mail1a.tilaa.nl', 'mail1a.tilaa.nl', '2a02:2770:6:0:21a:4aff:fea8:1328'))
def test_id(self):
"""Check the length and uniqueness of the generated id"""
s = Suspect('*****@*****.**', '*****@*****.**', '/dev/null')
known = []
for i in range(10000):
suspect_id = s._generate_id()
self.assertTrue(
suspect_id not in known, 'duplicate id %s generated' % suspect_id)
known.append(suspect_id)
self.assertEqual(len(suspect_id), 32)
for c in suspect_id:
self.assertTrue(c in string.hexdigits)
def test_id(self):
"""Check the length and uniqueness of the generated id"""
s = Suspect('*****@*****.**', '*****@*****.**', '/dev/null')
known = []
for i in range(10000):
suspect_id = s._generate_id()
self.assertTrue(suspect_id not in known,
'duplicate id %s generated' % suspect_id)
known.append(suspect_id)
self.assertEqual(len(suspect_id), 32)
for c in suspect_id:
self.assertTrue(c in string.hexdigits)
def test_forward_modified_stripped(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
action, message = self.saplugin.examine(suspect)
self.assertEqual(DUNNO, action)
try:
self.assertIsNotNone(suspect.get_tag('SAPlugin.report'))
except AttributeError:
# Python 2.6
self.assertTrue(suspect.get_tag('SAPlugin.report') is not None)
def get_suspect(self):
success=self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess=self.sess
fromaddr=sess.from_address
toaddr=sess.to_address
tempfilename=sess.tempfilename
suspect=Suspect(fromaddr,toaddr,tempfilename)
suspect.recipients=set(sess.recipients)
return suspect
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess = self.sess
fromaddr = sess.from_address
toaddr = sess.to_address
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr, toaddr, tempfilename)
suspect.recipients = set(sess.recipients)
return suspect
def get_suspect(self):
succ = self.sess.getincomingmail()
if not succ:
self.logger.error('MILTER SESSION NOT COMPLETED')
return None
sess = self.sess
fromaddr = sess.from_address
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr, sess.recipients, tempfilename)
if sess.helo is not None and sess.addr is not None and sess.rdns is not None:
suspect.clientinfo = sess.helo, sess.addr, sess.rdns
return suspect
def test_template(self):
"""Test Basic Template function"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
suspect.tags['nobounce'] = True
reason = "a three-headed monkey stole it"
template = """Your message '${subject}' from ${from_address} to ${to_address} could not be delivered because ${reason}"""
result = apply_template(template, suspect, dict(reason=reason))
expected = """Your message 'Hello world!' from [email protected] to [email protected] could not be delivered because a three-headed monkey stole it"""
self.assertEqual(
result, expected), "Got unexpected template result: %s" % result
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess = self.sess
fromaddr = "*****@*****.**"
toaddr = "*****@*****.**"
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr, toaddr, tempfilename)
suspect.recipients = [toaddr, ]
return suspect
def get_suspect(self):
success=self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess=self.sess
fromaddr="*****@*****.**"
toaddr="*****@*****.**"
tempfilename=sess.tempfilename
suspect=Suspect(fromaddr,toaddr,tempfilename)
suspect.recipients=[toaddr,]
return suspect
def test_parse_rcvd_header(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
self.assertEqual(
suspect._parse_rcvd_header("from helo1 (rdns1 [10.0.0.1])"),
('helo1', 'rdns1', '10.0.0.1'))
self.assertEqual(
suspect._parse_rcvd_header(
"from mx0.slc.paypal.com (mx1.slc.paypal.com [173.0.84.226])"),
('mx0.slc.paypal.com', 'mx1.slc.paypal.com', '173.0.84.226'))
self.assertEqual(
suspect._parse_rcvd_header(
"from mail1a.tilaa.nl (mail1a.tilaa.nl [IPv6:2a02:2770:6:0:21a:4aff:fea8:1328])"
), ('mail1a.tilaa.nl', 'mail1a.tilaa.nl',
'2a02:2770:6:0:21a:4aff:fea8:1328'))
def get_suspect(self):
succ = self.sess.getincomingmail()
if not succ:
self.logger.error('MILTER SESSION NOT COMPLETED')
return None
sess = self.sess
fromaddr = sess.from_address
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr, sess.recipients, tempfilename)
if sess.helo is not None and sess.addr is not None and sess.rdns is not None:
suspect.clientinfo = sess.helo, sess.addr, sess.rdns
return suspect
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess = self.sess
fromaddr = "*****@*****.**"
toaddr = ["*****@*****.**"]
# use envelope from/to from ncsession if available
if sess.from_address:
fromaddr = sess.from_address
if sess.recipients:
toaddr = sess.recipients
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr,
toaddr,
tempfilename,
att_cachelimit=self._att_mgr_cachesize)
if sess.tags:
# update suspect tags with the ones receivec
# during ncsession
suspect.tags.update(sess.tags)
return suspect
def test_score(self):
suspect=Suspect('*****@*****.**','*****@*****.**','/dev/null')
stream="""Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test scanner
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
"""
suspect.setSource(stream)
result=self.candidate.examine(suspect)
if type(result) is tuple:
result,message=result
score=int( suspect.get_tag('SAPlugin.spamscore'))
self.failUnless(score>999, "GTUBE mails should score ~1000 , we got %s"%score)
self.failUnless(result==REJECT,'High spam should be rejected')
def test_archivename(self):
"""Test check archive names"""
for testfile in ['6mbzipattachment.eml', '6mbrarattachment.eml']:
try:
# copy file rules
tempfilename = tempfile.mktemp(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy("%s/%s" % (TESTDATADIR, testfile), tempfilename)
user = '******'
conffile = self.tempdir + "/%s-archivenames.conf" % user
open(conffile, 'w').write(
"deny largefile user does not like the largefile within a zip\ndeny 6mbfile user does not like the largefile within a zip"
)
suspect = Suspect('*****@*****.**', user,
tempfilename)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.failIf(
result != DELETE,
'archive containing blocked filename was not blocked')
finally:
os.remove(tempfilename)
os.remove(conffile)
def test_extract_spamstatus(self):
"""Test if the spam status header gets extracted correctly"""
candidate = SAPlugin(self.config)
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
headername = 'X-Spam-Status'
headertests = [ # tuple header content, expected spamstatus, expected spam score
('YES', True, None), # _YESNOCAPS_
('NO', False, None), # _YESNOCAPS_
(' Yes, score=13.37', True, 13.37), # _YESNO_, score=_SCORE_
(' No, score=-2.826', False, -2.826), # _YESNO_, score=_SCORE_
# with test names, bug #24
("No, score=1.9 required=8.0 tests=BAYES_00,FROM_EQ_TO,TVD_SPACE_RATIO,TVD_SPACE_RATIO_MINFP autolearn=no autolearn_force=no version=3.4.0",
False, 1.9),
]
for headercontent, expectedspamstatus, expectedscore in headertests:
msgrep = Message()
msgrep[headername] = Header(headercontent).encode()
spamstatus, score, report = candidate._extract_spamstatus(
msgrep, headername, suspect)
self.assertEqual(
spamstatus, expectedspamstatus,
"spamstatus should be %s from %s" %
(expectedspamstatus, headercontent))
self.assertEqual(
score, expectedscore, "spamscore should be %s from %s" %
(expectedscore, headercontent))
def test_hiddenpart(self):
"""Test for hidden part in message epilogue"""
testfile = 'hiddenpart.eml'
try:
tmpfile = tempfile.NamedTemporaryFile(suffix='hidden',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy("%s/%s" % (TESTDATADIR, testfile), tmpfile.name)
user = '******'
conffile = self.tempdir + "/%s-filetypes.conf" % user
# the largefile in the test message is just a bunch of zeroes
open(conffile, 'w').write("deny application\/zip no zips allowed")
self.rulescache._loadrules()
suspect = Suspect('*****@*****.**', user, tmpfile.name)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.assertEqual(result, DELETE,
'hidden message part was not detected')
finally:
tmpfile.close()
os.remove(conffile)
def test_bounce_withenvsender(self, smtpmock):
"""Reference test bounce is called for setup, next test 'test_bounce_noenvsender' should behave differently."""
testfile = '6mbzipattachment.eml'
# copy file rules
tmpfile = tempfile.NamedTemporaryFile(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy("%s/%s" % (TESTDATADIR, testfile), tmpfile.name)
user = '******'
conffile = self.tempdir + "/%s-archivenames.conf" % user
open(conffile, 'w').write(
"deny largefile user does not like the largefile within a zip\ndeny 6mbfile user does not like the largefile within a zip"
)
self.rulescache._loadrules()
suspect = Suspect('*****@*****.**', user, tmpfile.name)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.assertEqual(
result, DELETE,
'archive containing blocked filename was not blocked')
tmpfile.close()
os.remove(conffile)
smtpmock_membercalls = [call[0] for call in smtpmock.mock_calls]
self.assertTrue(smtpmock.called)
self.assertTrue("().helo" in smtpmock_membercalls)
self.assertTrue("().sendmail" in smtpmock_membercalls)
self.assertTrue("().quit" in smtpmock_membercalls)
def test_dosexec(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/6mbrarattachment.eml')
attachmentmanager = suspect.att_mgr
objectlist = attachmentmanager.get_objectlist()
filenames = [obj.filename for obj in objectlist]
self.assertTrue("unnamed.txt" in filenames,
"unnamed.txt not in list %s" % ",".join(filenames))
for obj in objectlist:
if obj.filename == "unnamed.txt":
contenttype = obj.contenttype
# this is a cached property -> patch the cached value for this test
obj._property_cache['contenttype'] = 'application/x-dosexec'
# explicitly set to the blocking content
contenttype = obj.contenttype
self.assertEqual(
'application/x-dosexec', contenttype,
"-> check patching of dict for smart_cached_property")
print("content type: %s" % contenttype)
print("filename auto generated: %s" % obj.filename_generated)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.assertEqual(result, DUNNO, "no attachment should be blocked!")
def lint_blacklist(self):
if not self.config.has_option(
self.section,
'check_sql_blacklist') or not self.config.getboolean(
self.section, 'check_sql_blacklist'):
return True
from fuglu.extensions.sql import ENABLED, get_session
if not ENABLED:
print("SQL Blacklist requested but SQLALCHEMY is not enabled")
return False
session = get_session(
self.config.get(self.section, 'sql_blacklist_dbconnectstring'))
suspect = Suspect('*****@*****.**', '*****@*****.**',
'/dev/null')
conf_sql = self.config.get(self.section, 'sql_blacklist_sql')
sql, params = self._replace_sql_params(suspect, conf_sql)
try:
session.execute(sql, params)
print("Blacklist SQL Query OK")
return True
except Exception as e:
print(e)
return False
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming smtp transfer did not finish')
return None
sess = self.sess
fromaddr = sess.from_address
tempfilename = sess.tempfilename
try:
suspect = Suspect(fromaddr,
sess.recipients,
tempfilename,
att_cachelimit=self._att_mgr_cachesize,
smtp_options=sess.smtpoptions)
except ValueError as e:
if len(sess.recipients) > 0:
toaddr = sess.recipients[0]
else:
toaddr = ''
self.logger.error(
'failed to initialise suspect with from=<%s> to=<%s> : %s' %
(fromaddr, toaddr, str(e)))
raise
return suspect
def test_bounce_noenvsender(self, smtpmock):
"""Don't try to send a bounce if original env sender is empty"""
testfile = '6mbzipattachment.eml'
# copy file rules
tmpfile = tempfile.NamedTemporaryFile(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy("%s/%s" % (TESTDATADIR, testfile), tmpfile.name)
user = '******'
conffile = self.tempdir + "/%s-archivenames.conf" % user
open(conffile, 'w').write(
"deny largefile user does not like the largefile within a zip\ndeny 6mbfile user does not like the largefile within a zip"
)
self.rulescache._loadrules()
suspect = Suspect('', user, tmpfile.name)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.assertEqual(
result, DELETE,
'archive containing blocked filename was not blocked')
tmpfile.close()
os.remove(conffile)
self.assertFalse(
smtpmock.called,
"No SMTP client should have been created because there's no mail to send."
)
def test_archive_wrong_extension(self):
"""Test if archives don't fool us with forged file extensions"""
testfile = 'wrongextension.eml'
try:
tmpfile = tempfile.NamedTemporaryFile(suffix='wrongext',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy("%s/%s" % (TESTDATADIR, testfile), tmpfile.name)
user = '******'
conffile = self.tempdir + "/%s-archivenames.conf" % user
# the largefile in the test message is just a bunch of zeroes
open(conffile, 'w').write(
"deny \.exe$ exe detected in zip with wrong extension")
self.rulescache._loadrules()
suspect = Suspect('*****@*****.**', user, tmpfile.name)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
self.assertEqual(result, DELETE,
'exe in zip with .gz extension was not detected')
finally:
tmpfile.close()
os.remove(conffile)
def test_special_archive_name(self):
"""Check if gz file with exclamantion marks and points in archive name is extracted ok"""
import logging
import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
tmpfile = tempfile.NamedTemporaryFile(suffix='badattach',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy(TESTDATADIR + '/attachment_exclamation_marks_points.eml',
tmpfile.name)
suspect = Suspect('*****@*****.**',
'*****@*****.**', tmpfile.name)
self.assertTrue('aaa.aa!aaaaaaaaa.aa!2345678910!1234567891.xml' in
suspect.att_mgr.get_fileslist(None))
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
tmpfile.close()
self.assertEqual(result, DUNNO)
def parse_env_data_header(self, env_buffer):
"""
Parse envelope data string and store data internally
Args:
env_string (bytes):
"""
try:
mymsg = email.message_from_bytes(env_buffer)
except AttributeError:
mymsg = email.message_from_string(env_buffer)
for key, header in mymsg.items():
value = Suspect.decode_msg_header(header).strip()
if not value:
continue
if key == "X-ENV-SENDER":
self.from_address = value.strip()
self.logger.debug("Found env sender: %s" % value)
elif key == "X-ENV-RECIPIENT":
self.recipients.append(value.strip())
self.logger.debug("Found env recipient: %s" % value)
elif key == "X-DATA-PREPEND-START":
self.tags["prepend_identifier"] = value
self.logger.debug(
"set prepend identifier from Start header to: %s" % value)
elif key == "X-DATA-PREPEND-END":
self.tags["prepend_identifier"] = value
self.logger.debug(
"set prepend identifier from End header to: %s" % value)
else:
self.tags[key] = value
self.logger.debug("Store in Suspect TAG: (%s,%s)" %
(key, value))
def test_template(self):
"""Test Basic Template function"""
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
suspect.tags['nobounce'] = True
reason = "a three-headed monkey stole it"
template = """Your message '${subject}' from ${from_address} to ${to_address} could not be delivered because ${reason}"""
result = apply_template(template, suspect, dict(reason=reason))
expected = """Your message 'Hello world!' from [email protected] to [email protected] could not be delivered because a three-headed monkey stole it"""
self.assertEqual(
result, expected), "Got unexpected template result: %s" % result
def get_suspect(self):
succ=self.sess.getincomingmail()
if not succ:
self.logger.error('MILTER SESSION NOT COMPLETED')
return None
sess=self.sess
fromaddr=sess.from_address
toaddr=sess.to_address
tempfilename=sess.tempfilename
body=open(tempfilename).read()
#self.logger.info("BODY=%s"%body)
#self.logger.info("MILTER: from=%s to=%s"%(fromaddr,toaddr))
suspect=Suspect(fromaddr,toaddr,tempfilename)
suspect.recipients=set(sess.recipients)
return suspect
def test_hf(self):
"""Test header filters"""
suspect = Suspect('*****@*****.**',
'*****@*****.**',
TESTDATADIR + '/helloworld.eml')
suspect.tags['testtag'] = 'testvalue'
headermatches = self.candidate.get_args(suspect)
self.failUnless('Sent to unittest domain!' in headermatches,
"To_domain not found in headercheck")
self.failUnless(
'Envelope sender is [email protected]' in headermatches,
"Envelope Sender not matched in header chekc")
self.failUnless('Mime Version is 1.0' in headermatches,
"Standard header Mime Version not found")
self.failUnless('A tag match' in headermatches,
"Tag match did not work")
self.failUnless('Globbing works' in headermatches,
"header globbing failed")
self.failUnless('body rule works' in headermatches,
"decoded body rule failed")
self.failUnless('full body rule works' in headermatches,
"full body failed")
self.failUnless('mime rule works' in headermatches, "mime rule failed")
self.failIf('this should not match in a body rule' in headermatches,
'decoded body rule matched raw body')
# perl style advanced rules
self.failUnless('perl-style /-notation works!' in headermatches,
"new rule format failed: %s" % headermatches)
self.failUnless(
'perl-style recipient match' in headermatches,
"new rule format failed for to_domain: %s" % headermatches)
self.failIf('this should not match' in headermatches,
"rule flag ignorecase was not detected")
# TODO: raw body rules
# extended
(match, info) = self.candidate.matches(suspect, extended=True)
self.failUnless(match, 'Match should return True')
field, matchedvalue, arg, regex = info
self.failUnless(field == 'to_domain')
self.failUnless(matchedvalue == 'unittests.fuglu.org')
self.failUnless(arg == 'Sent to unittest domain!')
self.failUnless(regex == 'unittests\.fuglu\.org')
def test_headers(self):
"""Test full workflow and check headers"""
myclass = self.__class__.__name__
functionNameAsString = sys._getframe().f_code.co_name
loggername = "%s.%s" % (myclass,functionNameAsString)
logger = logging.getLogger(loggername)
config=ConfigParser.RawConfigParser()
configfile =b"""
[FuzorCheck]
redis=redis:6379:1
ttl=10
timeout=1
headername=X-FuZor
maxsize=600000
redispw=
stripoversize=False
"""
try:
config.readfp(BytesIO(configfile))
except TypeError:
config.read_string(force_uString(configfile))
fuzorplugin = FuzorCheck(config)
self.assertTrue(fuzorplugin.lint())
logger.debug("Create suspect")
suspect = Suspect("*****@*****.**", "*****@*****.**", TESTDATADIR + '/fuzor_html.eml')
logger.debug('generate test hash')
mailhash = FuzorDigest(suspect.get_message_rep()).digest
mailhash_expected = "df1d303855f0bf85d5a7e74c5a00f97166496b3a"
self.assertEqual(mailhash, mailhash_expected, 'generated mail hash %s is different than expected hash %s' % (mailhash, mailhash_expected))
logger.debug("examine suspect")
fuzorplugin.examine(suspect)
tag = suspect.get_tag('SAPlugin.tempheader')
self.assertIsNone(tag, "No header should have been added since hash should not have been found")
fuzorplugin.backend.redis.set(mailhash, 1, px=50)
fuzorplugin.examine(suspect)
time.sleep(50*1.0e-3) # sleep for 50ms to make sure key has expired
tag = suspect.get_tag('SAPlugin.tempheader')
self.assertIsNotNone(tag, "A header should have been added")
self.assertEqual(2, len(tag), "There should be two entries, one with the hash and one with the count")
self.assertEqual(["X-FuZor-ID: %s" % mailhash, "X-FuZor-Lvl: 1"], tag)
def test_client_info(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
helo, ip, revdns = suspect.client_info_from_rcvd(None, 0)
self.assertEquals(helo, 'helo1')
self.assertEquals(ip, '10.0.0.1')
self.assertEquals(revdns, 'rdns1')
helo, ip, revdns = suspect.client_info_from_rcvd(None, 1)
self.assertEquals(helo, 'helo2')
self.assertEquals(ip, '10.0.0.2')
self.assertEquals(revdns, 'rdns2')
helo, ip, revdns = suspect.client_info_from_rcvd('10\.0\.0\.2', 1)
self.assertEquals(helo, 'helo3')
self.assertEquals(ip, '10.0.0.3')
self.assertEquals(revdns, 'rdns3')
def get_suspect(self):
success = self.sess.getincomingmail()
if not success:
self.logger.error('incoming esmtp transfer did not finish')
return None
sess = self.sess
fromaddr = sess.from_address
toaddr = sess.to_address
tempfilename = sess.tempfilename
suspect = Suspect(fromaddr, toaddr, tempfilename)
suspect.recipients = set(sess.recipients)
if sess.xforward_helo is not None and sess.xforward_addr is not None and sess.xforward_rdns is not None:
suspect.clientinfo = sess.xforward_helo, sess.xforward_addr, sess.xforward_rdns
return suspect
def test_withSuspect_newDecode(self):
"""Test if new version of URIExtract gives same result as old one"""
myclass = self.__class__.__name__
functionNameAsString = sys._getframe().f_code.co_name
loggername = "%s.%s" % (myclass,functionNameAsString)
logger = logging.getLogger(loggername)
logger.debug("Read file content")
filecontent = BytesIO(mail_html).read()
logger.debug("Create suspect")
suspect = Suspect("*****@*****.**","*****@*****.**","/dev/null")
suspect.set_source(filecontent)
textparts_deprecated = self.candidate.get_decoded_textparts_deprecated(suspect)
textparts = self.candidate.get_decoded_textparts(suspect,bcompatible=False)
self.assertEqual(textparts_deprecated,textparts)
def test_script_normalexit(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
# we expect to find one test script
self.assertTrue(len(self.candidate.get_scripts()) == 1)
action, message = self.candidate.examine(suspect)
self.assertEqual(action, DUNNO)
self.assertEqual(message, 'accepted')
def test_hf(self):
"""Test header filters"""
suspect = Suspect('*****@*****.**',
'*****@*****.**', TESTDATADIR + '/helloworld.eml')
suspect.tags['testtag'] = 'testvalue'
headermatches = self.candidate.get_args(suspect)
self.failUnless(
'Sent to unittest domain!' in headermatches, "To_domain not found in headercheck")
self.failUnless('Envelope sender is [email protected]' in headermatches,
"Envelope Sender not matched in header chekc")
self.failUnless('Mime Version is 1.0' in headermatches,
"Standard header Mime Version not found")
self.failUnless(
'A tag match' in headermatches, "Tag match did not work")
self.failUnless(
'Globbing works' in headermatches, "header globbing failed")
self.failUnless(
'body rule works' in headermatches, "decoded body rule failed")
self.failUnless(
'full body rule works' in headermatches, "full body failed")
self.failUnless('mime rule works' in headermatches, "mime rule failed")
self.failIf('this should not match in a body rule' in headermatches,
'decoded body rule matched raw body')
# perl style advanced rules
self.failUnless('perl-style /-notation works!' in headermatches,
"new rule format failed: %s" % headermatches)
self.failUnless('perl-style recipient match' in headermatches,
"new rule format failed for to_domain: %s" % headermatches)
self.failIf('this should not match' in headermatches,
"rule flag ignorecase was not detected")
# TODO: raw body rules
# extended
(match, info) = self.candidate.matches(suspect, extended=True)
self.failUnless(match, 'Match should return True')
field, matchedvalue, arg, regex = info
self.failUnless(field == 'to_domain')
self.failUnless(matchedvalue == 'unittests.fuglu.org')
self.failUnless(arg == 'Sent to unittest domain!')
self.failUnless(regex == 'unittests\.fuglu\.org')
def _make_dummy_suspect(self, envelope_sender_domain='a.unittests.fuglu.org', header_from_domain='a.unittests.fuglu.org', recipient_domain='b.unittests.fuglu.org', file='/dev/null'):
s = Suspect('sender@%s' %
envelope_sender_domain, 'recipient@%s'%recipient_domain, file)
template="""From: sender@%s
Subject: Hello spear phished world!
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="----=_MIME_BOUNDARY_000_12140"
------=_MIME_BOUNDARY_000_12140
Content-Type: text/plain
blablabla
some <tagged>text</tagged>
------=_MIME_BOUNDARY_000_12140--
"""%header_from_domain
s.set_source(template)
return s
def test_virus(self):
"""Test if eicar is detected as virus"""
from fuglu.shared import Suspect
import shutil
self.mc.load_plugins()
if len(self.mc.plugins) == 0:
raise Exception("plugins not loaded")
sesshandler = SessionHandler(
None, self.mc.config, self.mc.prependers, self.mc.plugins, self.mc.appenders)
tempfilename = tempfile.mktemp(
suffix='virus', prefix='fuglu-unittest', dir='/tmp')
shutil.copy(TESTDATADIR + '/eicar.eml', tempfilename)
self.tempfiles.append(tempfilename)
suspect = Suspect(
'*****@*****.**', '*****@*****.**', tempfilename)
pluglist = sesshandler.run_prependers(suspect)
self.assertFalse(
len(pluglist) == 0, "Viruscheck will fail, pluginlist empty after run_prependers")
sesshandler.run_plugins(suspect, pluglist)
self.assertTrue(
suspect.is_virus(), "Eicar message was not detected as virus")
def test_vacation(self):
"""Test simple vacation use case"""
from fuglu.shared import Suspect
v = Vacation()
v.ignoresender = ""
v.awayuser = u'*****@*****.**'
v.created = datetime.now()
v.start = datetime.now()
v.end = v.start + timedelta(days=2)
v.subject = u'awaaay'
v.body = u'cya'
self.session.add(v)
self.session.flush()
self.session.expunge_all()
self.refreshcache()
suspect = Suspect(
u'*****@*****.**', '*****@*****.**', '/dev/null')
suspect.set_tag('nobounce', True)
candidatevacation = self.candidate.on_vacation(suspect)
self.assertTrue(
candidatevacation != None, "Vacation object not found in database")
self.assertTrue(self.candidate.should_send_vacation_message(
suspect), "Test Message should generate vacation reply")
self.candidate.log_bounce(suspect, candidatevacation)
# TODO: had to disable due to sqlalchemy error
# Instance <Vacation at 0x2938890> is not bound to a Session; attribute refresh operation cannot proceed
#self.assertFalse(self.candidate.should_send_vacation_message(suspect),"2nd test Message should NOT generate vacation reply")
suspect2 = Suspect(
u'*****@*****.**', '*****@*****.**', '/dev/null')
suspect2.set_tag('nobounce', True)
candidatevacation = self.candidate.on_vacation(suspect2)
self.assertFalse(candidatevacation != None,
"There should be no vacation object for this recipient")
self.assertFalse(self.candidate.should_send_vacation_message(
suspect2), "test Message should NOT generate vacation reply")
mailmessage[u'Date'] = formatdate()
# headers
if opts.headers:
for hdrpair in opts.headers:
name, val = hdrpair.split(':', 1)
try:
mailmessage.replace_header(name, val)
except KeyError:
mailmessage.add_header(name, val)
# create tempfile...
tmpfile = '/tmp/fuglu_dummy_message_in.eml'
open(tmpfile, 'w').write(mailmessage.as_string())
logging.info("Input file created as %s" % tmpfile)
suspect = Suspect(opts.sender, opts.recipients[0], tmpfile)
suspect.recipients = opts.recipients
# tags
if opts.tags:
for tagpair in opts.tags:
nme, valstr = tagpair.split(':', 1)
if valstr == 'TRUE':
val = True
elif valstr == 'FALSE':
val = False
else:
val = valstr
suspect.set_tag(nme, val)
scannerlist = mc.plugins
def _make_dummy_suspect(self, senderdomain, clientip, helo='foo.example.com'):
s = Suspect('sender@%s' %
senderdomain, '*****@*****.**', '/dev/null')
s.clientinfo = (helo, clientip, 'ptr.example.com')
return s