def test_extract_spamstatus(self):
"""Test if the spam status header gets extracted correctly"""
candidate = SAPlugin(self.config)
suspect = Suspect(
'*****@*****.**', '*****@*****.**', '/dev/null')
headername = 'X-Spam-Status'
headertests = [ # tuple header content, expected spamstatus, expected spam score
('YES', True, None), # _YESNOCAPS_
('NO', False, None), # _YESNOCAPS_
(' Yes, score=13.37', True, 13.37), # _YESNO_, score=_SCORE_
(' No, score=-2.826', False, -2.826), # _YESNO_, score=_SCORE_
# with test names, bug #24
("No, score=1.9 required=8.0 tests=BAYES_00,FROM_EQ_TO,TVD_SPACE_RATIO,TVD_SPACE_RATIO_MINFP autolearn=no autolearn_force=no version=3.4.0",
False, 1.9),
]
for headercontent, expectedspamstatus, expectedscore in headertests:
msgrep = Message()
msgrep[headername] = Header(headercontent).encode()
spamstatus, score = candidate._extract_spamstatus(
msgrep, headername, suspect)
self.assertEqual(spamstatus, expectedspamstatus, "spamstatus should be %s from %s" % (
expectedspamstatus, headercontent))
self.assertEqual(score, expectedscore, "spamscore should be %s from %s" % (
expectedscore, headercontent))
def test_extract_spamstatus(self):
"""Test if the spam status header gets extracted correctly"""
candidate = SAPlugin(self.config)
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
headername = 'X-Spam-Status'
headertests = [ # tuple header content, expected spamstatus, expected spam score
('YES', True, None), # _YESNOCAPS_
('NO', False, None), # _YESNOCAPS_
(' Yes, score=13.37', True, 13.37), # _YESNO_, score=_SCORE_
(' No, score=-2.826', False, -2.826), # _YESNO_, score=_SCORE_
# with test names, bug #24
("No, score=1.9 required=8.0 tests=BAYES_00,FROM_EQ_TO,TVD_SPACE_RATIO,TVD_SPACE_RATIO_MINFP autolearn=no autolearn_force=no version=3.4.0",
False, 1.9),
]
for headercontent, expectedspamstatus, expectedscore in headertests:
msgrep = Message()
msgrep[headername] = Header(headercontent).encode()
spamstatus, score, report = candidate._extract_spamstatus(
msgrep, headername, suspect)
self.assertEqual(
spamstatus, expectedspamstatus,
"spamstatus should be %s from %s" %
(expectedspamstatus, headercontent))
self.assertEqual(
score, expectedscore, "spamscore should be %s from %s" %
(expectedscore, headercontent))
def test_extract_spamstatus_fail(self):
"""Test correct return if extracting spamstatus fails because of missing header"""
candidate = SAPlugin(self.config)
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
headername = 'X-Spam-Status'
msgrep = Message()
spamstatus, score, report = candidate._extract_spamstatus(
msgrep, headername, suspect)
def setUp(self):
try:
from configparser import RawConfigParser
except ImportError:
from ConfigParser import RawConfigParser
config = RawConfigParser()
config.add_section('main')
config.set('main', 'prependaddedheaders', 'X-Fuglu-')
config.add_section('SAPlugin')
config.set('SAPlugin', 'host', '127.0.0.1')
config.set('SAPlugin', 'port', '783')
config.set('SAPlugin', 'timeout', '5')
config.set('SAPlugin', 'retries', '3')
config.set('SAPlugin', 'peruserconfig', '0')
config.set('SAPlugin', 'maxsize', '500000')
config.set('SAPlugin', 'spamheader', 'X-Spam-Status')
config.set('SAPlugin', 'lowspamaction', 'DUNNO')
config.set('SAPlugin', 'highspamaction', 'REJECT')
config.set('SAPlugin', 'problemaction', 'DEFER')
config.set('SAPlugin', 'highspamlevel', '15')
config.set('SAPlugin', 'forwardoriginal', 'False')
config.set('SAPlugin', 'scanoriginal', 'False')
config.set('SAPlugin', 'rejectmessage', '')
config.set('SAPlugin', 'strip_oversize', '0')
# sql blacklist
testfile = "/tmp/sa_test.db"
if os.path.exists(testfile):
os.remove(testfile)
# important: 4 slashes for absolute paths!
self.testdb = "sqlite:///%s" % testfile
sql = """SELECT value FROM userpref WHERE preference='blacklist_from' AND username in ('@GLOBAL','%' || ${to_domain},${to_address})"""
config.set('SAPlugin', 'sql_blacklist_dbconnectstring', self.testdb)
config.set('SAPlugin', 'sql_blacklist_sql', sql)
config.set('SAPlugin', 'check_sql_blacklist', 'False')
self.candidate = SAPlugin(config)
def setUp(self):
try:
from configparser import RawConfigParser
except ImportError:
from ConfigParser import RawConfigParser
config = RawConfigParser()
config.add_section('main')
config.set('main', 'prependaddedheaders', 'X-Fuglu-')
config.add_section('SAPlugin')
config.set('SAPlugin', 'host', '127.0.0.1')
config.set('SAPlugin', 'port', '783')
config.set('SAPlugin', 'timeout', '5')
config.set('SAPlugin', 'retries', '3')
config.set('SAPlugin', 'peruserconfig', '0')
config.set('SAPlugin', 'maxsize', '500000')
config.set('SAPlugin', 'spamheader', 'X-Spam-Status')
config.set('SAPlugin', 'lowspamaction', 'DUNNO')
config.set('SAPlugin', 'highspamaction', 'REJECT')
config.set('SAPlugin', 'problemaction', 'DEFER')
config.set('SAPlugin', 'highspamlevel', '15')
config.set('SAPlugin', 'forwardoriginal', 'False')
config.set('SAPlugin', 'scanoriginal', 'False')
config.set('SAPlugin', 'rejectmessage', '')
# sql blacklist
testfile = "/tmp/sa_test.db"
if os.path.exists(testfile):
os.remove(testfile)
# important: 4 slashes for absolute paths!
self.testdb = "sqlite:///%s" % testfile
sql = """SELECT value FROM userpref WHERE preference='blacklist_from' AND username in ('@GLOBAL','%' || ${to_domain},${to_address})"""
config.set('SAPlugin', 'sql_blacklist_dbconnectstring', self.testdb)
config.set('SAPlugin', 'sql_blacklist_sql', sql)
config.set('SAPlugin', 'check_sql_blacklist', 'False')
self.candidate = SAPlugin(config)
def setUp(self):
from ConfigParser import RawConfigParser
config = RawConfigParser()
config.add_section("main")
config.set("main", "prependaddedheaders", "X-Fuglu-")
config.add_section("SAPlugin")
config.set("SAPlugin", "host", "127.0.0.1")
config.set("SAPlugin", "port", "783")
config.set("SAPlugin", "timeout", "5")
config.set("SAPlugin", "retries", "3")
config.set("SAPlugin", "peruserconfig", "0")
config.set("SAPlugin", "maxsize", "500000")
config.set("SAPlugin", "spamheader", "X-Spam-Status")
config.set("SAPlugin", "lowspamaction", "DUNNO")
config.set("SAPlugin", "highspamaction", "REJECT")
config.set("SAPlugin", "problemaction", "DEFER")
config.set("SAPlugin", "highspamlevel", "15")
config.set("SAPlugin", "forwardoriginal", "False")
config.set("SAPlugin", "scanoriginal", "False")
config.set("SAPlugin", "rejectmessage", "")
# sql blacklist
testfile = "/tmp/sa_test.db"
if os.path.exists(testfile):
os.remove(testfile)
# important: 4 slashes for absolute paths!
self.testdb = "sqlite:///%s" % testfile
sql = """SELECT value FROM userpref WHERE preference='blacklist_from' AND username in ('@GLOBAL','%' || ${to_domain},${to_address})"""
config.set("SAPlugin", "sql_blacklist_dbconnectstring", self.testdb)
config.set("SAPlugin", "sql_blacklist_sql", sql)
config.set("SAPlugin", "check_sql_blacklist", "False")
self.candidate = SAPlugin(config)
def setUp(self):
"""Tests a message that is stripped and forwared in a modified way"""
config = RawConfigParser()
config.add_section('main')
config.set('main', 'prependaddedheaders', 'X-Fuglu-')
config.add_section('SAPlugin')
config.set('SAPlugin', 'host', '127.0.0.1')
config.set('SAPlugin', 'port', '783')
config.set('SAPlugin', 'timeout', '5')
config.set('SAPlugin', 'retries', '3')
config.set('SAPlugin', 'peruserconfig', '0')
config.set('SAPlugin', 'maxsize', '50')
config.set('SAPlugin', 'spamheader', 'X-Spam-Status')
config.set('SAPlugin', 'lowspamaction', 'DUNNO')
config.set('SAPlugin', 'highspamaction', 'REJECT')
config.set('SAPlugin', 'problemaction', 'DEFER')
config.set('SAPlugin', 'highspamlevel', '15')
config.set('SAPlugin', 'forwardoriginal', 'False')
config.set('SAPlugin', 'scanoriginal', 'False')
config.set('SAPlugin', 'rejectmessage', '')
config.set('SAPlugin', 'strip_oversize', '1')
config.set('SAPlugin', 'spamheader_prepend', 'X-Spam-')
saplug = SAPlugin(config)
saplug.safilter = MagicMock()
saplug.safilter.return_value = b'Received: from localhost by unknown\n' \
b'\twith SpamAssassin (version 3.4.2);\n' \
b'\tWed, 28 Nov 2018 08:43:56 +0100\n' \
b'X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on unknown\n' \
b'X-Spam-Flag: YES\n' \
b'X-Spam-Level: *********\n' \
b'X-Spam-Status: Yes, score=9.8 required=5.0 tests=EMPTY_MESSAGE,\n' \
b'\tMIME_HEADER_CTYPE_ONLY,MISSING_DATE,MISSING_FROM,MISSING_HEADERS,\n' \
b'\tMISSING_MID,MISSING_SUBJECT,NO_HEADERS_MESSAGE,NO_RECEIVED,NO_RELAYS\n' \
b'\tautolearn=no autolearn_force=no version=3.4.2\n' \
b'MIME-Version: 1.0\n' \
b'Content-Type: multipart/mixed; boundary="----------=_5BFE473C.76E9108C"\n' \
b'\n' \
b'This is a multi-part message in MIME format.\n' \
b'\n' \
b'------------=_5BFE473C.76E9108C\n' \
b'Content-Type: text/plain; charset=iso-8859-1\n' \
b'Content-Disposition: inline\n' \
b'Content-Transfer-Encoding: 8bit\n' \
b'\n' \
b'Spam detection software, running on the system "unknown",\n' \
b'has identified this incoming email as possible spam. The original\n' \
b'message has been attached to this so you can view it or label\n' \
b'similar future email. If you have any questions, see\n' \
b'@@CONTACT_ADDRESS@@ for details.\n' \
b'\n' \
b'Content preview: \n' \
b'\n' \
b'Content analysis details: (9.8 points, 5.0 required)\n' \
b'\n' \
b' pts rule name description\n' \
b'---- ---------------------- --------------------------------------------------\n' \
b'-0.0 NO_RELAYS Informational: message was not relayed via SMTP\n' \
b' 1.2 MISSING_HEADERS Missing To: header\n' \
b' 1.4 MISSING_DATE Missing Date: header\n' \
b' 2.0 MIME_HEADER_CTYPE_ONLY \'Content-Type\' found without required\n' \
b' MIME headers\n' \
b' 1.8 MISSING_SUBJECT Missing Subject: header\n' \
b' 1.0 MISSING_FROM Missing From: header\n' \
b' 0.1 MISSING_MID Missing Message-Id: header\n' \
b' 2.3 EMPTY_MESSAGE Message appears to have no textual parts and no\n' \
b' Subject: text\n' \
b'-0.0 NO_RECEIVED Informational: message has no Received headers\n' \
b' 0.0 NO_HEADERS_MESSAGE Message appears to be missing most RFC-822\n' \
b' headers\n' \
b'\n' \
b'The original message was not completely plain text, and may be unsafe to\n' \
b'open with some email clients; in particular, it may contain a virus,\n' \
b'or confirm that your address can receive spam. If you wish to view\n' \
b'it, it may be safer to save it to a file and open it with an editor.\n' \
b'\n' \
b'\n' \
b'------------=_5BFE473C.76E9108C\n' \
b'Content-Type: message/rfc822; x-spam-type=original\n' \
b'Content-Description: original message before SpamAssassin\n' \
b'Content-Disposition: attachment\n' \
b'Content-Transfer-Encoding: 8bit\n' \
b'\n' \
b'Content-Type: multipart/mixed; boundary="========' \
b'\n------------=_5BFE473C.76E9108C--\n' \
b'\n'
self.saplugin = saplug
class SAPluginTestCase(unittest.TestCase):
"""Testcases for the Stub Plugin"""
def setUp(self):
from ConfigParser import RawConfigParser
config = RawConfigParser()
config.add_section('main')
config.set('main', 'prependaddedheaders', 'X-Fuglu-')
config.add_section('SAPlugin')
config.set('SAPlugin', 'host', '127.0.0.1')
config.set('SAPlugin', 'port', '783')
config.set('SAPlugin', 'timeout', '5')
config.set('SAPlugin', 'retries', '3')
config.set('SAPlugin', 'peruserconfig', '0')
config.set('SAPlugin', 'maxsize', '500000')
config.set('SAPlugin', 'spamheader', 'X-Spam-Status')
config.set('SAPlugin', 'lowspamaction', 'DUNNO')
config.set('SAPlugin', 'highspamaction', 'REJECT')
config.set('SAPlugin', 'problemaction', 'DEFER')
config.set('SAPlugin', 'highspamlevel', '15')
config.set('SAPlugin', 'forwardoriginal', 'False')
config.set('SAPlugin', 'scanoriginal', 'False')
config.set('SAPlugin', 'rejectmessage', '')
# sql blacklist
testfile = "/tmp/sa_test.db"
if os.path.exists(testfile):
os.remove(testfile)
# important: 4 slashes for absolute paths!
self.testdb = "sqlite:///%s" % testfile
sql = """SELECT value FROM userpref WHERE preference='blacklist_from' AND username in ('@GLOBAL','%' || ${to_domain},${to_address})"""
config.set('SAPlugin', 'sql_blacklist_dbconnectstring', self.testdb)
config.set('SAPlugin', 'sql_blacklist_sql', sql)
config.set('SAPlugin', 'check_sql_blacklist', 'False')
self.candidate = SAPlugin(config)
def test_score(self):
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test scanner
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
"""
suspect.set_source(stream)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
score = int(suspect.get_tag('SAPlugin.spamscore'))
self.failUnless(score > 999,
"GTUBE mails should score ~1000 , we got %s" % score)
self.failUnless(result == REJECT, 'High spam should be rejected')
def test_symbols(self):
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test scanner
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
"""
spamstatus, spamscore, rules = self.candidate.safilter_symbols(
stream, '*****@*****.**')
self.failUnless('GTUBE' in rules, "GTUBE not found in SYMBOL scan")
self.failIf(spamscore < 500)
self.failUnless(spamstatus)
stream2 = """Received: from mail.python.org (mail.python.org [82.94.164.166])
by bla.fuglu.org (Postfix) with ESMTPS id 395743E03A5
for <*****@*****.**>; Sun, 22 Aug 2010 18:15:11 +0200 (CEST)
Date: Tue, 24 Aug 2010 09:20:57 +0200
To: [email protected]
From: [email protected]
Subject: test Tue, 24 Aug 2010 09:20:57 +0200
X-Mailer: swaks v20061116.0 jetmore.org/john/code/#swaks
Message-Id: <*****@*****.**>
This is a test mailing """
spamstatus, spamscore, rules = self.candidate.safilter_symbols(
stream2, '*****@*****.**')
# print rules
self.failIf(spamstatus, "This message should not be detected as spam")
def test_sql_blacklist(self):
self.candidate.config.set('SAPlugin', 'check_sql_blacklist', 'True')
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
import fuglu.extensions.sql
if not fuglu.extensions.sql.ENABLED:
print "Excluding test that needs sqlalchemy extension"
return
session = fuglu.extensions.sql.get_session(self.testdb)
createsql = """CREATE TABLE userpref (
username varchar(100) NOT NULL DEFAULT '',
preference varchar(30) NOT NULL DEFAULT '',
value varchar(100) NOT NULL DEFAULT ''
)"""
session.execute(createsql)
self.assertEquals(self.candidate.check_sql_blacklist(suspect),
DUNNO), 'sender is not blacklisted'
insertsql = """INSERT INTO userpref (username,preference,value) VALUES ('%unittests.fuglu.org','blacklist_from','*@unittests.fuglu.org')"""
session.execute(insertsql)
self.assertEquals(self.candidate.check_sql_blacklist(suspect),
REJECT), 'sender should be blacklisted'
fuglu.extensions.sql.ENABLED = False
self.assertEquals(self.candidate.check_sql_blacklist(suspect),
DUNNO), 'problem if sqlalchemy is not available'
fuglu.extensions.sql.ENABLED = True
self.candidate.config.set('SAPlugin', 'sql_blacklist_sql',
'this is a buggy sql statement')
self.assertEquals(self.candidate.check_sql_blacklist(suspect),
DUNNO), 'error coping with db problems'
# simulate unavailable db
self.candidate.config.set('SAPlugin', 'sql_blacklist_dbconnectstring',
'mysql://127.0.0.1:9977/idonotexist')
self.assertEquals(self.candidate.check_sql_blacklist(suspect),
DUNNO), 'error coping with db problems'
class SAPluginTestCase(unittest.TestCase):
"""Testcases for the Stub Plugin"""
def setUp(self):
from ConfigParser import RawConfigParser
config = RawConfigParser()
config.add_section("main")
config.set("main", "prependaddedheaders", "X-Fuglu-")
config.add_section("SAPlugin")
config.set("SAPlugin", "host", "127.0.0.1")
config.set("SAPlugin", "port", "783")
config.set("SAPlugin", "timeout", "5")
config.set("SAPlugin", "retries", "3")
config.set("SAPlugin", "peruserconfig", "0")
config.set("SAPlugin", "maxsize", "500000")
config.set("SAPlugin", "spamheader", "X-Spam-Status")
config.set("SAPlugin", "lowspamaction", "DUNNO")
config.set("SAPlugin", "highspamaction", "REJECT")
config.set("SAPlugin", "problemaction", "DEFER")
config.set("SAPlugin", "highspamlevel", "15")
config.set("SAPlugin", "forwardoriginal", "False")
config.set("SAPlugin", "scanoriginal", "False")
config.set("SAPlugin", "rejectmessage", "")
# sql blacklist
testfile = "/tmp/sa_test.db"
if os.path.exists(testfile):
os.remove(testfile)
# important: 4 slashes for absolute paths!
self.testdb = "sqlite:///%s" % testfile
sql = """SELECT value FROM userpref WHERE preference='blacklist_from' AND username in ('@GLOBAL','%' || ${to_domain},${to_address})"""
config.set("SAPlugin", "sql_blacklist_dbconnectstring", self.testdb)
config.set("SAPlugin", "sql_blacklist_sql", sql)
config.set("SAPlugin", "check_sql_blacklist", "False")
self.candidate = SAPlugin(config)
def test_score(self):
suspect = Suspect("*****@*****.**", "*****@*****.**", "/dev/null")
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test scanner
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
"""
suspect.set_source(stream)
result = self.candidate.examine(suspect)
if type(result) is tuple:
result, message = result
score = int(suspect.get_tag("SAPlugin.spamscore"))
self.failUnless(score > 999, "GTUBE mails should score ~1000 , we got %s" % score)
self.failUnless(result == REJECT, "High spam should be rejected")
def test_symbols(self):
stream = """Date: Mon, 08 Sep 2008 17:33:54 +0200
To: [email protected]
From: [email protected]
Subject: test scanner
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
"""
spamstatus, spamscore, rules = self.candidate.safilter_symbols(stream, "*****@*****.**")
self.failUnless("GTUBE" in rules, "GTUBE not found in SYMBOL scan")
self.failIf(spamscore < 500)
self.failUnless(spamstatus)
stream2 = """Received: from mail.python.org (mail.python.org [82.94.164.166])
by bla.fuglu.org (Postfix) with ESMTPS id 395743E03A5
for <*****@*****.**>; Sun, 22 Aug 2010 18:15:11 +0200 (CEST)
Date: Tue, 24 Aug 2010 09:20:57 +0200
To: [email protected]
From: [email protected]
Subject: test Tue, 24 Aug 2010 09:20:57 +0200
X-Mailer: swaks v20061116.0 jetmore.org/john/code/#swaks
Message-Id: <*****@*****.**>
This is a test mailing """
spamstatus, spamscore, rules = self.candidate.safilter_symbols(stream2, "*****@*****.**")
# print rules
self.failIf(spamstatus, "This message should not be detected as spam")
def test_sql_blacklist(self):
self.candidate.config.set("SAPlugin", "check_sql_blacklist", "True")
suspect = Suspect("*****@*****.**", "*****@*****.**", "/dev/null")
import fuglu.extensions.sql
if not fuglu.extensions.sql.ENABLED:
print "Excluding test that needs sqlalchemy extension"
return
session = fuglu.extensions.sql.get_session(self.testdb)
createsql = """CREATE TABLE userpref (
username varchar(100) NOT NULL DEFAULT '',
preference varchar(30) NOT NULL DEFAULT '',
value varchar(100) NOT NULL DEFAULT ''
)"""
session.execute(createsql)
self.assertEquals(self.candidate.check_sql_blacklist(suspect), DUNNO), "sender is not blacklisted"
insertsql = """INSERT INTO userpref (username,preference,value) VALUES ('%unittests.fuglu.org','blacklist_from','*@unittests.fuglu.org')"""
session.execute(insertsql)
self.assertEquals(self.candidate.check_sql_blacklist(suspect), REJECT), "sender should be blacklisted"
fuglu.extensions.sql.ENABLED = False
self.assertEquals(self.candidate.check_sql_blacklist(suspect), DUNNO), "problem if sqlalchemy is not available"
fuglu.extensions.sql.ENABLED = True
self.candidate.config.set("SAPlugin", "sql_blacklist_sql", "this is a buggy sql statement")
self.assertEquals(self.candidate.check_sql_blacklist(suspect), DUNNO), "error coping with db problems"
# simulate unavailable db
self.candidate.config.set("SAPlugin", "sql_blacklist_dbconnectstring", "mysql://127.0.0.1:9977/idonotexist")
self.assertEquals(self.candidate.check_sql_blacklist(suspect), DUNNO), "error coping with db problems"