def test_backend_reload(self):
"""Test reload with different backends"""
config = RawConfigParser()
config.add_section('performance')
# minimum scanner threads
config.set('performance', 'minthreads', 2)
# maximum scanner threads
config.set('performance', 'maxthreads', 40)
# Method for parallelism, either 'thread' or 'process'
config.set('performance', 'backend', 'process')
# Initial number of processes when backend='process'.
# If 0 (the default), automatically selects twice the number of available virtual cores.
# Despite its 'initial'-name, this number currently is not adapted automatically.
config.set('performance', 'initialprocs', 10)
config.set('performance', 'join_timeout', 2.0)
mc = MainController(config)
mc.propagate_core_defaults()
# usually the backend is loaded by "startup()" which is run
# in a separate thread because it goes to the event loop. I'll just
# directly start a the threadpool here...
mc.threadpool = mc._start_threadpool()
time.sleep(0.1)
try:
self.assertIsNone(mc.procpool)
self.assertIsNotNone(mc.threadpool)
except AttributeError:
# Python 2.6
self.assertTrue(mc.procpool is None)
self.assertTrue(mc.threadpool is not None)
# now reload will replace the threadpool by a procpool
mc.reload()
time.sleep(0.1)
try:
self.assertIsNone(mc.threadpool)
self.assertIsNotNone(mc.procpool)
except AttributeError:
# Python 2.6
self.assertTrue(mc.threadpool is None)
self.assertTrue(mc.procpool is not None)
config.set('performance', 'backend', 'thread')
# now reload will replace the procpool by a threadpool
mc.reload()
time.sleep(0.1)
try:
self.assertIsNone(mc.procpool)
self.assertIsNotNone(mc.threadpool)
except AttributeError:
# Python 2.6
self.assertTrue(mc.procpool is None)
self.assertTrue(mc.threadpool is not None)
mc.shutdown()
def test_multiple_mcs(self):
"""Just start multiple controllers """
config = RawConfigParser()
mclist = []
for i in range(10):
mc = MainController(config)
mc.propagate_core_defaults()
mclist.append(mc)
for mc in mclist:
# usually the backend is loaded by "startup()" which is run
# in a separate thread because it goes to the event loop. I'll just
# directly start a the threadpool here...
mc.threadpool = mc._start_threadpool()
time.sleep(0.1)
for mc in mclist:
mc.shutdown()
def test_multiple_mcs_reload(self):
"""
Even if there are multiple MainControllers they should not cause crashes as long as they
don't start control servers...
"""
config = RawConfigParser()
config.add_section('performance')
# minimum scanner threads
config.set('performance', 'minthreads', 2)
# maximum scanner threads
config.set('performance', 'maxthreads', 40)
# Method for parallelism, either 'thread' or 'process'
config.set('performance', 'backend', 'process')
# Initial number of processes when backend='process'.
# If 0 (the default), automatically selects twice the number of available virtual cores.
# Despite its 'initial'-name, this number currently is not adapted automatically.
config.set('performance', 'initialprocs', 0)
config = RawConfigParser()
mclist = []
for i in range(3):
mc = MainController(config)
mc.propagate_core_defaults()
mclist.append(mc)
for mc in mclist:
# usually the backend is loaded by "startup()" which is run
# in a separate thread because it goes to the event loop. I'll just
# directly start a the threadpool here...
mc.threadpool = mc._start_threadpool()
time.sleep(0.1)
for mc in mclist:
mc.reload()
time.sleep(0.1)
for mc in mclist:
mc.shutdown()
class EndtoEndTestTestCase(unittest.TestCase):
"""Full check if mail runs through"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7711
DUMMY_PORT = 7712
FUGLUCONTROL_PORT = 7713
def setUp(self):
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set('main', 'incomingport',
str(EndtoEndTestTestCase.FUGLU_PORT))
self.config.set('main', 'outgoinghost',
str(EndtoEndTestTestCase.FUGLU_HOST))
self.config.set('main', 'outgoingport',
str(EndtoEndTestTestCase.DUMMY_PORT))
self.config.set('main', 'controlport',
str(EndtoEndTestTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(self.config,
EndtoEndTestTestCase.DUMMY_PORT,
EndtoEndTestTestCase.FUGLU_HOST)
e2edss = threading.Thread(target=self.smtp.serve, args=())
e2edss.daemon = True
e2edss.start()
# start fuglu's listening server
fls = threading.Thread(target=self.mc.startup, args=())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testE2E(self):
"""test if a standard message runs through"""
# give fuglu time to start listener
time.sleep(1)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', EndtoEndTestTestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.e2e')
testmessage = """Hello World!\r
Don't dare you change any of my bytes or even remove one!"""
# TODO: this test fails if we don't put in the \r in there... (eg,
# fuglu adds it) - is this a bug or wrong test?
msg = MIMEText(testmessage)
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
inbytes = len(msg.get_payload())
smtpclient.sendmail('*****@*****.**', '*****@*****.**',
msgstring)
smtpclient.quit()
# get answer
gotback = self.smtp.suspect
self.assertFalse(gotback == None,
"Did not get message from dummy smtp server")
# check a few things on the received message
msgrep = gotback.get_message_rep()
self.assertTrue('X-Fuglutest-Spamstatus' in msgrep,
"Fuglu SPAM Header not found in message")
payload = msgrep.get_payload()
outbytes = len(payload)
self.assertEqual(
testmessage, payload,
"Message body has been altered. In: %s bytes, Out: %s bytes, teststring=->%s<- result=->%s<-"
% (inbytes, outbytes, testmessage, payload))
class SMIMETestCase(unittest.TestCase):
"""Email Signature Tests"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7721
DUMMY_PORT = 7722
FUGLUCONTROL_PORT = 7723
def setUp(self):
time.sleep(5)
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set('main', 'incomingport', str(SMIMETestCase.FUGLU_PORT))
self.config.set('main', 'outgoinghost', str(SMIMETestCase.FUGLU_HOST))
self.config.set('main', 'outgoingport', str(SMIMETestCase.DUMMY_PORT))
self.config.set('main', 'controlport',
str(SMIMETestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(self.config, SMIMETestCase.DUMMY_PORT,
SMIMETestCase.FUGLU_HOST)
smdss = threading.Thread(target=self.smtp.serve, args=())
smdss.daemon = True
smdss.start()
# start fuglu's listening server
fls = threading.Thread(target=self.mc.startup, args=())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testSMIME(self):
"""test if S/MIME mails still pass the signature"""
# give fuglu time to start listener
time.sleep(1)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', SMIMETestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.smime')
inputfile = TESTDATADIR + '/smime/signedmessage.eml'
(status, output) = self.verifyOpenSSL(inputfile)
self.assertTrue(status == 0,
"Testdata S/MIME verification failed: \n%s" % output)
msgstring = open(inputfile, 'r').read()
smtpclient.sendmail('*****@*****.**',
'*****@*****.**', msgstring)
smtpclient.quit()
# verify the smtp server stored the file correctly
tmpfile = self.smtp.tempfilename
#self.failUnlessEqual(msgstring, tmpcontent, "SMTP Server did not store the tempfile correctly: %s"%tmpfile)
(status, output) = self.verifyOpenSSL(tmpfile)
self.assertTrue(
status == 0, "S/MIME verification failed: \n%s\n tmpfile is:%s" %
(output, tmpfile))
def verifyOpenSSL(self, file):
(status, output) = getstatusoutput(
"openssl smime -verify -noverify -in %s" % file)
return (status, output)
class DKIMTestCase(unittest.TestCase):
"""DKIM Sig Test"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7731
DUMMY_PORT = 7732
FUGLUCONTROL_PORT = 7733
def setUp(self):
k = ''
for line in open(TESTDATADIR + '/dkim/testfuglu.org.public'):
if line.startswith('---'):
continue
k = k + line.strip()
record = "v=DKIM1; k=rsa; p=%s" % k
fuglu.lib.patcheddkimlib.dnstxt = mock.Mock(return_value=record)
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set('main', 'incomingport', str(DKIMTestCase.FUGLU_PORT))
self.config.set('main', 'outgoinghost', str(DKIMTestCase.FUGLU_HOST))
self.config.set('main', 'outgoingport', str(DKIMTestCase.DUMMY_PORT))
self.config.set('main', 'controlport',
str(DKIMTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(self.config,
self.config.getint('main', 'outgoingport'),
DKIMTestCase.FUGLU_HOST)
dkdss = threading.Thread(target=self.smtp.serve, args=())
dkdss.daemon = True
dkdss.start()
# start fuglu's listening server
fls = threading.Thread(target=self.mc.startup, args=())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testDKIM(self):
# give fuglu time to start listener
time.sleep(1)
inputfile = TESTDATADIR + '/helloworld.eml'
msgstring = open(inputfile, 'r').read()
dkimheader = sign(msgstring,
'whatever',
'testfuglu.org',
open(TESTDATADIR +
'/dkim/testfuglu.org.private').read(),
include_headers=['From', 'To'])
signedcontent = dkimheader + msgstring
logbuffer = StringIO()
self.assertTrue(
verify(signedcontent, debuglog=logbuffer),
"Failed DKIM verification immediately after signing %s" %
logbuffer.getvalue())
# send test message
try:
smtpclient = smtplib.SMTP('127.0.0.1', DKIMTestCase.FUGLU_PORT)
except Exception as e:
self.fail("Could not connect to fuglu on port %s : %s" %
(DKIMTestCase.FUGLU_PORT, str(e)))
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.dkim')
smtpclient.sendmail('*****@*****.**',
'*****@*****.**', signedcontent)
smtpclient.quit()
# verify the smtp server stored the file correctly
tmpfile = self.smtp.tempfilename
self.assertTrue(tmpfile != None, 'Send to dummy smtp server failed')
result = open(tmpfile, 'r').read()
logbuffer = StringIO()
verify_ok = verify(result, debuglog=logbuffer)
self.assertTrue(verify_ok,
"Failed DKIM verification: %s" % logbuffer.getvalue())
class EndtoEndTestTestCase(unittest.TestCase):
"""Full check if mail runs through"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7711
DUMMY_PORT = 7712
FUGLUCONTROL_PORT = 7713
def setUp(self):
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set(
'main', 'incomingport', str(EndtoEndTestTestCase.FUGLU_PORT))
self.config.set(
'main', 'outgoinghost', str(EndtoEndTestTestCase.FUGLU_HOST))
self.config.set(
'main', 'outgoingport', str(EndtoEndTestTestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(EndtoEndTestTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(
self.config, EndtoEndTestTestCase.DUMMY_PORT, EndtoEndTestTestCase.FUGLU_HOST)
e2edss = threading.Thread(target = self.smtp.serve, args = ())
e2edss.daemon = True
e2edss.start()
# start fuglu's listening server
fls = threading.Thread(target = self.mc.startup, args = ())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testE2E(self):
"""test if a standard message runs through"""
# give fuglu time to start listener
time.sleep(1)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', EndtoEndTestTestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.e2e')
testmessage = """Hello World!\r
Don't dare you change any of my bytes or even remove one!"""
# TODO: this test fails if we don't put in the \r in there... (eg,
# fuglu adds it) - is this a bug or wrong test?
msg = MIMEText(testmessage)
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
inbytes = len(msg.get_payload())
smtpclient.sendmail(
'*****@*****.**', '*****@*****.**', msgstring)
smtpclient.quit()
# get answer
gotback = self.smtp.suspect
self.assertFalse(
gotback == None, "Did not get message from dummy smtp server")
# check a few things on the received message
msgrep = gotback.get_message_rep()
self.assertTrue('X-Fuglutest-Spamstatus' in msgrep,
"Fuglu SPAM Header not found in message")
payload = msgrep.get_payload()
outbytes = len(payload)
self.assertEqual(testmessage, payload, "Message body has been altered. In: %s bytes, Out: %s bytes, teststring=->%s<- result=->%s<-" %
(inbytes, outbytes, testmessage, payload))
class SMIMETestCase(unittest.TestCase):
"""Email Signature Tests"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7721
DUMMY_PORT = 7722
FUGLUCONTROL_PORT = 7723
def setUp(self):
time.sleep(5)
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set('main', 'incomingport', str(SMIMETestCase.FUGLU_PORT))
self.config.set('main', 'outgoinghost', str(SMIMETestCase.FUGLU_HOST))
self.config.set('main', 'outgoingport', str(SMIMETestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(SMIMETestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(
self.config, SMIMETestCase.DUMMY_PORT, SMIMETestCase.FUGLU_HOST)
smdss = threading.Thread(target = self.smtp.serve, args=())
smdss.daemon = True
smdss.start()
# start fuglu's listening server
fls = threading.Thread(target = self.mc.startup, args = ())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testSMIME(self):
"""test if S/MIME mails still pass the signature"""
# give fuglu time to start listener
time.sleep(1)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', SMIMETestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.smime')
inputfile = TESTDATADIR + '/smime/signedmessage.eml'
(status, output) = self.verifyOpenSSL(inputfile)
self.assertTrue(
status == 0, "Testdata S/MIME verification failed: \n%s" % output)
msgstring = open(inputfile, 'r').read()
smtpclient.sendmail(
'*****@*****.**', '*****@*****.**', msgstring)
smtpclient.quit()
# verify the smtp server stored the file correctly
tmpfile = self.smtp.tempfilename
#self.failUnlessEqual(msgstring, tmpcontent, "SMTP Server did not store the tempfile correctly: %s"%tmpfile)
(status, output) = self.verifyOpenSSL(tmpfile)
self.assertTrue(
status == 0, "S/MIME verification failed: \n%s\n tmpfile is:%s" % (output, tmpfile))
def verifyOpenSSL(self, file):
(status, output) = getstatusoutput(
"openssl smime -verify -noverify -in %s" % file)
return (status, output)
class DKIMTestCase(unittest.TestCase):
"""DKIM Sig Test"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7731
DUMMY_PORT = 7732
FUGLUCONTROL_PORT = 7733
def setUp(self):
k = ''
for line in open(TESTDATADIR + '/dkim/testfuglu.org.public'):
if line.startswith('---'):
continue
k = k + line.strip()
record = "v=DKIM1; k=rsa; p=%s" % k
fuglu.lib.patcheddkimlib.dnstxt = mock.Mock(return_value=record)
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set('main', 'incomingport', str(DKIMTestCase.FUGLU_PORT))
self.config.set('main', 'outgoinghost', str(DKIMTestCase.FUGLU_HOST))
self.config.set('main', 'outgoingport', str(DKIMTestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(DKIMTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(self.config, self.config.getint(
'main', 'outgoingport'), DKIMTestCase.FUGLU_HOST)
dkdss = threading.Thread(target = self.smtp.serve, args = ())
dkdss.daemon = True
dkdss.start()
# start fuglu's listening server
fls = threading.Thread(target = self.mc.startup, args = ())
fls.daemon = True
fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
def testDKIM(self):
# give fuglu time to start listener
time.sleep(1)
inputfile = TESTDATADIR + '/helloworld.eml'
msgstring = open(inputfile, 'r').read()
dkimheader = sign(msgstring, 'whatever', 'testfuglu.org', open(
TESTDATADIR + '/dkim/testfuglu.org.private').read(), include_headers=['From', 'To'])
signedcontent = dkimheader + msgstring
logbuffer = StringIO()
self.assertTrue(verify(signedcontent, debuglog=logbuffer),
"Failed DKIM verification immediately after signing %s" % logbuffer.getvalue())
# send test message
try:
smtpclient = smtplib.SMTP('127.0.0.1', DKIMTestCase.FUGLU_PORT)
except Exception as e:
self.fail("Could not connect to fuglu on port %s : %s" %
(DKIMTestCase.FUGLU_PORT, str(e)))
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.dkim')
smtpclient.sendmail(
'*****@*****.**', '*****@*****.**', signedcontent)
smtpclient.quit()
# verify the smtp server stored the file correctly
tmpfile = self.smtp.tempfilename
self.assertTrue(tmpfile != None, 'Send to dummy smtp server failed')
result = open(tmpfile, 'r').read()
logbuffer = StringIO()
verify_ok = verify(result, debuglog=logbuffer)
self.assertTrue(
verify_ok, "Failed DKIM verification: %s" % logbuffer.getvalue())
class ReloadUnderLoadTest(unittest.TestCase):
"""Reload backend under load"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7841
DUMMY_PORT = 7842
FUGLUCONTROL_PORT = 7843
delay_by = 0.25 # seconds
num_procs = 5
def setUp(self):
logger = logging.getLogger("setUp")
logger.info("setup config")
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendbasetest.conf'])
# ------------ #
# config: main #
# ------------ #
self.config.set('main', 'incomingport',
str(ReloadUnderLoadTest.FUGLU_PORT))
self.config.set('main', 'outgoinghost',
str(ReloadUnderLoadTest.FUGLU_HOST))
self.config.set('main', 'outgoingport',
str(ReloadUnderLoadTest.DUMMY_PORT))
self.config.set('main', 'controlport',
str(ReloadUnderLoadTest.FUGLUCONTROL_PORT))
# ------------------- #
# config: performance #
# ------------------- #
# minimum scanner threads
self.config.set('performance', 'minthreads', 1)
# maximum scanner threads
self.config.set('performance', 'maxthreads', 1)
# Method for parallelism, either 'thread' or 'process'
self.config.set('performance', 'backend', 'process')
# Initial number of processes when backend='process'.
# If 0 (the default), automatically selects twice the number of available virtual cores.
# Despite its 'initial'-name, this number currently is not adapted automatically.
self.config.set('performance', 'initialprocs',
ReloadUnderLoadTest.num_procs)
# set timeout for joining the workers. Since the DummySMTPServer receives sequentially,
# we need at least number_of_procs*delayPlugin
self.config.set(
'performance', 'join_timeout',
10.0 * float(ReloadUnderLoadTest.num_procs) *
ReloadUnderLoadTest.delay_by)
self.config.set('main', 'plugins', 'fuglu.plugins.delay.DelayPlugin')
# -------------------- #
# config: delay plugin #
# -------------------- #
self.config.add_section("DelayPlugin")
# the delay created by this plugn
self.config.set("DelayPlugin", 'delay', ReloadUnderLoadTest.delay_by)
self.config.set("DelayPlugin", 'logfrequency',
ReloadUnderLoadTest.delay_by)
# -------------- #
# MainController #
# -------------- #
# init core
logger.info("setup MainController")
self.mc = MainController(self.config)
# ----------------- #
# Dummy SMTP Server #
# ----------------- #
logger.info("setup Dummy SMTP Server and start thread")
# start listening smtp dummy server to get fuglus answer
self.dsmtp = DummySMTPServer(self.config,
ReloadUnderLoadTest.DUMMY_PORT,
ReloadUnderLoadTest.FUGLU_HOST,
stayalive=True)
self.thread_dsmtp = threading.Thread(name="DummySMTPServer",
target=self.dsmtp.serve,
args=())
self.thread_dsmtp.daemon = True
self.thread_dsmtp.start()
# --
# start fuglu's listening server (MainController.startup)
# --
logger.info("setup Fuglu and start thread")
self.thread_fls = threading.Thread(name="MainController",
target=self.mc.startup,
args=())
self.thread_fls.daemon = True
self.thread_fls.start()
# give fuglu time to start listener
time.sleep(1)
setup_module()
def tearDown(self):
logger = logging.getLogger("tearDown")
# ---
# shutdown fuglu (BEFORE Dummy SMTPServer)
# ---
logger.debug(
"\n------------------\n Shutdown Fuglu MainController\n -------------------"
)
self.mc.shutdown()
logger.debug("Join Fuglu MainController Thread (fls)")
self.thread_fls.join()
logger.debug("fls joined")
# ---
# shutdown dummy smtp server
# ---
logger.debug(
"\n------------------\n Shutdown Dummy SMTP\n -------------------")
logger.debug("set DummySMTPServer.stayalive = False")
self.dsmtp.stayalive = False
# just connect and close to shutdown also the dummy SMTP server
logger.debug("Make dummy connection")
dsmtpclient = smtplib.SMTP(ReloadUnderLoadTest.FUGLU_HOST,
ReloadUnderLoadTest.DUMMY_PORT)
logger.debug("Close")
dsmtpclient.close()
logger.debug("End")
logger.debug("Join Dummy SMTP Thread (dsmtp)")
self.thread_dsmtp.join()
logger.debug("dsmtp joined")
logger.debug("Shutdown Dummy SMTP Server")
self.dsmtp.shutdown()
def create_and_send_message(self):
"""Helper routine to send messages in a separate thread"""
threadname = threading.current_thread().name
logger = logging.getLogger("create_and_send_message.%s" % threadname)
logger.debug(
"create client, connect to: %s:%u" %
(ReloadUnderLoadTest.FUGLU_HOST, ReloadUnderLoadTest.FUGLU_PORT))
try:
smtpclient = smtplib.SMTP(ReloadUnderLoadTest.FUGLU_HOST,
ReloadUnderLoadTest.FUGLU_PORT)
except smtplib.SMTPServerDisconnected as e:
logger.error("SMTP Connection Error")
print("%s: %s" % (threadname, e))
import traceback
traceback.print_exc()
return
# build identifier
sender = threadname.replace("(", "").replace(")", "").replace(
",", ".").replace("-", ".") + "@fuglu.org"
logger.debug("say helo...")
smtpclient.helo('test.e2e')
testmessage = """Hello World!"""
msg = MIMEText(testmessage)
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
logger.debug("send mail")
try:
smtpclient.sendmail(sender, '*****@*****.**', msgstring)
logger.info("mail sent... - check reply...")
code, response = smtpclient.quit()
logger.info("%u: %s" % (code, response))
logger.info("mail sent successfully")
except smtplib.SMTPServerDisconnected as e:
logger.error("sending mail")
print("%s: %s" % (threadname, e))
import traceback
traceback.print_exc()
@timed(60)
def test_reloadwhilebusy(self):
"""Test reloading processpool while under load. This test should just NOT hang!"""
logger = logging.getLogger("test_reloadwhilebusy")
# number of reloads
num_reloads = 1
# number of messages to send
num_messages_before = 2 * ReloadUnderLoadTest.num_procs # before reload
num_messages_after = ReloadUnderLoadTest.num_procs # after reload
for ireload in range(num_reloads):
logger.info(
"\n==========================\nRun %u\n==========================\n"
% ireload)
# backup original procpool
orig_procpool = self.mc.procpool
orig_workers = orig_procpool.workers
for worker in orig_workers:
self.assertTrue(worker.is_alive())
# send test message
messages = []
logger.info(
"\n--------------------------\nDump %u messages into queue\n--------------------------\n"
% num_messages_before)
for imessage in range(num_messages_before):
t = threading.Thread(name="(%u,%u)-before-MessageSender" %
(ireload, imessage),
target=self.create_and_send_message,
args=())
t.daemon = True
t.start()
messages.append(t)
time.sleep(
min(float(ReloadUnderLoadTest.num_procs),
num_messages_before / 2.) * ReloadUnderLoadTest.delay_by)
logger.info(
"\n--------------------------\nRELOAD - RELOAD - RELOAD\n--------------------------\n"
)
self.mc.reload()
# at this point all original workers should be closed
for worker in orig_workers:
self.assertFalse(worker.is_alive(), "%s" % worker)
logger.info(
"\n--------------------------\nDump 2 messages into queue\n--------------------------\n"
)
for imessage in range(num_messages_after):
t = threading.Thread(name="(%u,%u)-after-MessageSender" %
(ireload, imessage),
target=self.create_and_send_message,
args=())
t.daemon = True
t.start()
messages.append(t)
for t in messages:
t.join()
class EndtoEndBaseTestCase(unittest.TestCase):
"""Full check if mail runs through but no plugins applied"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7741
DUMMY_PORT = 7742
FUGLUCONTROL_PORT = 7743
def setUp(self):
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendbasetest.conf'])
self.config.set(
'main', 'incomingport', str(EndtoEndBaseTestCase.FUGLU_PORT))
self.config.set(
'main', 'outgoinghost', str(EndtoEndBaseTestCase.FUGLU_HOST))
self.config.set(
'main', 'outgoingport', str(EndtoEndBaseTestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(EndtoEndBaseTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(
self.config, EndtoEndBaseTestCase.DUMMY_PORT, EndtoEndBaseTestCase.FUGLU_HOST)
self.e2edss = threading.Thread(target = self.smtp.serve, args = ())
self.e2edss.daemon = True
self.e2edss.start()
# start fuglu's listening server
self.fls = threading.Thread(target = self.mc.startup, args = ())
self.fls.daemon = True
self.fls.start()
def tearDown(self):
# Check if Dummy SMTP Server is alive and still waiting for a connection
if self.smtp.is_waiting:
# just connect and close to shutdown also the dummy SMTP server
self.smtp.stayalive = False
dsmtpclient = smtplib.SMTP(EndtoEndBaseTestCase.FUGLU_HOST, EndtoEndBaseTestCase.DUMMY_PORT)
dsmtpclient.close()
self.mc.shutdown()
self.smtp.shutdown()
self.e2edss.join()
self.fls.join()
def test_SMTPUTF8_E2E(self):
"""test if a UTF-8 message runs through"""
# give fuglu time to start listener
time.sleep(1)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', EndtoEndBaseTestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
(code, msg) = smtpclient.ehlo('test.e2e')
msg = force_uString(msg.split())
self.assertEqual(250, code)
print("%s"%msg)
self.assertIn("SMTPUTF8", msg)
testunicodemessage = u"""Hello Wörld!\r
Don't där yü tschänsch äny of mai baits or iwen remüv ön!"""
# TODO: this test fails if we don't put in the \r in there... (eg,
# fuglu adds it) - is this a bug or wrong test?
msg = MIMEText(testunicodemessage, _charset='utf-8')
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
inbytes = len(msg.get_payload(decode=True))
# envelope sender/recipients
env_sender = u'sä[email protected]'
env_recipients = [u'rö[email protected]', u'récipiè[email protected]']
smtpclient.sendmail(force_uString(env_sender),
force_uString(env_recipients),
force_bString(msgstring), mail_options=["SMTPUTF8"])
smtpclient.quit()
# get answer (wait to give time to create suspect)
time.sleep(0.1)
gotback = self.smtp.suspect
self.assertFalse(gotback == None, "Did not get message from dummy smtp server")
# check a few things on the received message
msgrep = gotback.get_message_rep()
self.assertTrue('X-Fuglutest-Spamstatus' in msgrep,
"Fuglu SPAM Header not found in message")
payload = msgrep.get_payload(decode=True)
outbytes = len(payload)
self.assertEqual(inbytes, outbytes,"Message size change: bytes in: %u, bytes out %u" % (inbytes, outbytes))
self.assertEqual(testunicodemessage, force_uString(payload),
"Message body has been altered. In: %u bytes, Out: %u bytes, teststring=->%s<- result=->%s<-" %
(inbytes, outbytes, testunicodemessage, force_uString(payload)))
# check sender/recipients
self.assertEqual(env_sender, gotback.from_address)
self.assertEqual(env_recipients, gotback.recipients)
class RequeueMsgTestCase(unittest.TestCase):
"""Full check if mail runs through"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7771
DUMMY_PORT = 7772
FUGLUCONTROL_PORT = 7773
def setUp(self):
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendtest.conf'])
self.config.set(
'main', 'incomingport', str(RequeueMsgTestCase.FUGLU_PORT))
self.config.set(
'main', 'outgoinghost', str(RequeueMsgTestCase.FUGLU_HOST))
self.config.set(
'main', 'outgoingport', str(RequeueMsgTestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(RequeueMsgTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(
self.config, RequeueMsgTestCase.DUMMY_PORT, RequeueMsgTestCase.FUGLU_HOST)
self.e2edss = threading.Thread(target = self.smtp.serve, args = ())
self.e2edss.daemon = True
self.e2edss.start()
# start fuglu's listening server
self.fls = threading.Thread(target = self.mc.startup, args = ())
self.fls.daemon = True
self.fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
self.e2edss.join(timeout=3)
self.fls.join(timeout=3)
def test_requeue_message(self):
"""test end to end and check requeue message"""
# give fuglu time to start listener
time.sleep(1)
# send test message
smtpclient = FUSMTPClient('127.0.0.1', RequeueMsgTestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
smtpclient.helo('test.e2e')
testmessage = """Hello World!\r
Don't dare you change any of my bytes or even remove one!"""
msg = MIMEText(testmessage)
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
inbytes = len(msg.get_payload())
self.smtp.response_message = "2.0.0 Ok: queued as 42tk6y3qjCz5r2V"
smtpclient.sendmail(
'*****@*****.**', '*****@*****.**', msgstring)
lastserveranswer = smtpclient.lastserveranswer
smtpclient.quit()
# get answer (wait to give time to create suspect)
expected_answer = ("FUGLU REQUEUE(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx): %s" % (self.smtp.response_message))
# replace fuglu id for comparison
lastserveranswer = re.sub(r"\(.*\)",
"(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx)",
force_uString(lastserveranswer))
print("Expected: %s" % expected_answer)
print("Server : %s" % lastserveranswer)
self.assertEqual(expected_answer, lastserveranswer)
class ReinjectTmpErrorTestCase(unittest.TestCase):
"""Full check if mail runs through but no plugins applied"""
FUGLU_HOST = "127.0.0.1"
FUGLU_PORT = 7761
DUMMY_PORT = 7762
FUGLUCONTROL_PORT = 7763
def setUp(self):
self.config = RawConfigParser()
self.config.read([TESTDATADIR + '/endtoendbasetest.conf'])
self.config.set(
'main', 'incomingport', str(ReinjectTmpErrorTestCase.FUGLU_PORT))
self.config.set(
'main', 'outgoinghost', str(ReinjectTmpErrorTestCase.FUGLU_HOST))
self.config.set(
'main', 'outgoingport', str(ReinjectTmpErrorTestCase.DUMMY_PORT))
self.config.set(
'main', 'controlport', str(ReinjectTmpErrorTestCase.FUGLUCONTROL_PORT))
guess_clamav_socket(self.config)
# init core
self.mc = MainController(self.config)
# start listening smtp dummy server to get fuglus answer
self.smtp = DummySMTPServer(
self.config, ReinjectTmpErrorTestCase.DUMMY_PORT, ReinjectTmpErrorTestCase.FUGLU_HOST)
self.e2edss = threading.Thread(target=self.smtp.serve, args=())
self.e2edss.daemon = True
self.e2edss.start()
# start fuglu's listening server
self.fls = threading.Thread(target=self.mc.startup, args=())
self.fls.daemon = True
self.fls.start()
def tearDown(self):
self.mc.shutdown()
self.smtp.shutdown()
self.e2edss.join()
self.fls.join()
def test_reinject_tmp_error(self):
"""test if a reinject tmp error is passed"""
# give fuglu time to start listener
time.sleep(1)
import logging
import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
# send test message
smtpclient = smtplib.SMTP('127.0.0.1', ReinjectTmpErrorTestCase.FUGLU_PORT)
# smtpServer.set_debuglevel(1)
(code, msg) = smtpclient.helo('test.e2e')
self.assertEqual(250, code)
testmessage = u"""Hello World!"""
# TODO: this test fails if we don't put in the \r in there... (eg,
# fuglu adds it) - is this a bug or wrong test?
msg = MIMEText(testmessage)
msg["Subject"] = "End to End Test"
msgstring = msg.as_string()
inbytes = len(msg.get_payload(decode=True))
# envelope sender/recipients
env_sender = u'*****@*****.**'
env_recipients = [u'*****@*****.**']
self.smtp.response_code = 451
self.smtp.response_message = '4.5.1 Internal error'
try:
smtpclient.sendmail(force_uString(env_sender),
force_uString(env_recipients),
force_bString(msgstring))
except smtplib.SMTPDataError as e:
self.assertEqual(self.smtp.response_code, e.smtp_code)