def test_virus(self):
"""Test if eicar is detected as virus"""
from fuglu.shared import Suspect
import shutil
self.mc.load_plugins()
if len(self.mc.plugins) == 0:
raise Exception("plugins not loaded")
sesshandler = SessionHandler(None, self.mc.config, self.mc.prependers,
self.mc.plugins, self.mc.appenders)
tempfilename = tempfile.mktemp(suffix='virus',
prefix='fuglu-unittest',
dir='/tmp')
shutil.copy(TESTDATADIR + '/eicar.eml', tempfilename)
self.tempfiles.append(tempfilename)
suspect = Suspect('*****@*****.**',
'*****@*****.**', tempfilename)
pluglist = sesshandler.run_prependers(suspect)
self.assertFalse(
len(pluglist) == 0,
"Viruscheck will fail, pluginlist empty after run_prependers")
sesshandler.run_plugins(suspect, pluglist)
self.assertTrue(suspect.is_virus(),
"Eicar message was not detected as virus")
def serve(self):
#disable to debug...
use_multithreading=True
controller=self.controller
threading.currentThread().name='ESMTP Server on Port %s'%self.port
self.logger.info('ESMTP Server running on port %s'%self.port)
if use_multithreading:
threadpool=self.controller.threadpool
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
if not self.stayalive:
break
ph=ESMTPHandler(nsd[0], controller.config)
engine = SessionHandler(ph,controller.config,controller.prependers,controller.plugins,controller.appenders)
self.logger.debug('Incoming connection from %s'%str(nsd[1]))
if use_multithreading:
#this will block if queue is full
threadpool.add_task(engine)
else:
engine.handlesession()
except Exception,e:
self.logger.error('Exception in serve(): %s'%str(e))
def serve(self):
# disable to debug...
use_multithreading = True
controller = self.controller
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
if use_multithreading:
threadpool = self.controller.threadpool
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
if not self.stayalive:
break
ph = self.protohandlerclass(nsd[0], controller.config)
engine = SessionHandler(ph, controller.config,
controller.prependers,
controller.plugins,
controller.appenders)
self.logger.debug('Incoming connection from %s' % str(nsd[1]))
if use_multithreading:
# this will block if queue is full
threadpool.add_task(engine)
else:
engine.handlesession()
except Exception, e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state,child_to_server_messages, logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state,'loading configuration')
logger = logging.getLogger('fuglu.process')
logger.debug("New worker: %s" % logtools.createPIDinfo())
# load config and plugins
controller = fuglu.core.MainController(config,logQueue)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" % logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug("%s: Child process received poison pill - shut down" % logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended'
except Exception as e:
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" % logtools.createPIDinfo())
sock, handler_modulename, handler_classname = fuglu_process_unpack(task)
handler_class = getattr(importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config,prependers, plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s"%trb)
print(trb)
workerstate.workerstate = 'crashed'
finally:
controller.shutdown()
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders)
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task(engine)
elif self.controller.procpool:
# in multi processing, the other process manages configs and plugins itself, we only pass the minimum required information:
# a pickled version of the socket (this is no longer required in python 3.4, but in python 2 the multiprocessing queue can not handle sockets
# see https://stackoverflow.com/questions/36370724/python-passing-a-tcp-socket-object-to-a-multiprocessing-queue
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
task = forking_dumps(
sock), handler_modulename, handler_classname
self.controller.procpool.add_task(task)
else:
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
elif self.controller.procpool:
self.controller.procpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
else:
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders,
self.port)
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state,
child_to_server_messages):
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state, 'loading configuration')
logger = logging.getLogger('fuglu.process')
# load config and plugins
controller = fuglu.core.MainController(config)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(
lambda event: child_to_server_messages.put(event.as_message()))
try:
while True:
workerstate.workerstate = 'waiting for task'
task = queue.get()
if task is None: # poison pill
logger.debug("Child process received poison pill - shut down")
workerstate.workerstate = 'ended'
return
workerstate.workerstate = 'starting scan session'
pickled_socket, handler_modulename, handler_classname = task
sock = pickle.loads(pickled_socket)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config, prependers,
plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s" % trb)
print(trb)
workerstate.workerstate = 'crashed'
def serve(self):
controller = self.controller
threadpool = self.controller.threadpool
procpool = self.controller.procpool
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
ph = self.protohandlerclass(sock, controller.config)
engine = SessionHandler(ph, controller.config,
controller.prependers,
controller.plugins,
controller.appenders)
self.logger.debug('Incoming connection from %s' % str(addr))
if threadpool:
# this will block if queue is full
threadpool.add_task(engine)
elif procpool:
# in multi processing, the other process manages configs and plugins itself, we only pass the minimum required information:
# a pickled version of the socket (this is no longer required in python 3.4, but in python 2 the multiprocessing queue can not handle sockets
# see https://stackoverflow.com/questions/36370724/python-passing-a-tcp-socket-object-to-a-multiprocessing-queue
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
task = forking_dumps(
sock), handler_modulename, handler_classname
procpool.add_task(task)
else:
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def test_prepender(self):
"""Test prepender plugin raising exception"""
config = RawConfigParser()
# -------------#
# config: main #
# -------------#
config.add_section("main")
config.set('main', 'plugins', '')
config.set('main', 'prependers',
'sessionhandler_test.RaiseExceptionPrepender')
config.set('main', 'appenders', '')
# ------------------- #
# config: performance #
# ------------------- #
config.add_section("performance")
# minimum scanner threads
config.set('performance', 'minthreads', 1)
# maximum scanner threads
config.set('performance', 'maxthreads', 1)
# Method for parallelism, either 'thread' or 'process'
config.set('performance', 'backend', 'process')
mc = MainController(config)
mc.propagate_core_defaults()
ok = mc.load_plugins()
suspect = Suspect('*****@*****.**',
'*****@*****.**', '/dev/null')
shandler = SessionHandler(None, config, mc.prependers, mc.plugins,
mc.appenders, 0)
pluglist, applist = shandler.run_prependers(suspect)
shandler.run_plugins(suspect, pluglist)
ptags = suspect.get_tag("processingerrors")
self.assertEqual([
'Prepender RaiseExceptionPrepender failed: Prepender Plugin not implemented'
], ptags)
def fuglu_process_worker(queue, config, shared_state,child_to_server_messages):
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state,'loading configuration')
logger = logging.getLogger('fuglu.process')
# load config and plugins
controller = fuglu.core.MainController(config)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(lambda event: child_to_server_messages.put(event.as_message()))
try:
while True:
workerstate.workerstate = 'waiting for task'
task = queue.get()
if task is None: # poison pill
logger.debug("Child process received poison pill - shut down")
workerstate.workerstate = 'ended'
return
workerstate.workerstate = 'starting scan session'
pickled_socket, handler_modulename, handler_classname = task
sock = pickle.loads(pickled_socket)
handler_class = getattr(importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config,prependers, plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s"%trb)
print(trb)
workerstate.workerstate = 'crashed'
def test_virus(self):
"""Test if eicar is detected as virus"""
from fuglu.shared import Suspect
import shutil
self.mc.load_plugins()
if len(self.mc.plugins) == 0:
raise Exception("plugins not loaded")
sesshandler = SessionHandler(
None, self.mc.config, self.mc.prependers, self.mc.plugins, self.mc.appenders)
tempfilename = tempfile.mktemp(
suffix='virus', prefix='fuglu-unittest', dir='/tmp')
shutil.copy(TESTDATADIR + '/eicar.eml', tempfilename)
self.tempfiles.append(tempfilename)
suspect = Suspect(
'*****@*****.**', '*****@*****.**', tempfilename)
pluglist = sesshandler.run_prependers(suspect)
self.assertFalse(
len(pluglist) == 0, "Viruscheck will fail, pluginlist empty after run_prependers")
sesshandler.run_plugins(suspect, pluglist)
self.assertTrue(
suspect.is_virus(), "Eicar message was not detected as virus")
def get_task_sessionhandler(self, timeout=None):
"""
Get task from queue, create a session handler and return it
Returns:
SessionHandler: a session handler to handle the task
"""
if self._stayalive:
try:
task = self.tasks.get(True, timeout)
except queue.Empty:
task = None
if task is None:
# Poison pill or empty queue
return None
sock, handler_modulename, handler_classname, port = uncompress_task(
task)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
controller = self.controller()
if controller is None:
# weak ref will point to None once object has been removed
return None
handler_instance = handler_class(sock, controller.config)
handler = SessionHandler(handler_instance, controller.config,
controller.prependers, controller.plugins,
controller.appenders, port)
return handler
else:
return None
def fuglu_process_worker(queue, config, shared_state, child_to_server_messages,
logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state, 'loading configuration')
logger = logging.getLogger(
'fuglu.process.%s(%u)' %
(workerstate.process.name, workerstate.process.pid))
logger.debug("New worker: %s" % logtools.createPIDinfo())
# Setup address compliance checker
# -> Due to default linux forking behavior this should already
# have the correct setup but it's better not to rely on this
try:
address_check = config.get('main', 'address_compliance_checker')
except Exception as e:
# might happen for some tests which do not propagate defaults
address_check = "Default"
Addrcheck().set(address_check)
# load config and plugins
logger.debug("Create MainController")
controller = fuglu.core.MainController(config,
logQueue=logQueue,
nolog=True)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(
lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" %
logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug(
"%s: Child process received poison pill - shut down" %
logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended (poison pill)'
except Exception as e:
logger.debug(
"Exception setting workstate while getting poison pill"
)
logger.exception(e)
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" %
logtools.createPIDinfo())
sock, handler_modulename, handler_classname, port = uncompress_task(
task)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config, prependers,
plugins, appenders, port)
handler.handlesession(workerstate)
del handler
del handler_instance
del handler_class
del handler_modulename
del handler_classname
del sock
# developers only:
# for debugging memory this can be enabled
# Note this can NOT be copied to threadpool worker because
# it will create a memory leak
if OBJGRAPH_EXTENSION_ENABLED and False:
debug_procpoolworkermemory(logger, config)
except KeyboardInterrupt:
workerstate.workerstate = 'ended (keyboard interrupt)'
logger.debug("Keyboard interrupt")
except Exception as e:
logger.error("Exception in worker process: %s" % str(e))
workerstate.workerstate = 'crashed'
finally:
# this process will not put any object in queue
queue.close()
controller.shutdown()
