def serve(self):
#disable to debug...
use_multithreading=True
controller=self.controller
threading.currentThread().name='ESMTP Server on Port %s'%self.port
self.logger.info('ESMTP Server running on port %s'%self.port)
if use_multithreading:
threadpool=self.controller.threadpool
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
if not self.stayalive:
break
ph=ESMTPHandler(nsd[0], controller.config)
engine = SessionHandler(ph,controller.config,controller.prependers,controller.plugins,controller.appenders)
self.logger.debug('Incoming connection from %s'%str(nsd[1]))
if use_multithreading:
#this will block if queue is full
threadpool.add_task(engine)
else:
engine.handlesession()
except Exception,e:
self.logger.error('Exception in serve(): %s'%str(e))
def serve(self):
# disable to debug...
use_multithreading = True
controller = self.controller
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
if use_multithreading:
threadpool = self.controller.threadpool
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
if not self.stayalive:
break
ph = self.protohandlerclass(nsd[0], controller.config)
engine = SessionHandler(ph, controller.config,
controller.prependers,
controller.plugins,
controller.appenders)
self.logger.debug('Incoming connection from %s' % str(nsd[1]))
if use_multithreading:
# this will block if queue is full
threadpool.add_task(engine)
else:
engine.handlesession()
except Exception, e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state,child_to_server_messages, logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state,'loading configuration')
logger = logging.getLogger('fuglu.process')
logger.debug("New worker: %s" % logtools.createPIDinfo())
# load config and plugins
controller = fuglu.core.MainController(config,logQueue)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" % logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug("%s: Child process received poison pill - shut down" % logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended'
except Exception as e:
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" % logtools.createPIDinfo())
sock, handler_modulename, handler_classname = fuglu_process_unpack(task)
handler_class = getattr(importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config,prependers, plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s"%trb)
print(trb)
workerstate.workerstate = 'crashed'
finally:
controller.shutdown()
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders)
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task(engine)
elif self.controller.procpool:
# in multi processing, the other process manages configs and plugins itself, we only pass the minimum required information:
# a pickled version of the socket (this is no longer required in python 3.4, but in python 2 the multiprocessing queue can not handle sockets
# see https://stackoverflow.com/questions/36370724/python-passing-a-tcp-socket-object-to-a-multiprocessing-queue
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
task = forking_dumps(
sock), handler_modulename, handler_classname
self.controller.procpool.add_task(task)
else:
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
elif self.controller.procpool:
self.controller.procpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
else:
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders,
self.port)
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state,
child_to_server_messages):
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state, 'loading configuration')
logger = logging.getLogger('fuglu.process')
# load config and plugins
controller = fuglu.core.MainController(config)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(
lambda event: child_to_server_messages.put(event.as_message()))
try:
while True:
workerstate.workerstate = 'waiting for task'
task = queue.get()
if task is None: # poison pill
logger.debug("Child process received poison pill - shut down")
workerstate.workerstate = 'ended'
return
workerstate.workerstate = 'starting scan session'
pickled_socket, handler_modulename, handler_classname = task
sock = pickle.loads(pickled_socket)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config, prependers,
plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s" % trb)
print(trb)
workerstate.workerstate = 'crashed'
def serve(self):
controller = self.controller
threadpool = self.controller.threadpool
procpool = self.controller.procpool
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
ph = self.protohandlerclass(sock, controller.config)
engine = SessionHandler(ph, controller.config,
controller.prependers,
controller.plugins,
controller.appenders)
self.logger.debug('Incoming connection from %s' % str(addr))
if threadpool:
# this will block if queue is full
threadpool.add_task(engine)
elif procpool:
# in multi processing, the other process manages configs and plugins itself, we only pass the minimum required information:
# a pickled version of the socket (this is no longer required in python 3.4, but in python 2 the multiprocessing queue can not handle sockets
# see https://stackoverflow.com/questions/36370724/python-passing-a-tcp-socket-object-to-a-multiprocessing-queue
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
task = forking_dumps(
sock), handler_modulename, handler_classname
procpool.add_task(task)
else:
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state,child_to_server_messages):
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state,'loading configuration')
logger = logging.getLogger('fuglu.process')
# load config and plugins
controller = fuglu.core.MainController(config)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(lambda event: child_to_server_messages.put(event.as_message()))
try:
while True:
workerstate.workerstate = 'waiting for task'
task = queue.get()
if task is None: # poison pill
logger.debug("Child process received poison pill - shut down")
workerstate.workerstate = 'ended'
return
workerstate.workerstate = 'starting scan session'
pickled_socket, handler_modulename, handler_classname = task
sock = pickle.loads(pickled_socket)
handler_class = getattr(importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config,prependers, plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s"%trb)
print(trb)
workerstate.workerstate = 'crashed'
def fuglu_process_worker(queue, config, shared_state, child_to_server_messages,
logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state, 'loading configuration')
logger = logging.getLogger(
'fuglu.process.%s(%u)' %
(workerstate.process.name, workerstate.process.pid))
logger.debug("New worker: %s" % logtools.createPIDinfo())
# Setup address compliance checker
# -> Due to default linux forking behavior this should already
# have the correct setup but it's better not to rely on this
try:
address_check = config.get('main', 'address_compliance_checker')
except Exception as e:
# might happen for some tests which do not propagate defaults
address_check = "Default"
Addrcheck().set(address_check)
# load config and plugins
logger.debug("Create MainController")
controller = fuglu.core.MainController(config,
logQueue=logQueue,
nolog=True)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(
lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" %
logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug(
"%s: Child process received poison pill - shut down" %
logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended (poison pill)'
except Exception as e:
logger.debug(
"Exception setting workstate while getting poison pill"
)
logger.exception(e)
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" %
logtools.createPIDinfo())
sock, handler_modulename, handler_classname, port = uncompress_task(
task)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config, prependers,
plugins, appenders, port)
handler.handlesession(workerstate)
del handler
del handler_instance
del handler_class
del handler_modulename
del handler_classname
del sock
# developers only:
# for debugging memory this can be enabled
# Note this can NOT be copied to threadpool worker because
# it will create a memory leak
if OBJGRAPH_EXTENSION_ENABLED and False:
debug_procpoolworkermemory(logger, config)
except KeyboardInterrupt:
workerstate.workerstate = 'ended (keyboard interrupt)'
logger.debug("Keyboard interrupt")
except Exception as e:
logger.error("Exception in worker process: %s" % str(e))
workerstate.workerstate = 'crashed'
finally:
# this process will not put any object in queue
queue.close()
controller.shutdown()
