def __init__(self,
controller,
port=10125,
address="127.0.0.1",
protohandlerclass=None):
if protohandlerclass is None:
protohandlerclass = ProtocolHandler
self.protohandlerclass = protohandlerclass
self.logger = logging.getLogger("fuglu.incoming.%s" % port)
self.logger.debug('Starting incoming Server on Port %s, protocol=%s' %
(port, self.protohandlerclass.protoname))
self.logger.debug('Incoming server process info: %s' %
createPIDinfo())
self.logger.debug('(%s) Logger id is %s' % (createPIDinfo(), id(self)))
self.port = port
self.controller = controller
self.stayalive = True
addr_f = socket.getaddrinfo(address, 0)[0][0]
try:
self._socket = socket.socket(addr_f, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind((address, port))
self._socket.listen(5)
except Exception as e:
self.logger.error(
'Could not start incoming Server on port %s: %s' % (port, e))
self.stayalive = False
def fuglu_process_worker(queue, config, shared_state,child_to_server_messages, logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state,'loading configuration')
logger = logging.getLogger('fuglu.process')
logger.debug("New worker: %s" % logtools.createPIDinfo())
# load config and plugins
controller = fuglu.core.MainController(config,logQueue)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" % logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug("%s: Child process received poison pill - shut down" % logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended'
except Exception as e:
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" % logtools.createPIDinfo())
sock, handler_modulename, handler_classname = fuglu_process_unpack(task)
handler_class = getattr(importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config,prependers, plugins, appenders)
handler.handlesession(workerstate)
except KeyboardInterrupt:
workerstate.workerstate = 'ended'
except:
trb = traceback.format_exc()
logger.error("Exception in child process: %s"%trb)
print(trb)
workerstate.workerstate = 'crashed'
finally:
controller.shutdown()
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders)
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task(engine)
elif self.controller.procpool:
# in multi processing, the other process manages configs and plugins itself, we only pass the minimum required information:
# a pickled version of the socket (this is no longer required in python 3.4, but in python 2 the multiprocessing queue can not handle sockets
# see https://stackoverflow.com/questions/36370724/python-passing-a-tcp-socket-object-to-a-multiprocessing-queue
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
task = forking_dumps(
sock), handler_modulename, handler_classname
self.controller.procpool.add_task(task)
else:
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def serve(self):
# Important:
# -> do NOT create local variables which are copies of member variables like
# controller = self.controller
# threadpool = self.controller.threadpool
# procpool = self.controller.procpool
# Since thes variables might change while in the stayalive loop the process would get stuck,
# example: when sending SIGHUP which might recreate the processor pool or threads pool
# which would then still point to the wrong (old) memory location and is therefore not served anymore
threading.currentThread().name = '%s Server on Port %s' % (
self.protohandlerclass.protoname, self.port)
self.logger.info('%s Server running on port %s' %
(self.protohandlerclass.protoname, self.port))
while self.stayalive:
try:
self.logger.debug('Waiting for connection...')
nsd = self._socket.accept()
sock, addr = nsd
if not self.stayalive:
break
handler_classname = self.protohandlerclass.__name__
handler_modulename = self.protohandlerclass.__module__
self.logger.debug(
'(%s) Incoming connection [incoming server port: %s, prot: %s]'
% (createPIDinfo(), self.port,
self.protohandlerclass.protoname))
if self.controller.threadpool:
# this will block if queue is full
self.controller.threadpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
elif self.controller.procpool:
self.controller.procpool.add_task_from_socket(
sock, handler_modulename, handler_classname, self.port)
else:
ph = self.protohandlerclass(sock, self.controller.config)
engine = SessionHandler(ph, self.controller.config,
self.controller.prependers,
self.controller.plugins,
self.controller.appenders,
self.port)
engine.handlesession()
except Exception as e:
exc = traceback.format_exc()
self.logger.error('Exception in serve(): %s - %s' %
(str(e), exc))
def fuglu_process_worker(queue, config, shared_state, child_to_server_messages,
logQueue):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
logtools.client_configurer(logQueue)
logging.basicConfig(level=logging.DEBUG)
workerstate = WorkerStateWrapper(shared_state, 'loading configuration')
logger = logging.getLogger(
'fuglu.process.%s(%u)' %
(workerstate.process.name, workerstate.process.pid))
logger.debug("New worker: %s" % logtools.createPIDinfo())
# Setup address compliance checker
# -> Due to default linux forking behavior this should already
# have the correct setup but it's better not to rely on this
try:
address_check = config.get('main', 'address_compliance_checker')
except Exception as e:
# might happen for some tests which do not propagate defaults
address_check = "Default"
Addrcheck().set(address_check)
# load config and plugins
logger.debug("Create MainController")
controller = fuglu.core.MainController(config,
logQueue=logQueue,
nolog=True)
controller.load_extensions()
controller.load_plugins()
prependers = controller.prependers
plugins = controller.plugins
appenders = controller.appenders
# forward statistics counters to parent process
stats = Statskeeper()
stats.stat_listener_callback.append(
lambda event: child_to_server_messages.put(event.as_message()))
logger.debug("%s: Enter service loop..." % logtools.createPIDinfo())
try:
while True:
workerstate.workerstate = 'waiting for task'
logger.debug("%s: Child process waiting for task" %
logtools.createPIDinfo())
task = queue.get()
if task is None: # poison pill
logger.debug(
"%s: Child process received poison pill - shut down" %
logtools.createPIDinfo())
try:
# it might be possible it does not work to properly set the workerstate
# since this is a shared variable -> prevent exceptions
workerstate.workerstate = 'ended (poison pill)'
except Exception as e:
logger.debug(
"Exception setting workstate while getting poison pill"
)
logger.exception(e)
pass
finally:
return
workerstate.workerstate = 'starting scan session'
logger.debug("%s: Child process starting scan session" %
logtools.createPIDinfo())
sock, handler_modulename, handler_classname, port = uncompress_task(
task)
handler_class = getattr(
importlib.import_module(handler_modulename), handler_classname)
handler_instance = handler_class(sock, config)
handler = SessionHandler(handler_instance, config, prependers,
plugins, appenders, port)
handler.handlesession(workerstate)
del handler
del handler_instance
del handler_class
del handler_modulename
del handler_classname
del sock
# developers only:
# for debugging memory this can be enabled
# Note this can NOT be copied to threadpool worker because
# it will create a memory leak
if OBJGRAPH_EXTENSION_ENABLED and False:
debug_procpoolworkermemory(logger, config)
except KeyboardInterrupt:
workerstate.workerstate = 'ended (keyboard interrupt)'
logger.debug("Keyboard interrupt")
except Exception as e:
logger.error("Exception in worker process: %s" % str(e))
workerstate.workerstate = 'crashed'
finally:
# this process will not put any object in queue
queue.close()
controller.shutdown()