class OutputManager(Process):
"""
This class manages output. It has a list of output plugins and sends the
messages to every plugin on that list.
Logs are sent to the "out" instance (see module level variable) and using
a queue we forward the messages to this object which will be calling the
output plugins.
:author: Andres Riancho ([email protected])
"""
FLUSH_TIMEOUT = 60
# Thread locking to avoid starting the om many times from different threads
start_lock = threading.RLock()
def __init__(self):
super(OutputManager, self).__init__(name='OutputManager')
self.daemon = True
self.name = 'OutputManager'
# User configured options
self._output_plugin_instances = []
self._output_plugin_names = []
self._plugin_options = {}
# Internal variables
self.in_queue = SilentJoinableQueue()
self._w3af_core = None
self._last_output_flush = None
def set_w3af_core(self, w3af_core):
self._w3af_core = w3af_core
def get_in_queue(self):
"""
The output sinks need this queue, they will send the messages directly
to it from different threads/processes. Then the main process will
get the log messages and process them in the run() method below.
:return: A multiprocessing Queue
"""
return self.in_queue
def start(self):
with self.start_lock:
if not self.is_alive():
super(OutputManager, self).start()
def run(self):
"""
This method is one of the most important ones in the class, since it
will consume the work units from the queue and send them to the plugins
"""
while True:
try:
work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
except Queue.Empty:
self.flush_plugin_output()
continue
except EOFError:
# The queue which we're consuming ended abruptly, this is
# usually a side effect of the process ending and
# multiprocessing not handling things cleanly
break
if work_unit == POISON_PILL:
# This is added at fresh_output_manager_inst
break
else:
args, kwargs = work_unit
#
# Please note that error handling is done inside:
# _call_output_plugins_action
#
self._call_output_plugins_action(*args, **kwargs)
self.in_queue.task_done()
# Now that the new message has been processed by the output plugins
# we flush the output (if needed)
self.flush_plugin_output()
def flush_plugin_output(self):
"""
Call flush() on all plugins so they write their data to the external
file(s) / socket(s) if they want to. This is useful when the scan
takes a lot of time to complete.
By default output plugins have a no-op implementation for flush(), but
plugins like xml_file and csv_file will write to the user configured
files when called.
Only flush once every FLUSH_TIMEOUT seconds.
:see: https://github.com/andresriancho/w3af/issues/6726
:return: None
"""
if not self.should_flush():
return
self.update_last_output_flush()
for o_plugin in self._output_plugin_instances:
#
# Plugins might crash when calling .flush(), a good example was
# the unicodedecodeerror found in xml_file which was triggered
# when the findings were written to file.
#
try:
o_plugin.flush()
except Exception, exception:
self._handle_output_plugin_exception(o_plugin, exception)
class OutputManager(Process):
"""
This class manages output. It has a list of output plugins and sends the
messages to every plugin on that list.
Logs are sent to the "out" instance (see module level variable) and using
a queue we forward the messages to this object which will be calling the
output plugins.
:author: Andres Riancho ([email protected])
"""
FLUSH_TIMEOUT = 60
# Thread locking to avoid starting the om many times from different threads
start_lock = threading.RLock()
def __init__(self):
super(OutputManager, self).__init__(name='OutputManager')
self.daemon = True
self.name = 'OutputManager'
# User configured options
self._output_plugin_instances = []
self._output_plugin_names = []
self._plugin_options = {}
# Internal variables
self.in_queue = SilentJoinableQueue()
self._w3af_core = None
self._last_output_flush = None
def set_w3af_core(self, w3af_core):
self._w3af_core = w3af_core
def get_in_queue(self):
"""
The output sinks need this queue, they will send the messages directly
to it from different threads/processes. Then the main process will
get the log messages and process them in the run() method below.
:return: A multiprocessing Queue
"""
return self.in_queue
def start(self):
with self.start_lock:
if not self.is_alive():
super(OutputManager, self).start()
def run(self):
"""
This method is one of the most important ones in the class, since it
will consume the work units from the queue and send them to the plugins
"""
while True:
try:
work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
except Queue.Empty:
self.flush_plugin_output()
continue
except EOFError:
# The queue which we're consuming ended abruptly, this is
# usually a side effect of the process ending and
# multiprocessing not handling things cleanly
break
if work_unit == POISON_PILL:
# This is added at fresh_output_manager_inst
break
else:
args, kwargs = work_unit
#
# Please note that error handling is done inside:
# _call_output_plugins_action
#
self._call_output_plugins_action(*args, **kwargs)
self.in_queue.task_done()
# Now that the new message has been processed by the output plugins
# we flush the output (if needed)
self.flush_plugin_output()
def flush_plugin_output(self):
"""
Call flush() on all plugins so they write their data to the external
file(s) / socket(s) if they want to. This is useful when the scan
takes a lot of time to complete.
By default output plugins have a no-op implementation for flush(), but
plugins like xml_file and csv_file will write to the user configured
files when called.
Only flush once every FLUSH_TIMEOUT seconds.
:see: https://github.com/andresriancho/w3af/issues/6726
:return: None
"""
if not self.should_flush():
return
self.update_last_output_flush()
for o_plugin in self._output_plugin_instances:
#
# Plugins might crash when calling .flush(), a good example was
# the unicodedecodeerror found in xml_file which was triggered
# when the findings were written to file.
#
try:
o_plugin.flush()
except Exception, exception:
self._handle_output_plugin_exception(o_plugin, exception)
class OutputManager(Process):
"""
This class manages output. It has a list of output plugins and sends the
messages to every plugin on that list.
Logs are sent to the "out" instance (see module level variable) and using
a queue we forward the messages to this object which will be calling the
output plugins.
:author: Andres Riancho ([email protected])
"""
# If the FLUSH_TIMEOUT is larger (60 seconds for example) the time it takes
# for the plugin to process all the messages / vulnerabilities in a call to
# flush() will be larger, because w3af will (most likely) have found more
# vulnerabilities. So we're calling the flush() method more often and having
# shorter calls to flush() than before.
FLUSH_TIMEOUT = 30
# To prevent locks this always needs to be larger than the output plugins
# available in the w3af framework
WORKER_THREADS = 10
# Thread locking to avoid starting the om many times from different threads
start_lock = threading.RLock()
def __init__(self):
super(OutputManager, self).__init__(name='OutputManager')
self.daemon = True
self.name = 'OutputManager'
# User configured options
self._output_plugin_instances = []
self._output_plugin_names = []
self._plugin_options = {}
# Internal variables
self.in_queue = SilentJoinableQueue()
self._w3af_core = None
self._last_output_flush = None
self._is_shutting_down = False
self._worker_pool = self.get_worker_pool()
def set_w3af_core(self, w3af_core):
self._w3af_core = w3af_core
def get_worker_pool(self):
return Pool(self.WORKER_THREADS,
worker_names='WorkerThread',
max_queued_tasks=self.WORKER_THREADS * 10)
def get_in_queue(self):
"""
The output sinks need this queue, they will send the messages directly
to it from different threads/processes. Then the main process will
get the log messages and process them in the run() method below.
:return: A multiprocessing Queue
"""
return self.in_queue
def start(self):
with self.start_lock:
if not self.is_alive():
super(OutputManager, self).start()
def run(self):
"""
This method is one of the most important ones in the class, since it
will consume the work units from the queue and send them to the plugins
"""
while True:
try:
work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
except Queue.Empty:
self.flush_plugin_output()
continue
except EOFError:
# The queue which we're consuming ended abruptly, this is
# usually a side effect of the process ending and
# multiprocessing not handling things cleanly
try:
self.in_queue.task_done()
finally:
break
if work_unit == POISON_PILL:
# This is added at fresh_output_manager_inst
self.in_queue.task_done()
break
elif self._is_shutting_down:
# Just ignore the log message if we're in the process of shutting
# down the output manager. This prevents some race conditions where
# messages are processed after plugins are ended
self.in_queue.task_done()
else:
args, kwargs = work_unit
#
# Please note that error handling is done inside:
# _call_output_plugins_action
#
self._call_output_plugins_action(*args, **kwargs)
self.in_queue.task_done()
# Now that the new message has been processed by the output plugins
# we flush the output (if needed)
self.flush_plugin_output()
def flush_plugin_output(self):
"""
Call flush() on all plugins so they write their data to the external
file(s) / socket(s) if they want to. This is useful when the scan
takes a lot of time to complete.
By default output plugins have a no-op implementation for flush(), but
plugins like xml_file and csv_file will write to the user configured
files when called.
Only flush once every FLUSH_TIMEOUT seconds.
:see: https://github.com/andresriancho/w3af/issues/6726
:return: None
"""
if not self.should_flush():
return
pool = self._worker_pool
if pool.is_closed():
return
self.update_last_output_flush()
for o_plugin in self._output_plugin_instances:
pool.apply_async(func=self.__inner_flush_plugin_output,
args=(o_plugin,))
def __inner_flush_plugin_output(self, o_plugin):
"""
This method is meant to be run in a thread. We run flush() in a thread
to avoid some issues where a long time to process the xml flush was
impacting the messages from being written to the text file.
When we run the flush() call make sure that we're not calling flush again
on the same plugin. This will prevent multiple simultaneous calls to flush
running in an endless way during the scan.
:param o_plugin: The output plugin to run
:return: None, we don't care about the output of this method.
"""
# If for some reason the output plugin takes a lot of time to run
# and the output manager calls flush() for a second time while
# we're still running the first call, just ignore.
if o_plugin.is_running_flush:
import w3af.core.controllers.output_manager as om
msg = ('The %s plugin is still running flush(), the output'
' manager will not call flush() again to give the'
' plugin time to finish.')
args = (o_plugin.get_name(),)
om.out.debug(msg % args)
return
#
# Plugins might crash when calling .flush(), a good example was
# the unicodedecodeerror found in xml_file which was triggered
# when the findings were written to file.
#
start_time = time.time()
o_plugin.is_running_flush = True
try:
o_plugin.flush()
except Exception, exception:
self._handle_output_plugin_exception(o_plugin, exception)
finally:
class OutputManager(Process):
"""
This class manages output. It has a list of output plugins and sends the
messages to every plugin on that list.
Logs are sent to the "out" instance (see module level variable) and using
a queue we forward the messages to this object which will be calling the
output plugins.
:author: Andres Riancho ([email protected])
"""
# Thread locking to avoid starting the om many times from different threads
start_lock = threading.RLock()
def __init__(self):
super(OutputManager, self).__init__(name='OutputManager')
self.daemon = True
self.name = 'OutputManager'
# User configured options
self._output_plugin_instances = []
self._output_plugin_names = []
self._plugin_options = {}
# Internal variables
self.in_queue = SilentJoinableQueue()
self._w3af_core = None
def set_w3af_core(self, w3af_core):
self._w3af_core = w3af_core
def get_in_queue(self):
"""
The output sinks need this queue, they will send the messages directly
to it from different threads/processes. Then the main process will
get the log messages and process them in the run() method below.
:return: A multiprocessing Queue
"""
return self.in_queue
def start(self):
with self.start_lock:
if not self.is_alive():
super(OutputManager, self).start()
def run(self):
"""
This method is one of the most important ones in the class, since it
will consume the work units from the queue and send them to the plugins
"""
while True:
try:
work_unit = self.in_queue.get()
except EOFError:
# The queue which we're consuming ended abruptly, this is
# usually a side effect of the process ending and
# multiprocessing not handling things cleanly
break
if work_unit == POISON_PILL:
# This is added at fresh_output_manager_inst
break
else:
args, kwargs = work_unit
#
# Please note that error handling is done inside:
# _call_output_plugins_action
#
self._call_output_plugins_action(*args, **kwargs)
self.in_queue.task_done()
def end_output_plugins(self):
self.process_all_messages()
self.__end_output_plugins_impl()
@start_thread_on_demand
def process_all_messages(self):
"""
Blocks until all messages are processed
"""
self.in_queue.join()
def __end_output_plugins_impl(self):
for o_plugin in self._output_plugin_instances:
o_plugin.end()
# This is a neat trick which basically removes all plugin references
# from memory. Those plugins might have pointers to memory parts that
# are not required anymore (since someone is calling end_output_plugins
# which indicates that the scan is done).
#
# If the console plugin was enabled, I re-enable it since I don't want
# to loose the capability of seeing my log messages in the console
#
# Remember that the gtk_output plugin disappeared and was moved to
# core.ui.output
currently_enabled_plugins = self.get_output_plugins()
keep_enabled = [pname for pname in currently_enabled_plugins
if pname in ('console',)]
self.set_output_plugins(keep_enabled)
@start_thread_on_demand
def log_enabled_plugins(self, enabled_plugins, plugins_options):
"""
This method logs to the output plugins the enabled plugins and their
configuration.
:param enabled_plugins: As returned by w3afCore's
get_all_enabled_plugins() looks similar to:
{'audit':[],'grep':[],'bruteforce':[],'crawl':[],...}
:param plugins_options: As defined in the w3afCore, looks similar to:
{'audit':{},'grep':{},'bruteforce':{},'crawl':{},...}
"""
for o_plugin in self._output_plugin_instances:
o_plugin.log_enabled_plugins(enabled_plugins, plugins_options)
def _call_output_plugins_action(self, action_name, *args, **kwds):
"""
Internal method used to invoke the requested action on each plugin
in the output plugin list.
A caller to any of the METHODS can specify that the call he's doing
should NOT go to a specific plugin set specified in the ignore_plugins
keyword argument.
"""
encoded_params = []
# http://docs.python.org/2/howto/unicode.html
#
# The most important tip is:
# Software should only work with Unicode strings internally,
# converting to a particular encoding on output.
#
# Given that we don't want to convert to utf8 inside every plugin
# before sending to a file, we do it here
for arg in args:
if isinstance(arg, unicode):
arg = arg.encode(UTF8, 'replace')
encoded_params.append(arg)
args = tuple(encoded_params)
# A caller to any of the METHODS can specify that the call he's doing
# should NOT go to a specific plugin set specified in the ignore_plugins
# keyword argument
#
# We do a pop() here because the plugin method doesn't really receive
# the ignored plugin set, we just filter them at this level.
#
# This is used (mostly) for reporting errors generated in output
# plugins without having the risk of generating a cascading effect
# that would make the output manager go crazy, usually that's done
# by doing something like:
#
# om.out.error(msg, ignore_plugins=set([self.get_name()])
#
ignored_plugins = kwds.pop('ignore_plugins', set())
for o_plugin in self._output_plugin_instances:
if o_plugin.get_name() in ignored_plugins:
continue
try:
opl_func_ptr = getattr(o_plugin, action_name)
apply(opl_func_ptr, args, kwds)
except Exception, e:
if self._w3af_core is None:
return
# Smart error handling, much better than just crashing.
# Doing this here and not with something similar to:
# sys.excepthook = handle_crash because we want to handle
# plugin exceptions in this way, and not framework
# exceptions
#
# FIXME: I need to import this here because of the awful
# singletons I use all over the framework. If imported
# at the top, they will generate circular import errors
from w3af.core.controllers.core_helpers.status import w3af_core_status
class fake_status(w3af_core_status):
pass
status = fake_status(self._w3af_core)
status.set_current_fuzzable_request('output', 'n/a')
status.set_running_plugin('output', o_plugin.get_name(),
log=False)
exec_info = sys.exc_info()
enabled_plugins = 'n/a'
self._w3af_core.exception_handler.handle(status, e,
exec_info,
enabled_plugins)