Example #1
0
    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None
Example #2
0
    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None
        self._last_output_flush = None
        self._is_shutting_down = False
        self._worker_pool = self.get_worker_pool()
Example #3
0
class OutputManager(Process):
    """
    This class manages output. It has a list of output plugins and sends the
    messages to every plugin on that list.

    Logs are sent to the "out" instance (see module level variable) and using
    a queue we forward the messages to this object which will be calling the
    output plugins.

    :author: Andres Riancho ([email protected])
    """

    FLUSH_TIMEOUT = 60

    # Thread locking to avoid starting the om many times from different threads
    start_lock = threading.RLock()

    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None
        self._last_output_flush = None

    def set_w3af_core(self, w3af_core):
        self._w3af_core = w3af_core

    def get_in_queue(self):
        """
        The output sinks need this queue, they will send the messages directly
        to it from different threads/processes. Then the main process will
        get the log messages and process them in the run() method below.

        :return: A multiprocessing Queue
        """
        return self.in_queue

    def start(self):
        with self.start_lock:
            if not self.is_alive():
                super(OutputManager, self).start()

    def run(self):
        """
        This method is one of the most important ones in the class, since it
        will consume the work units from the queue and send them to the plugins
        """
        while True:
            try:
                work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
            except Queue.Empty:
                self.flush_plugin_output()
                continue

            except EOFError:
                # The queue which we're consuming ended abruptly, this is
                # usually a side effect of the process ending and
                # multiprocessing not handling things cleanly
                break

            if work_unit == POISON_PILL:
                # This is added at fresh_output_manager_inst
                break

            else:
                args, kwargs = work_unit
                #
                #    Please note that error handling is done inside:
                #        _call_output_plugins_action
                #
                self._call_output_plugins_action(*args, **kwargs)

                self.in_queue.task_done()

            # Now that the new message has been processed by the output plugins
            # we flush the output (if needed)
            self.flush_plugin_output()

    def flush_plugin_output(self):
        """
        Call flush() on all plugins so they write their data to the external
        file(s) / socket(s) if they want to. This is useful when the scan
        takes a lot of time to complete.

        By default output plugins have a no-op implementation for flush(), but
        plugins like xml_file and csv_file will write to the user configured
        files when called.

        Only flush once every FLUSH_TIMEOUT seconds.

        :see: https://github.com/andresriancho/w3af/issues/6726
        :return: None
        """
        if not self.should_flush():
            return

        self.update_last_output_flush()

        for o_plugin in self._output_plugin_instances:
            #
            #   Plugins might crash when calling .flush(), a good example was
            #   the unicodedecodeerror found in xml_file which was triggered
            #   when the findings were written to file.
            #
            try:
                o_plugin.flush()
            except Exception, exception:
                self._handle_output_plugin_exception(o_plugin, exception)
Example #4
0
class OutputManager(Process):
    """
    This class manages output. It has a list of output plugins and sends the
    messages to every plugin on that list.

    Logs are sent to the "out" instance (see module level variable) and using
    a queue we forward the messages to this object which will be calling the
    output plugins.

    :author: Andres Riancho ([email protected])
    """

    # If the FLUSH_TIMEOUT is larger (60 seconds for example) the time it takes
    # for the plugin to process all the messages / vulnerabilities in a call to
    # flush() will be larger, because w3af will (most likely) have found more
    # vulnerabilities. So we're calling the flush() method more often and having
    # shorter calls to flush() than before.
    FLUSH_TIMEOUT = 30

    # To prevent locks this always needs to be larger than the output plugins
    # available in the w3af framework
    WORKER_THREADS = 10

    # Thread locking to avoid starting the om many times from different threads
    start_lock = threading.RLock()

    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None
        self._last_output_flush = None
        self._is_shutting_down = False
        self._worker_pool = self.get_worker_pool()

    def set_w3af_core(self, w3af_core):
        self._w3af_core = w3af_core

    def get_worker_pool(self):
        return Pool(self.WORKER_THREADS,
                    worker_names='WorkerThread',
                    max_queued_tasks=self.WORKER_THREADS * 10)

    def get_in_queue(self):
        """
        The output sinks need this queue, they will send the messages directly
        to it from different threads/processes. Then the main process will
        get the log messages and process them in the run() method below.

        :return: A multiprocessing Queue
        """
        return self.in_queue

    def start(self):
        with self.start_lock:
            if not self.is_alive():
                super(OutputManager, self).start()

    def run(self):
        """
        This method is one of the most important ones in the class, since it
        will consume the work units from the queue and send them to the plugins
        """
        while True:
            try:
                work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
            except Queue.Empty:
                self.flush_plugin_output()
                continue

            except EOFError:
                # The queue which we're consuming ended abruptly, this is
                # usually a side effect of the process ending and
                # multiprocessing not handling things cleanly
                try:
                    self.in_queue.task_done()
                finally:
                    break

            if work_unit == POISON_PILL:
                # This is added at fresh_output_manager_inst
                self.in_queue.task_done()
                break

            elif self._is_shutting_down:
                # Just ignore the log message if we're in the process of shutting
                # down the output manager. This prevents some race conditions where
                # messages are processed after plugins are ended
                self.in_queue.task_done()

            else:
                args, kwargs = work_unit
                #
                #    Please note that error handling is done inside:
                #        _call_output_plugins_action
                #
                self._call_output_plugins_action(*args, **kwargs)

                self.in_queue.task_done()

            # Now that the new message has been processed by the output plugins
            # we flush the output (if needed)
            self.flush_plugin_output()

    def flush_plugin_output(self):
        """
        Call flush() on all plugins so they write their data to the external
        file(s) / socket(s) if they want to. This is useful when the scan
        takes a lot of time to complete.

        By default output plugins have a no-op implementation for flush(), but
        plugins like xml_file and csv_file will write to the user configured
        files when called.

        Only flush once every FLUSH_TIMEOUT seconds.

        :see: https://github.com/andresriancho/w3af/issues/6726
        :return: None
        """
        if not self.should_flush():
            return

        pool = self._worker_pool
        if pool.is_closed():
            return

        self.update_last_output_flush()

        for o_plugin in self._output_plugin_instances:
            pool.apply_async(func=self.__inner_flush_plugin_output,
                             args=(o_plugin,))

    def __inner_flush_plugin_output(self, o_plugin):
        """
        This method is meant to be run in a thread. We run flush() in a thread
        to avoid some issues where a long time to process the xml flush was
        impacting the messages from being written to the text file.

        When we run the flush() call make sure that we're not calling flush again
        on the same plugin. This will prevent multiple simultaneous calls to flush
        running in an endless way during the scan.

        :param o_plugin: The output plugin to run
        :return: None, we don't care about the output of this method.
        """
        # If for some reason the output plugin takes a lot of time to run
        # and the output manager calls flush() for a second time while
        # we're still running the first call, just ignore.
        if o_plugin.is_running_flush:
            import w3af.core.controllers.output_manager as om

            msg = ('The %s plugin is still running flush(), the output'
                   ' manager will not call flush() again to give the'
                   ' plugin time to finish.')
            args = (o_plugin.get_name(),)
            om.out.debug(msg % args)
            return

        #
        #   Plugins might crash when calling .flush(), a good example was
        #   the unicodedecodeerror found in xml_file which was triggered
        #   when the findings were written to file.
        #
        start_time = time.time()
        o_plugin.is_running_flush = True

        try:
            o_plugin.flush()
        except Exception, exception:
            self._handle_output_plugin_exception(o_plugin, exception)
        finally:
Example #5
0
class OutputManager(Process):
    """
    This class manages output. It has a list of output plugins and sends the
    messages to every plugin on that list.

    Logs are sent to the "out" instance (see module level variable) and using
    a queue we forward the messages to this object which will be calling the
    output plugins.

    :author: Andres Riancho ([email protected])
    """

    FLUSH_TIMEOUT = 60

    # Thread locking to avoid starting the om many times from different threads
    start_lock = threading.RLock()

    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None
        self._last_output_flush = None

    def set_w3af_core(self, w3af_core):
        self._w3af_core = w3af_core

    def get_in_queue(self):
        """
        The output sinks need this queue, they will send the messages directly
        to it from different threads/processes. Then the main process will
        get the log messages and process them in the run() method below.

        :return: A multiprocessing Queue
        """
        return self.in_queue

    def start(self):
        with self.start_lock:
            if not self.is_alive():
                super(OutputManager, self).start()

    def run(self):
        """
        This method is one of the most important ones in the class, since it
        will consume the work units from the queue and send them to the plugins
        """
        while True:
            try:
                work_unit = self.in_queue.get(timeout=self.FLUSH_TIMEOUT)
            except Queue.Empty:
                self.flush_plugin_output()
                continue

            except EOFError:
                # The queue which we're consuming ended abruptly, this is
                # usually a side effect of the process ending and
                # multiprocessing not handling things cleanly
                break

            if work_unit == POISON_PILL:
                # This is added at fresh_output_manager_inst
                break

            else:
                args, kwargs = work_unit
                #
                #    Please note that error handling is done inside:
                #        _call_output_plugins_action
                #
                self._call_output_plugins_action(*args, **kwargs)

                self.in_queue.task_done()

            # Now that the new message has been processed by the output plugins
            # we flush the output (if needed)
            self.flush_plugin_output()

    def flush_plugin_output(self):
        """
        Call flush() on all plugins so they write their data to the external
        file(s) / socket(s) if they want to. This is useful when the scan
        takes a lot of time to complete.

        By default output plugins have a no-op implementation for flush(), but
        plugins like xml_file and csv_file will write to the user configured
        files when called.

        Only flush once every FLUSH_TIMEOUT seconds.

        :see: https://github.com/andresriancho/w3af/issues/6726
        :return: None
        """
        if not self.should_flush():
            return

        self.update_last_output_flush()

        for o_plugin in self._output_plugin_instances:
            #
            #   Plugins might crash when calling .flush(), a good example was
            #   the unicodedecodeerror found in xml_file which was triggered
            #   when the findings were written to file.
            #
            try:
                o_plugin.flush()
            except Exception, exception:
                self._handle_output_plugin_exception(o_plugin, exception)
Example #6
0
class OutputManager(Process):
    """
    This class manages output. It has a list of output plugins and sends the
    messages to every plugin on that list.

    Logs are sent to the "out" instance (see module level variable) and using
    a queue we forward the messages to this object which will be calling the
    output plugins.

    :author: Andres Riancho ([email protected])
    """
    # Thread locking to avoid starting the om many times from different threads
    start_lock = threading.RLock()

    def __init__(self):
        super(OutputManager, self).__init__(name='OutputManager')
        self.daemon = True
        self.name = 'OutputManager'

        # User configured options
        self._output_plugin_instances = []
        self._output_plugin_names = []
        self._plugin_options = {}

        # Internal variables
        self.in_queue = SilentJoinableQueue()
        self._w3af_core = None

    def set_w3af_core(self, w3af_core):
        self._w3af_core = w3af_core

    def get_in_queue(self):
        """
        The output sinks need this queue, they will send the messages directly
        to it from different threads/processes. Then the main process will
        get the log messages and process them in the run() method below.

        :return: A multiprocessing Queue
        """
        return self.in_queue

    def start(self):
        with self.start_lock:
            if not self.is_alive():
                super(OutputManager, self).start()

    def run(self):
        """
        This method is one of the most important ones in the class, since it
        will consume the work units from the queue and send them to the plugins
        """
        while True:
            try:
                work_unit = self.in_queue.get()
            except EOFError:
                # The queue which we're consuming ended abruptly, this is
                # usually a side effect of the process ending and
                # multiprocessing not handling things cleanly
                break

            if work_unit == POISON_PILL:
                # This is added at fresh_output_manager_inst
                break

            else:
                args, kwargs = work_unit
                #
                #    Please note that error handling is done inside:
                #        _call_output_plugins_action
                #
                self._call_output_plugins_action(*args, **kwargs)

                self.in_queue.task_done()

    def end_output_plugins(self):
        self.process_all_messages()
        self.__end_output_plugins_impl()

    @start_thread_on_demand
    def process_all_messages(self):
        """
        Blocks until all messages are processed
        """
        self.in_queue.join()

    def __end_output_plugins_impl(self):
        for o_plugin in self._output_plugin_instances:
            o_plugin.end()

        # This is a neat trick which basically removes all plugin references
        # from memory. Those plugins might have pointers to memory parts that
        # are not required anymore (since someone is calling end_output_plugins
        # which indicates that the scan is done).
        #
        # If the console plugin was enabled, I re-enable it since I don't want
        # to loose the capability of seeing my log messages in the console
        #
        # Remember that the gtk_output plugin disappeared and was moved to
        # core.ui.output
        currently_enabled_plugins = self.get_output_plugins()
        keep_enabled = [pname for pname in currently_enabled_plugins
                        if pname in ('console',)]
        self.set_output_plugins(keep_enabled)

    @start_thread_on_demand
    def log_enabled_plugins(self, enabled_plugins, plugins_options):
        """
        This method logs to the output plugins the enabled plugins and their
        configuration.

        :param enabled_plugins: As returned by w3afCore's
                                get_all_enabled_plugins() looks similar to:
                   {'audit':[],'grep':[],'bruteforce':[],'crawl':[],...}

        :param plugins_options: As defined in the w3afCore, looks similar to:
                   {'audit':{},'grep':{},'bruteforce':{},'crawl':{},...}
        """
        for o_plugin in self._output_plugin_instances:
            o_plugin.log_enabled_plugins(enabled_plugins, plugins_options)

    def _call_output_plugins_action(self, action_name, *args, **kwds):
        """
        Internal method used to invoke the requested action on each plugin
        in the output plugin list.
        
        A caller to any of the METHODS can specify that the call he's doing
        should NOT go to a specific plugin set specified in the ignore_plugins
        keyword argument.
        
        """
        encoded_params = []

        # http://docs.python.org/2/howto/unicode.html
        #
        # The most important tip is:
        #     Software should only work with Unicode strings internally,
        #     converting to a particular encoding on output.
        #
        # Given that we don't want to convert to utf8 inside every plugin
        # before sending to a file, we do it here
        for arg in args:
            if isinstance(arg, unicode):
                arg = arg.encode(UTF8, 'replace')

            encoded_params.append(arg)

        args = tuple(encoded_params)
        
        # A caller to any of the METHODS can specify that the call he's doing
        # should NOT go to a specific plugin set specified in the ignore_plugins
        # keyword argument
        #
        # We do a pop() here because the plugin method doesn't really receive
        # the ignored plugin set, we just filter them at this level.
        #
        # This is used (mostly) for reporting errors generated in output
        # plugins without having the risk of generating a cascading effect
        # that would make the output manager go crazy, usually that's done
        # by doing something like:
        #
        #    om.out.error(msg, ignore_plugins=set([self.get_name()])
        #
        ignored_plugins = kwds.pop('ignore_plugins', set())
        
        for o_plugin in self._output_plugin_instances:
            
            if o_plugin.get_name() in ignored_plugins:
                continue
            
            try:
                opl_func_ptr = getattr(o_plugin, action_name)
                apply(opl_func_ptr, args, kwds)
            except Exception, e:
                if self._w3af_core is None:
                    return
                
                # Smart error handling, much better than just crashing.
                # Doing this here and not with something similar to:
                # sys.excepthook = handle_crash because we want to handle
                # plugin exceptions in this way, and not framework
                # exceptions
                #
                # FIXME: I need to import this here because of the awful
                #        singletons I use all over the framework. If imported
                #        at the top, they will generate circular import errors
                from w3af.core.controllers.core_helpers.status import w3af_core_status

                class fake_status(w3af_core_status):
                    pass

                status = fake_status(self._w3af_core)
                status.set_current_fuzzable_request('output', 'n/a')
                status.set_running_plugin('output', o_plugin.get_name(),
                                          log=False)

                exec_info = sys.exc_info()
                enabled_plugins = 'n/a'
                self._w3af_core.exception_handler.handle(status, e,
                                                         exec_info,
                                                         enabled_plugins)