예제 #1
0
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta())
        self.application = weakref_ref(application)
        self.agents = AgentList()
        if RUNNING_LINUX:
            if application.options.fast:
                self.system_calm = SystemCalm(0.75, 0.2)
            elif not application.options.slow:
                self.system_calm = SystemCalm(0.50, 0.5)
            else:
                self.system_calm = SystemCalm(0.30, 3.0)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = application.options.session
        self.success_score = 0.50  # minimum score for a successful session
        self.error_score = -0.50  # maximum score for a session failure
        self.max_success = application.options.success

        # Session
        self.session = None
        self.session_index = 0
        self.session_timeout = None  # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None

        # Add Application agents, order is important: MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if application.options.aggressivity is not None:
            self.aggressivity.setValue(application.options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)
예제 #2
0
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta())
        self.application = weakref_ref(application)
        self.agents = AgentList()
        if RUNNING_LINUX:
            if application.options.fast:
                self.system_calm = SystemCalm(0.75, 0.2)
            elif not application.options.slow:
                self.system_calm = SystemCalm(0.50, 0.5)
            else:
                self.system_calm = SystemCalm(0.30, 3.0)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = application.options.session
        self.success_score = 0.50 # minimum score for a successful session
        self.error_score = -0.50 # maximum score for a session failure
        self.max_success = application.options.success

        # Session
        self.session = None
        self.session_index = 0
        self.session_timeout = None # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None

        # Add Application agents, order is important: MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if application.options.aggressivity is not None:
            self.aggressivity.setValue(application.options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)
예제 #3
0
class Project(ProjectAgent):
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta())
        self.application = weakref_ref(application)
        self.agents = AgentList()
        if RUNNING_LINUX:
            if application.options.fast:
                self.system_calm = SystemCalm(0.75, 0.2)
            elif not application.options.slow:
                self.system_calm = SystemCalm(0.50, 0.5)
            else:
                self.system_calm = SystemCalm(0.30, 3.0)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = application.options.session
        self.success_score = 0.50 # minimum score for a successful session
        self.error_score = -0.50 # maximum score for a session failure
        self.max_success = application.options.success

        # Session
        self.session = None
        self.session_index = 0
        self.session_timeout = None # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None

        # Add Application agents, order is important: MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if application.options.aggressivity is not None:
            self.aggressivity.setValue(application.options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)

    def registerAgent(self, agent):
        self.debug("Register %r" % agent)
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.debug("Unregister %r" % agent)
        self.agents.remove(agent, destroy)

    def init(self):
        self.directory = ProjectDirectory(self)
        self.directory.activate()
        self.error("Use directory: %s" % self.directory.directory)

        self.initLog()

        self.project_start = time()
        self.step = None
        self.nb_success = 0
        self.createSession()

    def initLog(self):
        # Move fusil.log into run-xxx/project.log: copy fusil.log content
        # and then remove fusil.log file and log handler)
        logger = self.application().logger
        filename = self.directory.uniqueFilename("project.log")
        copyfile(logger.filename, filename)
        self.log_handler = logger.addFileHandler(filename, mode='a')
        logger.removeFileHandler(logger.file_handler)
        unlink(logger.filename)
        logger.filename = filename

    def deinit(self):
        self.summarize()
        self.aggressivity = None
        self.debug("Remove all project agents")
        for agent in self.application().agents:
            self.agents.remove(agent, False)
        self.agents.clear()

        remove = self.directory.destroy()
        if remove:
            logger = self.application().logger
            logger.removeFileHandler(self.log_handler)
            logger.filename = None

        self.directory = None
        if RUNNING_PYPY:
            gc_collect()

    def createSession(self):
        # Wait until system is calm
        if self.system_calm:
            self.system_calm.wait(self)

        self.info("Create session")
        self.step = 0
        self.session_index += 1
        self.session_start = time()

        # Enable project agents
        for agent in self.agents:
            if not agent.is_active:
                agent.activate()

        # Create session
        self.session = Session(self)

        # Send 'project_start' and 'session_start' message
        if self.session_index == 1:
            self.send('project_start')
        self.send('session_start')
        self.error("Start session")

    def destroySession(self):
        self.info("Destroy session")

        # Update statistics
        if not self.application().exitcode:
            self.session_executed += 1
            self.session_total_duration += (time() - self.session_start)

        # First deactivate session agents
        self.session.deactivate()

        # Deactivate project agents
        application_agents = self.application().agents
        for agent in self.agents:
            if agent not in application_agents:
                agent.deactivate()

        # Clear session variables
        self.step = None
        self.session = None

        # Remove waiting messages
        for agent in application_agents:
            agent.mailbox.clear()
        self.mta().clear()

    def on_session_done(self, session_score):
        self.send('project_session_destroy', session_score)

    def on_project_stop(self):
        self.send('univers_stop')

    def on_univers_stop(self):
        if self.session:
            self.destroySession()

    def on_project_session_destroy(self, session_score):
        # Use session score
        self.session.score = session_score
        duration = time() - self.session_start
        if self.project().success_score <= session_score:
            log = self.error
        else:
            log = self.warning
        log("End of session: score=%.1f%%, duration=%.3f second" % (
            session_score*100, duration))

        # Destroy session
        self.destroySession()

        # Session success? project is done
        if self.project().success_score <= session_score:
            self.nb_success += 1
            self.error("Success %s/%s!" % (self.nb_success, self.max_success))
            if self.max_success <= self.nb_success:
                self.send('univers_stop')
                return

        # Hit maximum number of session?
        if self.max_session and self.max_session <= self.session_index:
            self.error("Stop (limited to %s session)" % self.max_session)
            self.send('univers_stop')
            return

        # Otherwise: start new session
        self.createSession()

    def live(self):
        if self.step is not None:
            self.step += 1
        if not self.session:
            return
        if not self.session_timeout:
            return
        duration = time() - self.session_start
        if self.session_timeout <= duration:
            self.error("Timeout!")
            self.send('session_stop')

    def summarize(self):
        count = self.session_executed
        info = []
        if count:
            duration = self.session_total_duration
            info.append("%s session in %.1f second (%.1f ms per session)"
                % (count, duration, duration * 1000 / count))
        duration = time() - self.project_start
        info.append("total %.1f second" % duration)
        info.append("aggresssivity: %s" % self.aggressivity)
        self.error("Project done: %s" % ", ".join(info))
        self.error("Total: %s success" % self.nb_success)
예제 #4
0
 def __init__(self, project):
     self.agents = AgentList()
     self.score = None
     self.log_handler = None
     SessionAgent.__init__(self, self, "session", project=project)
예제 #5
0
class Session(SessionAgent):
    def __init__(self, project):
        self.agents = AgentList()
        self.score = None
        self.log_handler = None
        SessionAgent.__init__(self, self, "session", project=project)

    def isSuccess(self):
        if self.score is None:
            return False
        return self.project().success_score <= self.score

    def computeScore(self, verbose=False):
        session_score = 0
        if verbose:
            self.info("Compute score")
        for agent in self.project().agents:
            if not issubclass(agent.__class__, ProjectAgent):
                # Skip application agent which has no score
                continue
            if not agent.is_active:
                continue
            score = agent.getScore()
            if score is None:
                continue
            score = normalizeScore(score)
            score *= agent.score_weight
            if verbose:
                self.info("%s score: %.1f%%" % (agent, score*100))
            session_score += score
        return session_score

    def writeAgents(self):
        self.debug("Project agents:")
        for agent in self.project().agents:
            self.debug("- %r" % agent)

    def registerAgent(self, agent):
        self.debug("Register %r" % agent)
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.debug("Unregister %r" % agent)
        self.agents.remove(agent, destroy)

    def init(self):
        directory = self.project().directory.directory
        self.directory = SessionDirectory(self, directory)

        log_filename = self.directory.uniqueFilename("session.log")
        self.log_handler = self.logger.addFileHandler(
            log_filename, level=INFO, formatter=SessionFormatter())

        self.stopped = False

    def on_session_start(self):
        self.writeAgents()

    def deinit(self):
        if self.log_handler:
            self.logger.removeFileHandler(self.log_handler)
        self.debug("Remove all session agents")
        self.agents.clear()
        if RUNNING_PYPY:
            gc_collect()

    def live(self):
        if self.stopped:
            return
        score = self.computeScore()
        if score is None:
            return
        project = self.project()
        if not(project.success_score <= score or score <= project.error_score):
            return
        self.send('session_stop')

    def on_session_stop(self):
        if self.stopped:
            return
        self.stopped = True
        score = self.computeScore(True)
        self.send('session_done', score)
예제 #6
0
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta(), application=application)
        self.config = application.config
        options = application.options
        self.agents = AgentList()
        if RUNNING_LINUX:
            if options.fast:
                self.system_calm = None
            elif not options.slow:
                self.system_calm = SystemCalm(
                    self.config.fusil_normal_calm_load,
                    self.config.fusil_normal_calm_sleep)
            else:
                self.system_calm = SystemCalm(
                    self.config.fusil_slow_calm_load,
                    self.config.fusil_slow_calm_sleep)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = options.sessions
        self.success_score = self.config.fusil_success_score
        self.error_score = self.config.fusil_error_score
        self.max_success = options.success

        # Session
        self.step = None
        self.nb_success = 0
        self.session = None
        self.session_index = 0
        self.session_timeout = None # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None
        self._destroyed = False

        # Add Application agents, order is important:
        # MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if options.aggressivity is not None:
            self.aggressivity.setValue(options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)

        # Create the debugger
        self.debugger = Debugger(self)

        # Create the working directory
        self.directory = ProjectDirectory(self)
        self.directory.activate()
        self.error("Use directory: %s" % self.directory.directory)

        # Initilize project logging
        self.initLog()
예제 #7
0
class Project(ProjectAgent):
    """
    A fuzzer project runs fuzzing sessions until we get enough successes or the
    user interrupts the project. Initialize all agents before a session starts,
    and cleanup agents at the session end.

    Before a session start, the project sleeps until the system load is under
    50% (may change with command line options).
    """
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta(), application=application)
        self.config = application.config
        options = application.options
        self.agents = AgentList()
        if RUNNING_LINUX:
            if options.fast:
                self.system_calm = None
            elif not options.slow:
                self.system_calm = SystemCalm(
                    self.config.fusil_normal_calm_load,
                    self.config.fusil_normal_calm_sleep)
            else:
                self.system_calm = SystemCalm(
                    self.config.fusil_slow_calm_load,
                    self.config.fusil_slow_calm_sleep)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = options.sessions
        self.success_score = self.config.fusil_success_score
        self.error_score = self.config.fusil_error_score
        self.max_success = options.success

        # Session
        self.step = None
        self.nb_success = 0
        self.session = None
        self.session_index = 0
        self.session_timeout = None # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None
        self._destroyed = False

        # Add Application agents, order is important:
        # MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if options.aggressivity is not None:
            self.aggressivity.setValue(options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)

        # Create the debugger
        self.debugger = Debugger(self)

        # Create the working directory
        self.directory = ProjectDirectory(self)
        self.directory.activate()
        self.error("Use directory: %s" % self.directory.directory)

        # Initilize project logging
        self.initLog()

    def registerAgent(self, agent):
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.agents.remove(agent, destroy)

    def init(self):
        """
        Function called once on project creation: create the project working
        directory, prepare the logging and create the first session.
        """
        self.project_start = time()
        self.createSession()

    def initLog(self):
        # Move fusil.log into run-xxx/project.log: copy fusil.log content
        # and then remove fusil.log file and log handler)
        logger = self.application().logger
        filename = self.createFilename("project.log")
        if logger.filename:
            copyfile(logger.filename, filename)
            logger.unlinkFile()
            mode = 'a'
        else:
            mode = 'w'
        logger.file_handler = logger.addFileHandler(filename, mode=mode)
        logger.filename = filename

    def deinit(self):
        if self.session_executed:
            self.summarize()

    def destroy(self):
        if self._destroyed:
            return
        self._destroyed = True

        # Destroy all project agents
        self.aggressivity = None
        self.debugger = None
        for agent in self.application().agents:
            self.agents.remove(agent, False)
        self.agents.clear()

        # Keep project directory?
        keep = self.directory.keepDirectory()
        if not keep:
            # Don't keep the directory: destroy log file
            logger = self.application().logger
            logger.unlinkFile()
            # And then remove the whole directory
            self.directory.rmtree()
        self.directory = None

        if RUNNING_PYPY:
            gc_collect()

    def createSession(self):
        """
        Create a new session:
         - make sure that system load is under 50%
         - activate all project agents
         - send project_start (only for the first session)
           and session_start messages
        """
        # Wait until system is calm
        if self.system_calm:
            self.system_calm.wait(self)

        self.info("Create session")
        self.step = 0
        self.session_index += 1
        self.use_timeout = bool(self.session_timeout)
        self.session_start = time()

        # Enable project agents
        for agent in self.agents:
            if not agent.is_active:
                agent.activate()

        # Create session
        self.session = Session(self)

        # Send 'project_start' and 'session_start' message
        if self.session_index == 1:
            self.send('project_start')
        self.send('session_start')
        text = "Start session"
        if self.max_session:
            percent = self.session_index * 100.0 / self.max_session
            text += " (%.1f%%)" % percent
        self.error(text)


    def destroySession(self):
        """
        Destroy the current session:
         - deactive all project agents
         - clear agents mailbox
        """
        # Update statistics
        if not self.application().exitcode:
            self.session_executed += 1
            self.session_total_duration += (time() - self.session_start)

        # First deactivate session agents
        self.session.deactivate()

        # Deactivate project agents
        application_agents = self.application().agents
        for agent in self.agents:
            if agent not in application_agents:
                agent.deactivate()

        # Clear session variables
        self.step = None
        self.session = None

        # Remove waiting messages
        for agent in application_agents:
            agent.mailbox.clear()
        self.mta().clear()

    def on_session_done(self, session_score):
        self.send('project_session_destroy', session_score)

    def on_project_stop(self):
        self.send('univers_stop')

    def on_univers_stop(self):
        if self.session:
            self.destroySession()

    def on_project_session_destroy(self, session_score):
        # Use session score
        self.session.score = session_score
        duration = time() - self.session_start
        if self.success_score <= session_score:
            log = self.error
        else:
            log = self.warning
        log("End of session: score=%.1f%%, duration=%.3f second" % (
            session_score*100, duration))

        # Destroy session
        self.destroySession()

        # Session success? project is done
        if self.success_score <= session_score:
            self.nb_success += 1
            text = "#%s" % self.nb_success
            if 0 < self.max_success:
                percent = self.nb_success * 100.0 / self.max_success
                text += "/%s (%.1f%%)" % (self.max_success, percent)
            self.error("Success %s!" % text)
            if 0 < self.max_success \
            and self.max_success <= self.nb_success:
                self.error("Stop! Limited to %s successes, use --success option for more" % self.max_success)
                self.send('univers_stop')
                return

        # Hit maximum number of session?
        if 0 < self.max_session \
        and self.max_session <= self.session_index:
            self.error("Stop! Limited to %s sessions, use --sessions option for more" % self.max_session)
            self.send('univers_stop')
            return

        # Otherwise: start new session
        self.createSession()

    def live(self):
        if self.step is not None:
            self.step += 1
        if not self.session:
            return
        if not self.use_timeout:
            return
        duration = time() - self.session_start
        if self.session_timeout <= duration:
            self.error("Project session timeout!")
            self.send('session_stop')
            self.use_timeout = False

    def summarize(self):
        """
        Display a summary of all executed sessions
        """
        count = self.session_executed
        info = []
        if count:
            duration = self.session_total_duration
            info.append("%s sessions in %.1f seconds (%.1f ms per session)"
                % (count, duration, duration * 1000 / count))
        duration = time() - self.project_start
        info.append("total %.1f seconds" % duration)
        info.append("aggresssivity: %s" % self.aggressivity)
        self.error("Project done: %s" % ", ".join(info))
        self.error("Total: %s success" % self.nb_success)

    def createFilename(self, filename, count=None):
        """
        Create a filename in the project working directory: add directory
        prefix and make sure that the generated filename is unique.
        """
        return self.directory.uniqueFilename(filename, count=count)
예제 #8
0
 def __init__(self, project):
     self.agents = AgentList()
     self.score = None
     self.log_handler = None
     SessionAgent.__init__(self, self, "session", project=project)
예제 #9
0
class Session(SessionAgent):
    def __init__(self, project):
        self.agents = AgentList()
        self.score = None
        self.log_handler = None
        SessionAgent.__init__(self, self, "session", project=project)

    def isSuccess(self):
        if self.score is None:
            return False
        return self.project().success_score <= self.score

    def computeScore(self, verbose=False):
        session_score = 0
        if verbose:
            self.info("Compute score")
        for agent in self.project().agents:
            if not issubclass(agent.__class__, ProjectAgent):
                # Skip application agent which has no score
                continue
            if not agent.is_active:
                continue
            score = agent.getScore()
            if score is None:
                continue
            score = normalizeScore(score)
            score *= agent.score_weight
            if verbose:
                self.info("%s score: %.1f%%" % (agent, score * 100))
            session_score += score
        return session_score

    def writeAgents(self):
        self.debug("Project agents:")
        for agent in self.project().agents:
            self.debug("- %r" % agent)

    def registerAgent(self, agent):
        self.debug("Register %r" % agent)
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.debug("Unregister %r" % agent)
        self.agents.remove(agent, destroy)

    def init(self):
        directory = self.project().directory.directory
        self.directory = SessionDirectory(self, directory)

        log_filename = self.directory.uniqueFilename("session.log")
        self.log_handler = self.logger.addFileHandler(
            log_filename, level=INFO, formatter=SessionFormatter())

        self.stopped = False

    def on_session_start(self):
        self.writeAgents()

    def deinit(self):
        if self.log_handler:
            self.logger.removeFileHandler(self.log_handler)
        self.debug("Remove all session agents")
        self.agents.clear()
        if RUNNING_PYPY:
            gc_collect()

    def live(self):
        if self.stopped:
            return
        score = self.computeScore()
        if score is None:
            return
        project = self.project()
        if not (project.success_score <= score
                or score <= project.error_score):
            return
        self.send('session_stop')

    def on_session_stop(self):
        if self.stopped:
            return
        self.stopped = True
        score = self.computeScore(True)
        self.send('session_done', score)
예제 #10
0
 def __init__(self, project):
     self.agents = AgentList()
     self.score = None
     self.log_handler = None
     name = "session %s" % project.session_index
     SessionAgent.__init__(self, self, name, project=project)
예제 #11
0
class Session(SessionAgent):
    """
    A session of the fuzzer:
     - create a directory as working directory
     - compute the score of the session
    """
    def __init__(self, project):
        self.agents = AgentList()
        self.score = None
        self.log_handler = None
        name = "session %s" % project.session_index
        SessionAgent.__init__(self, self, name, project=project)

    def isSuccess(self):
        if self.score is None:
            return False
        return self.project().success_score <= self.score

    def computeScore(self, verbose=False):
        """
        Compute the score of the session:
         - call getScore() method of all agents
         - normalize the score in [-1.0; 1.0]
         - apply score factor (weight)
         - compute the sum of all scores
        """
        session_score = 0
        for agent in self.project().agents:
            if not issubclass(agent.__class__, ProjectAgent):
                # Skip application agent which has no score
                continue
            if not agent.is_active:
                continue
            score = agent.getScore()
            if score is None:
                continue
            score = normalizeScore(score)
            score *= agent.score_weight
            score = normalizeScore(score)
            if verbose and score:
                self.info("- %s score: %.1f%%" % (agent, score*100))
            session_score += score
        return session_score

    def registerAgent(self, agent):
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.agents.remove(agent, destroy)

    def init(self):
        self.directory = SessionDirectory(self)

        log_filename = self.createFilename("session.log")
        self.log_handler = self.logger.addFileHandler(
            log_filename, level=INFO, formatter_class=SessionFormatter)

        self.stopped = False

    def deinit(self):
        if self.log_handler:
            self.logger.removeFileHandler(self.log_handler)
        self.agents.clear()
        if RUNNING_PYPY:
            gc_collect()

    def live(self):
        """
        Compute the score of the session and stop the session if the score is
        smaller than -50% or bigger than 50%.
        """
        if self.stopped:
            return
        score = self.computeScore()
        if score is None:
            return
        project = self.project()
        if not(project.success_score <= score or score <= project.error_score):
            return
        self.send('session_stop')

    def on_session_stop(self):
        if self.stopped:
            return
        self.stopped = True
        score = self.computeScore(True)
        if self.project().success_score <= score:
            self.send('session_success')
        self.send('session_done', score)

    def createFilename(self, filename, count=None):
        """
        Create a filename in the session working directory: add directory
        prefix and make sure that the generated filename is unique.
        """
        return self.directory.uniqueFilename(filename, count=count)
예제 #12
0
 def __init__(self):
     self.agents = AgentList()
     ApplicationAgent.__init__(self, "application", self, None)
     self.setup()
예제 #13
0
class Application(ApplicationAgent):
    def __init__(self):
        self.agents = AgentList()
        ApplicationAgent.__init__(self, "application", self, None)
        self.setup()

    def registerAgent(self, agent):
        self.debug("Register %r" % agent)
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.debug("Unregister %r" % agent)
        self.agents.remove(agent, destroy)

    def parseOptions(self):
        parser = OptionParser(
            usage="%prog [options] --project=NAME [arg1 arg2 ...]")
        parser.add_option("--project",
                          '-p',
                          help="Project filename",
                          type="str",
                          default=None)
        parser.add_option("--session",
                          help="Maximum number of session (default: none)",
                          type="int")
        parser.add_option(
            "--success",
            help="Maximum number of success sessions (default: 5)",
            type="int",
            default=5)
        parser.add_option(
            "--remove-generated-files",
            help=
            "Remove a session directory even if it contains generated files",
            action="store_true")
        parser.add_option("--keep-sessions",
                          help="Do not remove session directories",
                          action="store_true")
        parser.add_option("--fast",
                          help="Run faster as possible (opposite of --slow)",
                          action="store_true")
        parser.add_option(
            "--slow",
            help=
            "Try to keep system load low: be nice with CPU (opposite of --fast)",
            action="store_true")
        parser.add_option("--version",
                          help="Display Fusil version (%s) and exit" % VERSION,
                          action="store_true")
        parser.add_option(
            "--aggressivity",
            help=
            "Initial aggressivity factor in percent, value in -100.0..100.0 (default: 0.0%%)",
            type="float",
            default=None)
        parser.add_option(
            '-v',
            "--verbose",
            help="Enable verbose mode (set log level to WARNING)",
            action="store_true")
        parser.add_option(
            "--quiet",
            help="Be quiet (lowest log level), don't create log file",
            action="store_true")
        parser.add_option("--profiler",
                          help="Enable Python profiler",
                          action="store_true")
        parser.add_option("--debug",
                          help="Enable debug mode (set log level to DEBUG)",
                          action="store_true")
        self.options, self.arguments = parser.parse_args()

        # Just want to know the version?
        if self.options.version:
            print "Fusil version %s" % VERSION
            print "License: %s" % LICENSE
            print "Website: %s" % WEBSITE
            print
            exit(0)

        if self.options.quiet:
            self.options.debug = False
            self.options.verbose = False
        if self.options.debug:
            self.options.verbose = True
        if not self.options.project:
            parser.print_help()
            exit(1)

    def setup(self):
        # Read command line options
        self.parseOptions()

        # Application objects
        self.max_memory = 100 * 1024 * 1024
        self.exitcode = 0
        self.project = None

        # Limit Fusil environment
        beNice(True)
        if self.max_memory:
            limitMemory(self.max_memory)

        # Create logger
        self.logger = ApplicationLogger(self)
        self.error("Fusil version %s -- %s" % (VERSION, LICENSE))
        self.error(WEBSITE)
        dumpProcessInfo(self.info, getpid())

        # Create multi agent system
        self.createMAS()

    def createMAS(self):
        # Create mail transfer agent (MTA)
        self.mta = None
        mta = MTA(self)

        # Create univers
        if self.options.fast:
            step_sleep = 0.005
        elif not self.options.slow:
            step_sleep = 0.010
        else:
            step_sleep = 0.050
        self.univers = Univers(self, mta, step_sleep)

        # Finish to setup application
        self.setupMTA(mta, self.logger)
        self.registerAgent(self)

        # Activate agents
        mta.activate()
        self.activate()
        self.univers.activate()

    def exit(self):
        if self.logger.filename:
            self.error("Fusil log written into %s" % self.logger.filename)
        self.error("Exit Fusil")
        self.mta = None
        self.univers = None
        self.agents.clear()

    def getInputFilename(self, description):
        arguments = self.arguments
        if not arguments:
            raise RuntimeError("Missing filename argument: %s" % description)
        return arguments[0]

    def executeProject(self):
        self.project.activate()
        self.univers.execute(self.project)
        self.project.deactivate()

    def runProject(self, filename):
        # Load project
        self.error("Load project %s" % filename)
        self.project = loadProject(self, filename)
        self.registerAgent(self.project)

        # Execute project
        self.executeProject()

        # Destroy project
        self.info("Destroy project")
        self.unregisterAgent(self.project)
        self.project = None

    def on_application_interrupt(self):
        self.error("User interrupt!")
        self.send('univers_stop')

    def on_application_error(self, message):
        self.error(message)
        self.exitcode = 1
        self.send('univers_stop')

    def main(self):
        try:
            if self.options.profiler:
                from fusil.profiler import runProfiler
                runProfiler(self, self.runProject, (self.options.project, ))
            else:
                self.runProject(self.options.project)
        except KeyboardInterrupt:
            self.error("Project interrupted!")
            self.exitcode = 1
        except FUSIL_ERRORS, error:
            writeError(self, error)
            self.exitcode = 1
        return self.exitcode
예제 #14
0
class Project(ProjectAgent):
    def __init__(self, application):
        ProjectAgent.__init__(self, self, "project", mta=application.mta())
        self.application = weakref_ref(application)
        self.agents = AgentList()
        if RUNNING_LINUX:
            if application.options.fast:
                self.system_calm = SystemCalm(0.75, 0.2)
            elif not application.options.slow:
                self.system_calm = SystemCalm(0.50, 0.5)
            else:
                self.system_calm = SystemCalm(0.30, 3.0)
        else:
            self.warning("SystemCalm class is not available")
            self.system_calm = None

        # Configuration
        self.max_session = application.options.session
        self.success_score = 0.50  # minimum score for a successful session
        self.error_score = -0.50  # maximum score for a session failure
        self.max_success = application.options.success

        # Session
        self.session = None
        self.session_index = 0
        self.session_timeout = None  # in second

        # Statistics
        self.session_executed = 0
        self.session_total_duration = 0
        self.total_duration = None

        # Add Application agents, order is important: MTA have to be the first agent
        for agent in application.agents:
            self.registerAgent(agent)
        self.registerAgent(self)

        # Create aggressivity agent
        self.aggressivity = AggressivityAgent(self)

        # Initial aggresssivity value
        if application.options.aggressivity is not None:
            self.aggressivity.setValue(application.options.aggressivity / 100)
            self.error("Initial aggressivity: %s" % self.aggressivity)

    def registerAgent(self, agent):
        self.debug("Register %r" % agent)
        self.agents.append(agent)

    def unregisterAgent(self, agent, destroy=True):
        if agent not in self.agents:
            return
        self.debug("Unregister %r" % agent)
        self.agents.remove(agent, destroy)

    def init(self):
        self.directory = ProjectDirectory(self)
        self.directory.activate()
        self.error("Use directory: %s" % self.directory.directory)

        self.initLog()

        self.project_start = time()
        self.step = None
        self.nb_success = 0
        self.createSession()

    def initLog(self):
        # Move fusil.log into run-xxx/project.log: copy fusil.log content
        # and then remove fusil.log file and log handler)
        logger = self.application().logger
        filename = self.directory.uniqueFilename("project.log")
        copyfile(logger.filename, filename)
        self.log_handler = logger.addFileHandler(filename, mode='a')
        logger.removeFileHandler(logger.file_handler)
        unlink(logger.filename)
        logger.filename = filename

    def deinit(self):
        self.summarize()
        self.aggressivity = None
        self.debug("Remove all project agents")
        for agent in self.application().agents:
            self.agents.remove(agent, False)
        self.agents.clear()

        remove = self.directory.destroy()
        if remove:
            logger = self.application().logger
            logger.removeFileHandler(self.log_handler)
            logger.filename = None

        self.directory = None
        if RUNNING_PYPY:
            gc_collect()

    def createSession(self):
        # Wait until system is calm
        if self.system_calm:
            self.system_calm.wait(self)

        self.info("Create session")
        self.step = 0
        self.session_index += 1
        self.session_start = time()

        # Enable project agents
        for agent in self.agents:
            if not agent.is_active:
                agent.activate()

        # Create session
        self.session = Session(self)

        # Send 'project_start' and 'session_start' message
        if self.session_index == 1:
            self.send('project_start')
        self.send('session_start')
        self.error("Start session")

    def destroySession(self):
        self.info("Destroy session")

        # Update statistics
        if not self.application().exitcode:
            self.session_executed += 1
            self.session_total_duration += (time() - self.session_start)

        # First deactivate session agents
        self.session.deactivate()

        # Deactivate project agents
        application_agents = self.application().agents
        for agent in self.agents:
            if agent not in application_agents:
                agent.deactivate()

        # Clear session variables
        self.step = None
        self.session = None

        # Remove waiting messages
        for agent in application_agents:
            agent.mailbox.clear()
        self.mta().clear()

    def on_session_done(self, session_score):
        self.send('project_session_destroy', session_score)

    def on_project_stop(self):
        self.send('univers_stop')

    def on_univers_stop(self):
        if self.session:
            self.destroySession()

    def on_project_session_destroy(self, session_score):
        # Use session score
        self.session.score = session_score
        duration = time() - self.session_start
        if self.project().success_score <= session_score:
            log = self.error
        else:
            log = self.warning
        log("End of session: score=%.1f%%, duration=%.3f second" %
            (session_score * 100, duration))

        # Destroy session
        self.destroySession()

        # Session success? project is done
        if self.project().success_score <= session_score:
            self.nb_success += 1
            self.error("Success %s/%s!" % (self.nb_success, self.max_success))
            if self.max_success <= self.nb_success:
                self.send('univers_stop')
                return

        # Hit maximum number of session?
        if self.max_session and self.max_session <= self.session_index:
            self.error("Stop (limited to %s session)" % self.max_session)
            self.send('univers_stop')
            return

        # Otherwise: start new session
        self.createSession()

    def live(self):
        if self.step is not None:
            self.step += 1
        if not self.session:
            return
        if not self.session_timeout:
            return
        duration = time() - self.session_start
        if self.session_timeout <= duration:
            self.error("Timeout!")
            self.send('session_stop')

    def summarize(self):
        count = self.session_executed
        info = []
        if count:
            duration = self.session_total_duration
            info.append("%s session in %.1f second (%.1f ms per session)" %
                        (count, duration, duration * 1000 / count))
        duration = time() - self.project_start
        info.append("total %.1f second" % duration)
        info.append("aggresssivity: %s" % self.aggressivity)
        self.error("Project done: %s" % ", ".join(info))
        self.error("Total: %s success" % self.nb_success)