def process_workspaces(mappers_manager, plugin_manager, query, disable_polling): report_managers = [] controllers = [] for workspace in query.all(): pending_actions = Queue() plugin_controller = PluginController( 'PluginController', plugin_manager, mappers_manager, pending_actions ) mappers_manager.createMappers(workspace.name) controller = ModelController(mappers_manager, pending_actions) workspace_manager = WorkspaceManager(mappers_manager) setUpAPIs(controller, workspace_manager, hostname=None, port=None) controller.start() controllers.append(controller) report_manager = ReportManager( 0.1, workspace.name, plugin_controller, polling=not disable_polling ) report_managers.append(report_manager) report_manager.start() for report_manager in report_managers: report_manager.stop() for controller in controllers: controller.stop()
def test_controller_stop_when_is_not_processing(): mappers_manager = MapperManager() pending_actions = Queue() controller = ModelController(mappers_manager, pending_actions) assert controller.processing is False assert controller._stop is False controller.start() assert controller.isAlive() controller.stop() assert controller._stop is True controller.join() assert controller.isAlive() is False
def test_controller_plugin_start_action_updates_internal_state(): mappers_manager = MapperManager() pending_actions = Queue() controller = ModelController(mappers_manager, pending_actions) controller.start() controller.add_action((Modelactions.PLUGINSTART, "test", None)) time.sleep(1) assert controller.active_plugins_count == 1 assert controller.processing controller.add_action((Modelactions.PLUGINEND, "test", None)) time.sleep(1) assert controller.active_plugins_count == 0 assert controller.processing is False controller.stop() controller.join() assert controller.isAlive() is False
def test_controller_cant_be_stopped_when_is_processing(): """ If someone tells the controller to stop and it is processing then it will stop when the processing finishes """ mappers_manager = MapperManager() pending_actions = Queue() controller = ModelController(mappers_manager, pending_actions) assert controller.processing is False assert controller._stop is False controller.start() controller.processing = True controller.active_plugins_count = 1 assert controller.isAlive() controller.stop() assert controller._stop assert controller.processing controller.join(timeout=2) assert controller.isAlive() controller.processing = False controller.join() assert controller.isAlive() is False
class MainApplication(object): def __init__(self, args): self._original_excepthook = sys.excepthook self.args = args self._mappers_manager = MapperManager() self._model_controller = ModelController(self._mappers_manager) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) self._workspace_manager = WorkspaceManager(self._mappers_manager) # Create a PluginController and send this to UI selected. self._plugin_controller = PluginController('PluginController', self._plugin_manager, self._mappers_manager) if self.args.cli: self.app = CliApp(self._workspace_manager, self._plugin_controller) CONF.setMergeStrategy("new") else: self.app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, self._plugin_controller, self.args.gui) self.timer = TimerClass() self.timer.start() def on_connection_lost(self): """All it does is send a notification to the notification center""" model.guiapi.notification_center.CouchDBConnectionProblem() def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def start(self): try: signal.signal(signal.SIGINT, self.ctrlC) model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") if not self.args.workspace: workspace = CONF.getLastWorkspace() self.args.workspace = workspace model.api.setUpAPIs(self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort()) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs(self._plugin_controller, self._model_controller, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort()) model.api.devlog("Faraday ready...") exit_code = self.app.run(self.args) except Exception as exception: print "There was an error while starting Faraday:" print "*" * 3, print exception, # instead of traceback.print_exc() print "*" * 3 exit_code = -1 finally: return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.log("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._model_controller.stop() if self._model_controller.isAlive(): # runs only if thread has started, i.e. self._model_controller.start() is run first self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.app.quit()
class MainApplication(object): """ """ def __init__(self, args): self._original_excepthook = sys.excepthook self._configuration = CONF self._security_manager = SecurityManager() self._mappers_manager = MapperManager() self._changes_controller = ChangeController() self._db_manager = DbManager() self._model_controller = ModelController(self._security_manager, self._mappers_manager) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins"), self._mappers_manager) self._reports_manager = ReportManager( 10, self._plugin_manager.createController("ReportManager")) self._workspace_manager = WorkspaceManager(self._db_manager, self._mappers_manager, self._changes_controller, self._reports_manager) self.gui_app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, args.gui) self.gui_app.setSplashImage( os.path.join(CONF.getImagePath(), "splash2.png")) def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def disableLogin(self): CONF.setAuth(sys.disablelogin) def start(self): try: self.gui_app.startSplashScreen() self.gui_app.splashMessage("Starting Faraday") signal.signal(signal.SIGINT, self.ctrlC) logged = True while True: username, password = "******", "password" if username is None and password is None: break result = self._security_manager.authenticateUser( username, password) if result == codes.successfulLogin: logged = True break if logged: model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") # We need to create the last used workspace (or the default # workspace) before we start the model controller and the # report manager last_workspace = CONF.getLastWorkspace() if not self._workspace_manager.workspaceExists(last_workspace): getLogger(self).info( "Your last workspace (" + last_workspace + ") wasn't accessible, check configuration...") self._workspace_manager.openDefaultWorkspace() #self._workspace_manager.createWorkspace(last_workspace, 'default workspace, probably created already in couchb') else: self._workspace_manager.openWorkspace(last_workspace) model.api.setUpAPIs(self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort()) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs(self._plugin_manager, self._model_controller, self._mappers_manager, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort()) # Start report manager here getLogger(self).debug("Starting Reports Manager Thread") self._reports_manager.startWatch() model.api.devlog("Faraday ready...") model.api.__current_logged_user = username self.gui_app.splashMessage("Loading workspace... Please wait.") self.gui_app.loadWorkspaces() self.gui_app.stopSplashScreen() except Exception: print "There was an error while starting Faraday" print "-" * 50 traceback.print_exc() print "-" * 50 self.__exit(-1) if logged: exit_code = self.gui_app.run([]) #exit_code = self.app.exec_loop() else: exit_code = -1 return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.devlog("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._reports_manager.stop() self._changes_controller.stop() self._model_controller.stop() self._model_controller.join() self.gui_app.quit() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.gui_app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.__exit(exit_code=0) def getWorkspaceManager(self): return self._workspace_manager
class RawReportProcessor(Thread): def __init__(self): super(RawReportProcessor, self).__init__() from faraday import setupPlugins setupPlugins() self.pending_actions = Queue() try: plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins")) except AttributeError: get_logger().warning( "Upload reports in WEB-UI not configurated, run Faraday client and try again...") self._stop = True return mappers_manager = MapperManager() self.model_controller = ModelController(mappers_manager, self.pending_actions) self.model_controller.start() self.end_event = Event() plugin_controller = PluginController( 'PluginController', plugin_manager, mappers_manager, self.pending_actions, self.end_event) self.processor = ReportProcessor(plugin_controller, None) self._stop = False def stop(self): self.model_controller.stop() self._stop = True def run(self): logger.info('Tool report processor started') while not self._stop: try: workspace, file_path, cookie = UPLOAD_REPORTS_QUEUE.get(False, timeout=0.1) get_logger().info('Processing raw report {0}'.format(file_path)) # Cookie of user, used to create objects in server with the right owner. server.FARADAY_UPLOAD_REPORTS_WEB_COOKIE = cookie server.FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL = "http://{0}:{1}".format( FaradayServerConfig.faraday_server.bind_address, FaradayServerConfig.faraday_server.port) self.processor.ws_name = workspace command_id = self.processor.processReport(file_path) UPLOAD_REPORTS_CMD_QUEUE.put(command_id) if not command_id: continue self.end_event.wait() get_logger().info('Report processing of report {0} finished'.format(file_path)) self.end_event.clear() except Empty: time.sleep(0.1) except KeyboardInterrupt as ex: get_logger().info('Keyboard interrupt, stopping report processing thread') self.stop() except Exception as ex: get_logger().exception(ex) continue
class MainApplication(object): def __init__(self, args): self._original_excepthook = sys.excepthook self.args = args self._mappers_manager = MapperManager() self._model_controller = ModelController(self._mappers_manager) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) self._workspace_manager = WorkspaceManager( self._mappers_manager) # Create a PluginController and send this to UI selected. self._plugin_controller = PluginController( 'PluginController', self._plugin_manager, self._mappers_manager ) if self.args.cli: self.app = CliApp(self._workspace_manager, self._plugin_controller) CONF.setMergeStrategy("new") else: self.app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, self._plugin_controller, self.args.gui) self.timer = TimerClass() self.timer.start() def on_connection_lost(self): """All it does is send a notification to the notification center""" model.guiapi.notification_center.CouchDBConnectionProblem() def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def start(self): try: signal.signal(signal.SIGINT, self.ctrlC) model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") if not self.args.workspace: workspace = CONF.getLastWorkspace() self.args.workspace = workspace model.api.setUpAPIs( self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort()) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs( self._plugin_controller, self._model_controller, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort() ) model.api.devlog("Faraday ready...") exit_code = self.app.run(self.args) except Exception: print "There was an error while starting Faraday" print "-" * 50 traceback.print_exc() print "-" * 50 exit_code = -1 finally: return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.log("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._model_controller.stop() self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.app.quit()
class MainApplication(object): """ """ def __init__(self, args): self._original_excepthook = sys.excepthook self._configuration = CONF self._security_manager = SecurityManager() self._mappers_manager = MapperManager() self._changes_controller = ChangeController() self._db_manager = DbManager() self._model_controller = ModelController(self._security_manager, self._mappers_manager) self._plugin_manager = PluginManager(os.path.join(CONF.getConfigPath(), "plugins"), self._mappers_manager) self._reports_manager = ReportManager(10, self._plugin_manager.createController("ReportManager")) self._workspace_manager = WorkspaceManager( self._db_manager, self._mappers_manager, self._changes_controller, self._reports_manager ) self.gui_app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, args.gui) self.gui_app.setSplashImage(os.path.join(CONF.getImagePath(), "splash2.png")) self.timer = TimerClass() self.timer.start() def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def disableLogin(self): CONF.setAuth(sys.disablelogin) def start(self): try: self.gui_app.startSplashScreen() self.gui_app.splashMessage("Starting Faraday") signal.signal(signal.SIGINT, self.ctrlC) logged = True while True: username, password = "******", "password" if username is None and password is None: break result = self._security_manager.authenticateUser(username, password) if result == codes.successfulLogin: logged = True break if logged: model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") # We need to create the last used workspace (or the default # workspace) before we start the model controller and the # report manager last_workspace = CONF.getLastWorkspace() try: if not self._workspace_manager.workspaceExists(last_workspace): getLogger(self).info( "Your last workspace (" + str(last_workspace) + ") wasn't accessible, check configuration..." ) self._workspace_manager.openDefaultWorkspace() # self._workspace_manager.createWorkspace(last_workspace, 'default workspace, probably created already in couchb') else: self._workspace_manager.openWorkspace(last_workspace) except restkit.errors.Unauthorized: print "You are trying to enter CouchDB with authentication" print "Add your credentials to your user configuration file in $HOME/.faraday/config/user.xml" print "For example: <couch_uri>http://john:[email protected]:5984</couch_uri>" return model.api.setUpAPIs( self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort() ) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs( self._plugin_manager, self._model_controller, self._mappers_manager, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort(), ) # Start report manager here getLogger(self).debug("Starting Reports Manager Thread") self._reports_manager.startWatch() model.api.devlog("Faraday ready...") model.api.__current_logged_user = username self.gui_app.splashMessage("Loading workspace... Please wait.") self.gui_app.loadWorkspaces() self.gui_app.stopSplashScreen() except Exception: print "There was an error while starting Faraday" print "-" * 50 traceback.print_exc() print "-" * 50 self.__exit(-1) if logged: exit_code = self.gui_app.run([]) # exit_code = self.app.exec_loop() else: exit_code = -1 return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.devlog("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._reports_manager.stop() self._changes_controller.stop() self._model_controller.stop() self._model_controller.join() self.gui_app.quit() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.gui_app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.__exit(exit_code=0) def getWorkspaceManager(self): return self._workspace_manager
class RawReportProcessor(Thread): def __init__(self): super(RawReportProcessor, self).__init__() from faraday import setupPlugins setupPlugins() self.pending_actions = Queue() try: plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins")) except AttributeError: get_logger().warning( "Upload reports in WEB-UI not configurated, run Faraday client and try again..." ) self._stop = True return mappers_manager = MapperManager() self.model_controller = ModelController(mappers_manager, self.pending_actions) self.model_controller.start() self.end_event = Event() plugin_controller = PluginController('PluginController', plugin_manager, mappers_manager, self.pending_actions, self.end_event) self.processor = ReportProcessor(plugin_controller, None) self._stop = False def stop(self): self.model_controller.stop() self._stop = True def run(self): while not self._stop: try: workspace, file_path, cookie = UPLOAD_REPORTS_QUEUE.get( False, timeout=0.1) get_logger().info( 'Processing raw report {0}'.format(file_path)) # Cookie of user, used to create objects in server with the right owner. server.FARADAY_UPLOAD_REPORTS_WEB_COOKIE = cookie server.FARADAY_UPLOAD_REPORTS_OVERWRITE_SERVER_URL = "http://{0}:{1}".format( FaradayServerConfig.faraday_server.bind_address, FaradayServerConfig.faraday_server.port) self.processor.ws_name = workspace command_id = self.processor.processReport(file_path) UPLOAD_REPORTS_CMD_QUEUE.put(command_id) if not command_id: continue self.end_event.wait() get_logger().info( 'Report processing of report {0} finished'.format( file_path)) self.end_event.clear() except Empty: time.sleep(0.1) except KeyboardInterrupt as ex: get_logger().info( 'Keyboard interrupt, stopping report processing thread') self.stop() except Exception as ex: get_logger().exception(ex) continue
class MainApplication(object): def __init__(self, args): self._original_excepthook = sys.excepthook self.args = args logger = getLogger(self) if args.creds_file: try: with open(args.creds_file, 'r') as fp: creds = json.loads(fp.read()) username = creds.get('username') password = creds.get('password') session_cookie = login_user(CONF.getServerURI(), username, password) if session_cookie: logger.info('Login successful') CONF.setDBUser(username) CONF.setDBSessionCookies(session_cookie) else: logger.error('Login failed') except (IOError, ValueError): logger.error("Credentials file couldn't be loaded") self._mappers_manager = MapperManager() pending_actions = Queue() self._model_controller = ModelController(self._mappers_manager, pending_actions) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins"), pending_actions=pending_actions, ) self._workspace_manager = WorkspaceManager( self._mappers_manager) # Create a PluginController and send this to UI selected. self._plugin_controller = PluginController( 'PluginController', self._plugin_manager, self._mappers_manager, pending_actions ) if self.args.cli: self.app = CliApp(self._workspace_manager, self._plugin_controller) if self.args.keep_old: CONF.setMergeStrategy("old") else: CONF.setMergeStrategy("new") else: self.app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, self._plugin_controller, self.args.gui) self.timer = TimerClass() self.timer.start() def on_connection_lost(self): """All it does is send a notification to the notification center""" model.guiapi.notification_center.DBConnectionProblem() def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def start(self): try: signal.signal(signal.SIGINT, self.ctrlC) model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") if not self.args.workspace: workspace = CONF.getLastWorkspace() self.args.workspace = workspace model.api.setUpAPIs( self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort()) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs( self._plugin_controller, self._model_controller, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort() ) model.api.devlog("Faraday ready...") exit_code = self.app.run(self.args) except Exception as exception: print("There was an error while starting Faraday:") print("*" * 3) print(exception) # instead of traceback.print_exc() print("*" * 3) exit_code = -1 finally: return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.log("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._model_controller.stop() if self._model_controller.isAlive(): # runs only if thread has started, i.e. self._model_controller.start() is run first self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.app.quit()
class MainApplication(object): def __init__(self, args): self._original_excepthook = sys.excepthook self.args = args logger = getLogger(self) if args.creds_file: try: with open(args.creds_file, 'r') as fp: creds = json.loads(fp.read()) username = creds.get('username') password = creds.get('password') session_cookie = login_user(CONF.getServerURI(), username, password) if session_cookie: logger.info('Login successful') CONF.setDBUser(username) CONF.setDBSessionCookies(session_cookie) else: logger.error('Login failed') except (IOError, ValueError): logger.error("Credentials file couldn't be loaded") self._mappers_manager = MapperManager() pending_actions = Queue() self._model_controller = ModelController(self._mappers_manager, pending_actions) self._plugin_manager = PluginManager( os.path.join(CONF.getConfigPath(), "plugins"), pending_actions=pending_actions, ) self._workspace_manager = WorkspaceManager(self._mappers_manager) # Create a PluginController and send this to UI selected. self._plugin_controller = PluginController('PluginController', self._plugin_manager, self._mappers_manager, pending_actions) if self.args.cli: self.app = CliApp(self._workspace_manager, self._plugin_controller) if self.args.keep_old: CONF.setMergeStrategy("old") else: CONF.setMergeStrategy("new") else: self.app = UiFactory.create(self._model_controller, self._plugin_manager, self._workspace_manager, self._plugin_controller, self.args.gui) self.timer = TimerClass() self.timer.start() def on_connection_lost(self): """All it does is send a notification to the notification center""" model.guiapi.notification_center.DBConnectionProblem() def enableExceptHook(self): sys.excepthook = exception_handler installThreadExcepthook() def start(self): try: signal.signal(signal.SIGINT, self.ctrlC) model.api.devlog("Starting application...") model.api.devlog("Setting up remote API's...") if not self.args.workspace: workspace = CONF.getLastWorkspace() self.args.workspace = workspace model.api.setUpAPIs(self._model_controller, self._workspace_manager, CONF.getApiConInfoHost(), CONF.getApiConInfoPort()) model.guiapi.setUpGUIAPIs(self._model_controller) model.api.devlog("Starting model controller daemon...") self._model_controller.start() model.api.startAPIServer() restapi.startAPIs(self._plugin_controller, self._model_controller, CONF.getApiConInfoHost(), CONF.getApiRestfulConInfoPort()) model.api.devlog("Faraday ready...") exit_code = self.app.run(self.args) except Exception as exception: print("There was an error while starting Faraday:") print("*" * 3) print(exception) # instead of traceback.print_exc() print("*" * 3) exit_code = -1 finally: return self.__exit(exit_code) def __exit(self, exit_code=0): """ Exits the application with the provided code. It also waits until all app threads end. """ model.api.log("Closing Faraday...") model.api.devlog("stopping model controller thread...") model.api.stopAPIServer() restapi.stopServer() self._model_controller.stop() if self._model_controller.isAlive(): # runs only if thread has started, i.e. self._model_controller.start() is run first self._model_controller.join() self.timer.stop() model.api.devlog("Waiting for controller threads to end...") return exit_code def quit(self): """ Redefined quit handler to nicely end up things """ self.app.quit() def ctrlC(self, signal, frame): getLogger(self).info("Exiting...") self.app.quit()