def load_install(self, _install_folder): _exp_path = os.path.expanduser(_install_folder) self.plugins_location = None # Does the config file exist? if os.path.exists(os.path.join(_exp_path, "config.json")): _config = JSONXPath(os.path.join(_exp_path, "config.json")) self.plugins_location = _config.get_path("broker/pluginsFolder", _default="plugins") self.install_location = _install_folder if self.plugins_location is None: # If there is no config file, try with the default location for the plugins folder self.plugins_location = os.path.join(_exp_path, "plugins") self.plugins = [] # Enumerate plugins if not os.path.exists(self.plugins_location): # TODO: Write warning to status pass else: # Loading _plugin_names = os.listdir(self.plugins_location) for _plugin_name in _plugin_names: _curr_plugin_folder = os.path.join(self.plugins_location, _plugin_name) # Only look att non-hidden and non system directories if os.path.isdir(_curr_plugin_folder) and _plugin_name[0:2] != "__" and \ _plugin_name[0] != ".": _definitions = JSONXPath(os.path.join(_curr_plugin_folder, "definitions.json")) _description = _definitions.get("plugins/" + _plugin_name + "/description","No description found in plugin definition") self.plugins.append({"name": _plugin_name, "description": _description})
def start_broker(): """ Starts the broker; Loads settings, connects to database, registers process and starts the web server. """ global process_id, database_access, address, web_socket_plugin, repository_parent_folder, \ web_config, schema_tools, namespaces, log_to_database_severity, plugins, settings process_id = str(ObjectId()) of.common.logging.callback = log_locally write_srvc_dbg("=====Starting broker=============================") try: _cfg_filename = resolve_config_path() settings = JSONXPath(_cfg_filename) except Exception as e: if os.name == "nt": write_to_log(_data="Error loading settings from " + _cfg_filename, _category=EC_SERVICE, _severity=SEV_FATAL) raise Exception("Error loading settings:" + str(e)) if os.name != "nt": x_logger = logging.FileHandler("/var/log/of.log") of.common.logging.severity = of.common.logging.severity_identifiers.index( settings.get("broker/logging/severityLevel", _default="warning")) log_to_database_severity = of.common.logging.severity_identifiers.index( settings.get("broker/logging/databaseLevel", _default="warning")) write_srvc_dbg("Loaded settings from " + _cfg_filename) # An address is completely neccessary. address = settings.get("broker/address", _default=None) if not address or address == "": write_to_log(_data="Broker cannot start, missing [broker] address setting in configuration file.", _category=EC_SERVICE, _severity=SEV_FATAL) raise Exception("Broker cannot start, missing address.") # TODO: Reorganize. It is likely that almost everything but external database credentials should be stored in the db PROD-105 # Initialize schema tools (of_uri_handler is later replaced by the general one) schema_tools = SchemaTools(_json_schema_folders=[os.path.join(script_dir, "../schemas/namespaces/")], _uri_handlers={"ref": of_uri_handler}) namespaces = CumulativeDict(_default={"schemas": []}) write_srvc_dbg("Load plugin data") # Find the plugin directory _plugins_folder = settings.get_path("broker/pluginsFolder", _default="plugins") # Load all plugin data plugins = CherryPyPlugins(_plugins_folder=_plugins_folder, _schema_tools=schema_tools, _namespaces=namespaces, _process_id=process_id, _no_package_name_override=settings.get("broker/packageNameOverride")) # Plugins may want to load settings or add globals plugins.call_hook("init_broker_scope", _broker_scope=globals(), _settings=settings) write_srvc_dbg("===Register signal handlers===") register_signals(stop_broker) plugins.call_hook("before_db_connect", _broker_scope=globals()) # Connect to the database _host = settings.get("broker/database/host", _default="127.0.0.1") _user = settings.get("broker/database/username", _default=None) _password = settings.get("broker/database/password", _default=None) if _user: write_srvc_dbg("===Connecting to remote MongoDB backend " + _host + "===") # http://api.mongodb.org/python/current/examples/authentication.html _client = MongoClient("mongodb://" + _user + ":" + _password + "@" + _host) else: write_srvc_dbg("===Connecting to local MongoDB backend===") _client = MongoClient() _database_name = settings.get("broker/database/databaseName", _default="optimalframework") write_srvc_dbg("Using database name :" + _database_name) _database = _client[_database_name] database_access = DatabaseAccess(_database=_database, _schema_tools=schema_tools) of.common.logging.callback = log_to_database database_access.save(store_process_system_document(_process_id=process_id, _name="Broker instance(" + address + ")"), _user=None, _allow_save_id=True) plugins.call_hook("after_db_connect", _broker_scope=globals()) # TODO: It is possible that one would like to initialize, or at least read the plugins *before* trying to connect to the database # Must have a valid CherryPy version if hasattr(cherrypy.engine, "subscribe"): # CherryPy >= 3.1 pass else: write_to_log(_data="This application requires CherryPy >= 3.1 or higher.", _category=EC_SERVICE, _severity=SEV_FATAL) raise Exception("Broker init: This application requires CherryPy >= 3.1 or higher.") # cherrypy.engine.on_stop_engine_list.append(_save_data) def ssl_path(): # Figure out the path to the ssl-certificates # TODO: Load from database instead. Or not? (PROD-19) return os.path.dirname(_cfg_filename) # Initialize CherryPy:s global configuration; note that this could be moved into a configuration file cherrypy.config.update({ "tools.encode.on": True, "tools.encode.encoding": "utf-8", "tools.decode.on": True, "tools.trailing_slash.on": True, "tools.staticdir.root": os.path.abspath(os.path.dirname(__file__)), "server.ssl_module": "builtin", # TODO: Remove this when this bug is fixed: # https://bitbucket.org/cherrypy/cherrypy/issue/1341/autoreloader-also-fails-if-six-is-present "engine.autoreload.on": False, 'server.socket_host': '0.0.0.0', "server.ssl_certificate": os.path.join(ssl_path(), "optimalframework_test_cert.pem"), "server.ssl_private_key": os.path.join(ssl_path(), "optimalframework_test_privkey.pem"), "error_page.default": error_message_default }) write_srvc_dbg("Starting CherryPy, ssl at " + os.path.join(ssl_path(), "optimalframework_test_privkey.pem")) web_config = { # The UI root "/": { "tools.staticdir.on": True, "tools.staticdir.dir": "ui", "tools.trailing_slash.on": True, "tools.staticdir.index": "index.html", }, # Note that the web socket handling is put under /socket. "/socket": { "tools.websocket.on": True, "tools.websocket.handler_cls": BrokerWebSocket } } global web_root cherrypy._global_conf_alias.update(web_config) web_socket_plugin = WebSocketPlugin(cherrypy.engine) web_socket_plugin.subscribe() cherrypy.tools.websocket = WebSocketTool() cherrypy.engine.signals.bus.signal_handler.handlers = {'SIGUSR1': cherrypy.engine.signals.bus.graceful} # Initialize the decorator-based authentication framework init_authentication(MongoDBAuthBackend(database_access)) # Initialize root UI web_root = CherryPyBroker(_process_id=process_id, _address=address, _database_access=database_access) # Initialize messaging of.common.messaging.websocket.monitor = Monitor(_handler=BrokerWebSocketHandler(process_id, _peers=web_root.peers, _database_access=database_access, _schema_tools=database_access.schema_tools, _address=address)) web_root.plugins = plugins # Generate the static content, initialisation plugins.call_hook("init_web", _broker_scope = globals()) _web_config_debug = "Broker configured. Starting web server. Web config:\n" for _curr_key, _curr_config in web_config.items(): if "tools.staticdir.dir" in _curr_config: _web_config_debug += "Path: " + _curr_key + " directory: " + _curr_config["tools.staticdir.dir"] else: _web_config_debug += "Path: " + _curr_key + " - no static dir" write_to_log(_web_config_debug, _category=EC_SERVICE, _severity=SEV_INFO) plugins.call_hook("pre_webserver_start", web_config=web_config, globals=globals()) cherrypy.quickstart(web_root, "/", web_config)
def start_agent(_cfg_filename = None): """ Starts the agent; Loads settings, connects to database, registers process and starts the web server. """ global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \ _username, _password, _peers, _log_to_database_severity, _verify_SSL _process_id = str(ObjectId()) of.common.logging.callback = log_locally _terminated = False # Handle multiprocessing on windows freeze_support() write_srvc_dbg("=====start_agent===============================") try: if _cfg_filename is None: _cfg_filename = resolve_config_path() _settings = JSONXPath(_cfg_filename) except Exception as e: write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL, _process_id=_process_id) return of.common.logging.severity = of.common.logging.severity_identifiers.index( _settings.get("agent/logging/severityLevel", _default="warning")) _log_to_database_severity = of.common.logging.severity_identifiers.index( _settings.get("agent/logging/brokerLevel", _default="warning")) write_srvc_dbg("===register signal handlers===") register_signals(stop_agent) # An _address is completely necessary. _address = _settings.get("agent/address", _default=None) if not _address or _address == "": raise Exception(write_to_log( "Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.", _category=EC_SERVICE, _severity=SEV_FATAL)) # An _address is completely necessary. _verify_SSL = _settings.get("agent/verifySSL", _default=True) # Gather credentials _broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080") _username = _settings.get("agent/username") if not _username: raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL)) _password = _settings.get("agent/password") if not _password: raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL)) _retries = int(_settings.get("agent/connectionRetries", 5)) # Register at the broker if not register_agent(_retries): raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str( _retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) of.common.logging.callback = log_to_database _repository_base_folder = _settings.get_path("agent/repositoryFolder", _default=os.path.join(os.path.dirname(__file__), "repositories")) write_srvc_dbg("Load schema tool") try: # Initiate a schema tools instance for validation other purposes. _schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(), os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces")) ], _uri_handlers={"ref": None}) except Exception as e: raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) return write_srvc_dbg("Load schema tool done") try: write_srvc_dbg("Initializing monitors") # Init the monitor for incoming messages _message_monitor = Monitor( _handler=AgentWebSocketHandler(_process_id=_process_id, _peers=_peers, _schema_tools=_schema_tools, _address=_address, _broker_address="broker")) # The manager for the process queue _process_queue_manager = multiprocessing.Manager() # Init the monitor for the worker queue _worker_monitor = Monitor( _handler=WorkerSupervisor(_process_id=_process_id, _message_monitor=_message_monitor, _repo_base_folder=_repository_base_folder, _severity=of.common.logging.severity), _queue=_process_queue_manager.Queue()) # Init the monitor for the agent queue _control_monitor = Monitor( _handler=ControlHandler(_process_id=_process_id, _message_monitor=_message_monitor, _worker_monitor=_worker_monitor, _stop_agent=stop_agent )) # The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33) of.common.messaging.websocket.monitor = _message_monitor write_srvc_dbg("Initializing monitors done") except Exception as e: raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e), _category=EC_SERVICE, _severity=SEV_FATAL)) os._exit(1) # Try to connect to websocket, quit on failure if not connect_to_websocket(): os._exit(1) write_srvc_dbg("Register agent system process") _control_monitor.handler.message_monitor.queue.put( [None, store_process_system_document(_process_id=_process_id, _name="Agent instance(" + _address + ")")]) write_srvc_dbg("Log agent system state") _control_monitor.handler.message_monitor.queue.put([None, log_process_state_message(_changed_by=zero_object_id, _state="running", _process_id=_process_id, _reason="Agent starting up at " + _address)]) # Security check to remind broker if it is unsecured if not _verify_SSL: try: call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True) except SSLError as e: write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n" "This is a security risk, and and important thing to _address.", _category=EC_NOTIFICATION, _severity=SEV_WARNING) except Exception as e: write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e), _category=EC_NOTIFICATION, _severity=SEV_ERROR) write_srvc_dbg("Agent up and running.") while not _terminated: time.sleep(0.1) write_srvc_dbg("Exiting main thread")