def load_install(self, _install_folder):
_exp_path = os.path.expanduser(_install_folder)
self.plugins_location = None
# Does the config file exist?
if os.path.exists(os.path.join(_exp_path, "config.json")):
_config = JSONXPath(os.path.join(_exp_path, "config.json"))
self.plugins_location = _config.get_path("broker/pluginsFolder", _default="plugins")
self.install_location = _install_folder
if self.plugins_location is None:
# If there is no config file, try with the default location for the plugins folder
self.plugins_location = os.path.join(_exp_path, "plugins")
self.plugins = []
# Enumerate plugins
if not os.path.exists(self.plugins_location):
# TODO: Write warning to status
pass
else:
# Loading
_plugin_names = os.listdir(self.plugins_location)
for _plugin_name in _plugin_names:
_curr_plugin_folder = os.path.join(self.plugins_location, _plugin_name)
# Only look att non-hidden and non system directories
if os.path.isdir(_curr_plugin_folder) and _plugin_name[0:2] != "__" and \
_plugin_name[0] != ".":
_definitions = JSONXPath(os.path.join(_curr_plugin_folder, "definitions.json"))
_description = _definitions.get("plugins/" + _plugin_name + "/description","No description found in plugin definition")
self.plugins.append({"name": _plugin_name, "description": _description})
def start_broker():
"""
Starts the broker; Loads settings, connects to database, registers process and starts the web server.
"""
global process_id, database_access, address, web_socket_plugin, repository_parent_folder, \
web_config, schema_tools, namespaces, log_to_database_severity, plugins, settings
process_id = str(ObjectId())
of.common.logging.callback = log_locally
write_srvc_dbg("=====Starting broker=============================")
try:
_cfg_filename = resolve_config_path()
settings = JSONXPath(_cfg_filename)
except Exception as e:
if os.name == "nt":
write_to_log(_data="Error loading settings from " + _cfg_filename,
_category=EC_SERVICE, _severity=SEV_FATAL)
raise Exception("Error loading settings:" + str(e))
if os.name != "nt":
x_logger = logging.FileHandler("/var/log/of.log")
of.common.logging.severity = of.common.logging.severity_identifiers.index(
settings.get("broker/logging/severityLevel", _default="warning"))
log_to_database_severity = of.common.logging.severity_identifiers.index(
settings.get("broker/logging/databaseLevel", _default="warning"))
write_srvc_dbg("Loaded settings from " + _cfg_filename)
# An address is completely neccessary.
address = settings.get("broker/address", _default=None)
if not address or address == "":
write_to_log(_data="Broker cannot start, missing [broker] address setting in configuration file.",
_category=EC_SERVICE, _severity=SEV_FATAL)
raise Exception("Broker cannot start, missing address.")
# TODO: Reorganize. It is likely that almost everything but external database credentials should be stored in the db PROD-105
# Initialize schema tools (of_uri_handler is later replaced by the general one)
schema_tools = SchemaTools(_json_schema_folders=[os.path.join(script_dir, "../schemas/namespaces/")],
_uri_handlers={"ref": of_uri_handler})
namespaces = CumulativeDict(_default={"schemas": []})
write_srvc_dbg("Load plugin data")
# Find the plugin directory
_plugins_folder = settings.get_path("broker/pluginsFolder", _default="plugins")
# Load all plugin data
plugins = CherryPyPlugins(_plugins_folder=_plugins_folder, _schema_tools=schema_tools, _namespaces=namespaces,
_process_id=process_id,
_no_package_name_override=settings.get("broker/packageNameOverride"))
# Plugins may want to load settings or add globals
plugins.call_hook("init_broker_scope", _broker_scope=globals(), _settings=settings)
write_srvc_dbg("===Register signal handlers===")
register_signals(stop_broker)
plugins.call_hook("before_db_connect", _broker_scope=globals())
# Connect to the database
_host = settings.get("broker/database/host", _default="127.0.0.1")
_user = settings.get("broker/database/username", _default=None)
_password = settings.get("broker/database/password", _default=None)
if _user:
write_srvc_dbg("===Connecting to remote MongoDB backend " + _host + "===")
# http://api.mongodb.org/python/current/examples/authentication.html
_client = MongoClient("mongodb://" + _user + ":" + _password + "@" + _host)
else:
write_srvc_dbg("===Connecting to local MongoDB backend===")
_client = MongoClient()
_database_name = settings.get("broker/database/databaseName", _default="optimalframework")
write_srvc_dbg("Using database name :" + _database_name)
_database = _client[_database_name]
database_access = DatabaseAccess(_database=_database, _schema_tools=schema_tools)
of.common.logging.callback = log_to_database
database_access.save(store_process_system_document(_process_id=process_id,
_name="Broker instance(" + address + ")"),
_user=None,
_allow_save_id=True)
plugins.call_hook("after_db_connect", _broker_scope=globals())
# TODO: It is possible that one would like to initialize, or at least read the plugins *before* trying to connect to the database
# Must have a valid CherryPy version
if hasattr(cherrypy.engine, "subscribe"): # CherryPy >= 3.1
pass
else:
write_to_log(_data="This application requires CherryPy >= 3.1 or higher.", _category=EC_SERVICE,
_severity=SEV_FATAL)
raise Exception("Broker init: This application requires CherryPy >= 3.1 or higher.")
# cherrypy.engine.on_stop_engine_list.append(_save_data)
def ssl_path():
# Figure out the path to the ssl-certificates
# TODO: Load from database instead. Or not? (PROD-19)
return os.path.dirname(_cfg_filename)
# Initialize CherryPy:s global configuration; note that this could be moved into a configuration file
cherrypy.config.update({
"tools.encode.on": True,
"tools.encode.encoding": "utf-8",
"tools.decode.on": True,
"tools.trailing_slash.on": True,
"tools.staticdir.root": os.path.abspath(os.path.dirname(__file__)),
"server.ssl_module": "builtin",
# TODO: Remove this when this bug is fixed:
# https://bitbucket.org/cherrypy/cherrypy/issue/1341/autoreloader-also-fails-if-six-is-present
"engine.autoreload.on": False,
'server.socket_host': '0.0.0.0',
"server.ssl_certificate": os.path.join(ssl_path(), "optimalframework_test_cert.pem"),
"server.ssl_private_key": os.path.join(ssl_path(), "optimalframework_test_privkey.pem"),
"error_page.default": error_message_default
})
write_srvc_dbg("Starting CherryPy, ssl at " + os.path.join(ssl_path(), "optimalframework_test_privkey.pem"))
web_config = {
# The UI root
"/": {
"tools.staticdir.on": True,
"tools.staticdir.dir": "ui",
"tools.trailing_slash.on": True,
"tools.staticdir.index": "index.html",
},
# Note that the web socket handling is put under /socket.
"/socket": {
"tools.websocket.on": True,
"tools.websocket.handler_cls": BrokerWebSocket
}
}
global web_root
cherrypy._global_conf_alias.update(web_config)
web_socket_plugin = WebSocketPlugin(cherrypy.engine)
web_socket_plugin.subscribe()
cherrypy.tools.websocket = WebSocketTool()
cherrypy.engine.signals.bus.signal_handler.handlers = {'SIGUSR1': cherrypy.engine.signals.bus.graceful}
# Initialize the decorator-based authentication framework
init_authentication(MongoDBAuthBackend(database_access))
# Initialize root UI
web_root = CherryPyBroker(_process_id=process_id, _address=address, _database_access=database_access)
# Initialize messaging
of.common.messaging.websocket.monitor = Monitor(_handler=BrokerWebSocketHandler(process_id, _peers=web_root.peers,
_database_access=database_access,
_schema_tools=database_access.schema_tools,
_address=address))
web_root.plugins = plugins
# Generate the static content, initialisation
plugins.call_hook("init_web", _broker_scope = globals())
_web_config_debug = "Broker configured. Starting web server. Web config:\n"
for _curr_key, _curr_config in web_config.items():
if "tools.staticdir.dir" in _curr_config:
_web_config_debug += "Path: " + _curr_key + " directory: " + _curr_config["tools.staticdir.dir"]
else:
_web_config_debug += "Path: " + _curr_key + " - no static dir"
write_to_log(_web_config_debug, _category=EC_SERVICE, _severity=SEV_INFO)
plugins.call_hook("pre_webserver_start", web_config=web_config, globals=globals())
cherrypy.quickstart(web_root, "/", web_config)
def start_agent(_cfg_filename = None):
"""
Starts the agent; Loads settings, connects to database, registers process and starts the web server.
"""
global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \
_username, _password, _peers, _log_to_database_severity, _verify_SSL
_process_id = str(ObjectId())
of.common.logging.callback = log_locally
_terminated = False
# Handle multiprocessing on windows
freeze_support()
write_srvc_dbg("=====start_agent===============================")
try:
if _cfg_filename is None:
_cfg_filename = resolve_config_path()
_settings = JSONXPath(_cfg_filename)
except Exception as e:
write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL,
_process_id=_process_id)
return
of.common.logging.severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/severityLevel", _default="warning"))
_log_to_database_severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/brokerLevel", _default="warning"))
write_srvc_dbg("===register signal handlers===")
register_signals(stop_agent)
# An _address is completely necessary.
_address = _settings.get("agent/address", _default=None)
if not _address or _address == "":
raise Exception(write_to_log(
"Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.",
_category=EC_SERVICE, _severity=SEV_FATAL))
# An _address is completely necessary.
_verify_SSL = _settings.get("agent/verifySSL", _default=True)
# Gather credentials
_broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080")
_username = _settings.get("agent/username")
if not _username:
raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_password = _settings.get("agent/password")
if not _password:
raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_retries = int(_settings.get("agent/connectionRetries", 5))
# Register at the broker
if not register_agent(_retries):
raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str(
_retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
of.common.logging.callback = log_to_database
_repository_base_folder = _settings.get_path("agent/repositoryFolder",
_default=os.path.join(os.path.dirname(__file__), "repositories"))
write_srvc_dbg("Load schema tool")
try:
# Initiate a schema tools instance for validation other purposes.
_schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(),
os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces"))
],
_uri_handlers={"ref": None})
except Exception as e:
raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
return
write_srvc_dbg("Load schema tool done")
try:
write_srvc_dbg("Initializing monitors")
# Init the monitor for incoming messages
_message_monitor = Monitor(
_handler=AgentWebSocketHandler(_process_id=_process_id,
_peers=_peers,
_schema_tools=_schema_tools,
_address=_address,
_broker_address="broker"))
# The manager for the process queue
_process_queue_manager = multiprocessing.Manager()
# Init the monitor for the worker queue
_worker_monitor = Monitor(
_handler=WorkerSupervisor(_process_id=_process_id,
_message_monitor=_message_monitor,
_repo_base_folder=_repository_base_folder,
_severity=of.common.logging.severity),
_queue=_process_queue_manager.Queue())
# Init the monitor for the agent queue
_control_monitor = Monitor(
_handler=ControlHandler(_process_id=_process_id,
_message_monitor=_message_monitor,
_worker_monitor=_worker_monitor,
_stop_agent=stop_agent
))
# The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33)
of.common.messaging.websocket.monitor = _message_monitor
write_srvc_dbg("Initializing monitors done")
except Exception as e:
raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
# Try to connect to websocket, quit on failure
if not connect_to_websocket():
os._exit(1)
write_srvc_dbg("Register agent system process")
_control_monitor.handler.message_monitor.queue.put(
[None, store_process_system_document(_process_id=_process_id,
_name="Agent instance(" + _address + ")")])
write_srvc_dbg("Log agent system state")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="running",
_process_id=_process_id,
_reason="Agent starting up at " +
_address)])
# Security check to remind broker if it is unsecured
if not _verify_SSL:
try:
call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True)
except SSLError as e:
write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n"
"This is a security risk, and and important thing to _address.",
_category=EC_NOTIFICATION, _severity=SEV_WARNING)
except Exception as e:
write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
write_srvc_dbg("Agent up and running.")
while not _terminated:
time.sleep(0.1)
write_srvc_dbg("Exiting main thread")
