def __init__(self, host="localhost", port=8080, config_overrides={}):
self.host = host
self.port = port
self.config_manager = ConfigManager(config_overrides)
self.config = self.config_manager.get_config()
self.api = None
def __init__(self, cli_overrides):
history_file = FileHistory(".msa_cli_history")
self.prompt_session = PromptSession(
lexer=PygmentsLexer(Python3Lexer), history=history_file
)
self.config_manager = ConfigManager("msa_config.json", cli_overrides)
self.config = self.config_manager.get_config()
self.loop = asyncio.get_event_loop()
self.api = None
self.quit = False
self.exit_code = 0
self.locals = {}
self.globals = {}
self.func_locals = {}
self.buffer = ""
self.indent_level = 0
self.indent_size = 4
self.recording = False
self.record_buffer_name = ""
self.record_buffer = ""
def test_get_config_success(self):
fake_config_contents = {
"agent": {"name": "Masa-chan", "user_title": "Onee-chan"},
"plugin_modules": [],
"module_config": {},
"logging": {
"global_log_level": "debug",
"log_file_location": "msa.log",
"truncate_log_file": False,
"granular_log_levels": [],
},
}
fake_config_contents_str = json.dumps(fake_config_contents)
cli_overrides = {}
with patch(
"builtins.open", mock_open(read_data=fake_config_contents_str)
) as mock_file:
config_manager = ConfigManager("fake_file.json", cli_overrides)
config = config_manager.get_config()
assert config["agent"] == fake_config_contents["agent"]
assert config["plugin_modules"] == fake_config_contents["plugin_modules"]
assert config["module_config"] == fake_config_contents["module_config"]
assert config["logging"]["global_log_level"] == "DEBUG"
assert (
config["logging"]["log_file_location"]
== fake_config_contents["logging"]["log_file_location"]
)
assert (
config["logging"]["log_file_location"]
== fake_config_contents["logging"]["log_file_location"]
)
assert (
config["logging"]["truncate_log_file"]
== fake_config_contents["logging"]["truncate_log_file"]
)
assert (
config["logging"]["granular_log_levels"]
== fake_config_contents["logging"]["granular_log_levels"]
)
class MsaRestClientLoader:
def __init__(self, host="localhost", port=8080, config_overrides={}):
self.host = host
self.port = port
self.config_manager = ConfigManager(config_overrides)
self.config = self.config_manager.get_config()
self.api = None
def load(self):
if self.api is not None:
return self.api
self.api = get_api(
ApiContext.rest,
self.config["plugin_modules"],
host=self.host,
port=self.port,
)
run_async(self.api.client.connect())
return self.api
def init(self, loop, cli_config, route_adapter):
"""Initializes the supervisor.
Parameters
----------
loop : Asynio Event Loop
An asyncio event loop the supervisor should use.
cli_config: Dict
A dictionary containing configuration options derived from the command line interface.
route_adapter: ** fix docstrings **
"""
if not os.environ.get("TEST"):
self.loop = loop
self.event_bus = EventBus(self.loop)
self.event_queue = asyncio.Queue(self.loop)
# block getting a loop if we are running unit tests
# helps suppress a warning.
# ### PLACEHOLDER - Load Configuration file here --
self.config_manager = ConfigManager(cli_config["cli_overrides"])
config = self.config_manager.get_config()
client_api_binder = get_api(ApiContext.local,
config["plugin_modules"],
loop=loop)
server_api_binder = route_adapter
# Initialize logging
self.init_logging(config["logging"])
plugin_names = config["plugin_modules"]
# ### Loading Modules
self.logger.info("Loading modules.")
# load builtin modules
self.logger.debug("Loading builtin modules.")
bultin_modules = load_builtin_modules()
self.logger.debug("Finished loading builtin modules.")
# load plugin modules
self.logger.debug("Loading plugin modules.")
plugin_modules = load_plugin_modules(plugin_names)
self.logger.debug("Finished loading plugin modules.")
self.logger.info("Finished loading modules.")
self.loaded_modules = bultin_modules + plugin_modules
# ### Registering Handlers
self.logger.info("Registering handlers.")
for module in self.loaded_modules:
# Note client api registration is handled by the patcher
self.logger.debug(
"Registering server api endpoints for module msa.{}".format(
module.__name__))
if hasattr(module, "register_server_api") and callable(
module.register_server_api):
module.register_server_api(server_api_binder)
if hasattr(module, "entities_list") and isinstance(
module.entities_list, list):
__models__.extend(module.entities_list)
module_name_tail = module.__name__.split(".")[-1]
module_config = config["module_config"].get(module_name_tail, None)
if not (hasattr(module, "config_schema")
and isinstance(module.config_schema, Schema)):
raise Exception(
"All modules must define a `config_schema` property that is an instance of "
"schema.Schema")
self.logger.debug(
f"Validating module {module.__name__} config schema.")
validated_config = module.config_schema.validate(module_config)
self.logger.debug("Registering handlers for module msa.{}".format(
module.__name__))
for handler in module.handler_factories:
namespace = "{}.{}".format(module.__name__[4:],
handler.__name__)
full_namespace = "msa.{}".format(namespace)
self.logger.debug(
"Registering handler: msa.{}".format(namespace))
handler_logger = self.root_logger.getChild(namespace)
self.loggers[full_namespace] = handler_logger
inited_handler = handler(self.loop, self.event_bus,
handler_logger, validated_config)
self.initialized_event_handlers.append(inited_handler)
self.handler_lookup[handler] = inited_handler
self.logger.debug(
"Finished registering handler: {}".format(full_namespace))
self.logger.debug(
"Finished registering handlers for module {}".format(
module.__name__))
self.logger.info("Finished registering handlers.")
self.apply_granular_log_levels(
config["logging"]["granular_log_levels"])
class Supervisor:
"""The supervisor is responsible for managing the execution of the application and orchestrating the event system."""
def __init__(self):
self.config_manager = None
self.stop_loop = False
self.stop_future = None
self.coroutine_futures = []
self.loaded_modules = []
self.initialized_event_handlers = []
self.handler_lookup = {}
self.shutdown_callbacks = []
self.executor = ThreadPoolExecutor()
self.root_logger = None
self.logger = None
self.loggers = {}
def init_logging(self, logging_config):
"""
Initializes application logging, setting up the global log namespace, and the supervisor log namespace.
"""
self.root_logger = logging.getLogger("msa")
self.root_logger.setLevel(logging_config["global_log_level"])
mode = "w" if logging_config["truncate_log_file"] else "a"
file_handler = logging.FileHandler(logging_config["log_file_location"],
mode=mode)
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s - %(levelname)s - %(name)s - %(message)s")
file_handler.setFormatter(formatter)
self.root_logger.addHandler(file_handler)
self.logger = self.root_logger.getChild("core.supervisor")
self.loggers["core.supervisor"] = self.logger
def apply_granular_log_levels(self, granular_level_config):
"""
Applies the granular log levels configured in the conficuration file.
Parameters
----------
granular_level_config : List[Dict[String, String]]
A list of namespace to log level mappings to be applied.
"""
self.logger.info("Setting granular log levels.")
# order by length shortest namespaces will be the highest level, and should be applied first so that lower
# level rules may be applied without being overwritten.
granular_level_config = sorted(granular_level_config,
key=lambda e: len(e["namespace"]),
reverse=True)
for log_config in granular_level_config:
for namespace, logger in self.loggers.items():
if log_config["namespace"] in namespace:
effective_log_level = log_config.get(
"level", logger.getEffectiveLevel())
logger.setLevel(effective_log_level)
self.logger.info("Finished setting granular log levels.")
def init(self, loop, cli_config, route_adapter):
"""Initializes the supervisor.
Parameters
----------
loop : Asynio Event Loop
An asyncio event loop the supervisor should use.
cli_config: Dict
A dictionary containing configuration options derived from the command line interface.
route_adapter: ** fix docstrings **
"""
if not os.environ.get("TEST"):
self.loop = loop
self.event_bus = EventBus(self.loop)
self.event_queue = asyncio.Queue(self.loop)
# block getting a loop if we are running unit tests
# helps suppress a warning.
# ### PLACEHOLDER - Load Configuration file here --
self.config_manager = ConfigManager(cli_config["cli_overrides"])
config = self.config_manager.get_config()
client_api_binder = get_api(ApiContext.local,
config["plugin_modules"],
loop=loop)
server_api_binder = route_adapter
# Initialize logging
self.init_logging(config["logging"])
plugin_names = config["plugin_modules"]
# ### Loading Modules
self.logger.info("Loading modules.")
# load builtin modules
self.logger.debug("Loading builtin modules.")
bultin_modules = load_builtin_modules()
self.logger.debug("Finished loading builtin modules.")
# load plugin modules
self.logger.debug("Loading plugin modules.")
plugin_modules = load_plugin_modules(plugin_names)
self.logger.debug("Finished loading plugin modules.")
self.logger.info("Finished loading modules.")
self.loaded_modules = bultin_modules + plugin_modules
# ### Registering Handlers
self.logger.info("Registering handlers.")
for module in self.loaded_modules:
# Note client api registration is handled by the patcher
self.logger.debug(
"Registering server api endpoints for module msa.{}".format(
module.__name__))
if hasattr(module, "register_server_api") and callable(
module.register_server_api):
module.register_server_api(server_api_binder)
if hasattr(module, "entities_list") and isinstance(
module.entities_list, list):
__models__.extend(module.entities_list)
module_name_tail = module.__name__.split(".")[-1]
module_config = config["module_config"].get(module_name_tail, None)
if not (hasattr(module, "config_schema")
and isinstance(module.config_schema, Schema)):
raise Exception(
"All modules must define a `config_schema` property that is an instance of "
"schema.Schema")
self.logger.debug(
f"Validating module {module.__name__} config schema.")
validated_config = module.config_schema.validate(module_config)
self.logger.debug("Registering handlers for module msa.{}".format(
module.__name__))
for handler in module.handler_factories:
namespace = "{}.{}".format(module.__name__[4:],
handler.__name__)
full_namespace = "msa.{}".format(namespace)
self.logger.debug(
"Registering handler: msa.{}".format(namespace))
handler_logger = self.root_logger.getChild(namespace)
self.loggers[full_namespace] = handler_logger
inited_handler = handler(self.loop, self.event_bus,
handler_logger, validated_config)
self.initialized_event_handlers.append(inited_handler)
self.handler_lookup[handler] = inited_handler
self.logger.debug(
"Finished registering handler: {}".format(full_namespace))
self.logger.debug(
"Finished registering handlers for module {}".format(
module.__name__))
self.logger.info("Finished registering handlers.")
self.apply_granular_log_levels(
config["logging"]["granular_log_levels"])
def start(self, additional_coros=[]):
r"""Starts the supervisor.
Parameters
----------
additional_coros : List[Coroutines]
a list of other coroutines to be started. Acts as a hook for specialized
startup scenarios.
"""
self.logger.info("Starting startup coroutine.")
with suppress(asyncio.CancelledError):
self.logger.debug("Priming startup coroutine")
primed_coro = self.startup_coroutine(additional_coros)
self.logger.debug(
"Startup coroutine primed, executing in the loop.")
self.loop.create_task(primed_coro)
self.logger.debug("Finished running startup coroutine.")
async def exit(self):
"""Shuts down running tasks and stops the event loop, exiting the application."""
self.logger.info("Stopping handlers.")
self.stop_loop = True
self.logger.debug("Shutting down executor threads.")
self.executor.shutdown()
self.logger.debug("Calling shutdown callbacks.")
for callback in self.shutdown_callbacks:
callback()
self.logger.debug("Exit: Cancelling remaining futures.")
for future in self.coroutine_futures:
if future is not None:
with suppress(asyncio.CancelledError):
await future
self.logger.debug("Exit: Cancelling event bus futures.")
with suppress(asyncio.CancelledError):
if self.event_bus.task is not None:
self.event_bus.task.cancel()
await self.event_bus.task
# cancel and suppress exit future
if self.stop_future is not None:
asyncio.gather(self.stop_future)
# await asyncio.sleep(0.5) # let most of the handlers finish their current loop
# await asyncio.sleep(0.5) # let most of the handlers finish their current loop
self.logger.info("Exit: finished shutting down supervisor.")
print("\rGoodbye!\n")
def fire_event(self, new_event: Event):
"""Fires an event to all event listeners.
Parameters
----------
new_event : `Event`
A new instance of a subclass of `Event` to be propagated to other event handlers.
"""
self.logger.debug("Fire event: {}".format(new_event))
def fire():
self.loop.create_task(self.event_bus.fire_event(new_event))
self.loop.call_soon(fire)
async def listen_for_result(self, event_type):
return await self.event_bus.listen_for_result(event_type)
async def startup_coroutine(self, additional_coros=[]):
"""The main coroutine that manages starting the handlers, and waiting for a shutdown signal.
Parameters
----------
additional_coros : List[Coroutines]
Additional coroutines to be run in the event loop.
"""
self.logger.debug("Startup coroutine executing.")
# ### Initialize database requirements for modules
self.logger.info("Initializing database add-ons")
for module in self.loaded_modules:
if hasattr(module, "entity_setup"):
self.logger.debug(
f"Initializing database for module msa.{module.__name__}")
await module.entity_setup()
self.logger.debug(
f"Finished initializing database for module msa.{module.__name__}"
)
self.logger.debug("Startup Coroutine: Call async init on handlers.")
init_coros = [
handler.init() for handler in self.initialized_event_handlers
]
scheduled_coros = self._get_prep_scheduled_coros()
await asyncio.gather(*init_coros)
self.logger.debug("Startup Coroutine: Prime EventBus coroutine.")
primed_coros = [self.event_bus.listen(), *scheduled_coros]
self.logger.debug(
"Startup Coroutine: Prime additional coroutines: {}".format(
len(additional_coros)))
if len(additional_coros) > 0:
primed_coros.extend(additional_coros)
try:
self.logger.debug("Beginning handler execution.")
with suppress(asyncio.CancelledError):
self.coroutine_futures = await asyncio.gather(*primed_coros)
except Exception as err:
self.logger.exception(err)
self.logger.debug("Startup Coroutine: Finished startup")
def should_stop(self):
"""Indicates whether the supervisor is in the process is shutting down.
Used for signaling event_handlers to cancel rescheduling.
"""
return self.stop_loop
def get_handler(self, handler_type):
"""Returns the handler instance for a given type of handler. Used for unit tests.
Parameters
----------
- handler_type: A type of handler."""
return self.handler_lookup.get(handler_type)
def _get_prep_scheduled_coros(self):
coros = []
for handler in self.initialized_event_handlers:
if not hasattr(handler.schedule, "base_class"):
# then we know that this subclass has overwritten the base class with an implementation
coros_to_schedule = handler.schedule()
for [tab, coro] in coros_to_schedule:
coros.append(self._wrap_scheduled_coro(tab, coro))
return coros
@staticmethod
async def _wrap_scheduled_coro(tab_str, coro):
tab = crontab(tab_str, func=lambda: datetime.now(), start=False)
while True:
time = await tab.next()
await coro(time)
def test_get_config_success(self, Path_mock: MagicMock):
fake_config_contents = {
"agent": {
"name": "Masa-chan",
"user_title": "Onee-chan"
},
"plugin_modules": [],
"module_config": {},
"logging": {
"global_log_level": "debug",
"log_file_location": "msa.log",
"truncate_log_file": False,
"granular_log_levels": [],
},
}
fake_config_contents_str = json.dumps(fake_config_contents)
cli_overrides = {}
local_dir_path_mock = MagicMock()
custom_path_mock = MagicMock()
local_dir_path_mock.exists = MagicMock(return_value=True)
local_dir_path_mock.open = mock_open(
read_data=fake_config_contents_str)
path_mocks = {"./msa_config.json": local_dir_path_mock}
Path_mock.side_effect = lambda arg: path_mocks[arg]
home_path_mock = MagicMock()
home_path_extended_mock = MagicMock()
home_path_extended_mock.exists = MagicMock(return_value=False)
home_path_mock.join = MagicMock(return_value=home_path_extended_mock)
Path_mock.home.return_value = home_path_mock
config_manager = ConfigManager(cli_overrides)
config = config_manager.get_config()
self.assertEqual(config["agent"], fake_config_contents["agent"])
self.assertEqual(config["plugin_modules"],
fake_config_contents["plugin_modules"])
self.assertEqual(config["module_config"],
fake_config_contents["module_config"])
self.assertEqual(config["logging"]["global_log_level"], "DEBUG")
self.assertEqual(
config["logging"]["log_file_location"],
fake_config_contents["logging"]["log_file_location"],
)
self.assertEqual(
config["logging"]["log_file_location"],
fake_config_contents["logging"]["log_file_location"],
)
self.assertEqual(
config["logging"]["truncate_log_file"],
fake_config_contents["logging"]["truncate_log_file"],
)
self.assertEqual(
config["logging"]["granular_log_levels"],
fake_config_contents["logging"]["granular_log_levels"],
)
class Interpreter:
def __init__(self, cli_overrides):
history_file = FileHistory(".msa_cli_history")
self.prompt_session = PromptSession(lexer=PygmentsLexer(Python3Lexer),
history=history_file)
self.config_manager = ConfigManager(cli_overrides)
self.config = self.config_manager.get_config()
self.loop = asyncio.get_event_loop()
self.api = None
self.quit = False
self.exit_code = 0
self.locals = {}
self.globals = {}
self.func_locals = {}
self.buffer = ""
self.indent_level = 0
self.indent_size = 4
self.recording = False
self.record_buffer_name = ""
self.record_buffer = ""
async def startup_check(self):
await self.api.check_connection()
await self.api.check_version(quiet=True)
def execute_script(self, script):
with open(script, "r") as f:
text = f.read()
print("Running script:")
print("| " + "\n| ".join(text.split("\n")))
print("Script output:")
self.execute_block(text)
def start(self):
self.api = get_api(
ApiContext.websocket,
self.config["plugin_modules"],
loop=self.loop,
interact=self.interact,
propagate=self.propagate,
host="localhost",
port=8080,
)
self.globals["msa_api"] = self.api
self.loop.run_until_complete(self._start())
async def _start(self):
# startup checks
await self.api.client.connect()
async def propagate(self, event_queue):
while True:
event = await event_queue.get()
# print("Got background event", event)
return
async def interact(self):
try:
await self.startup_check()
while True:
try:
prompt_text, prompt_default = self.generate_prompt_text()
text = await self.prompt_session.prompt(
prompt_text, default=prompt_default, async_=True)
except KeyboardInterrupt:
continue
except EOFError:
break
except Exception as e:
print(e)
time.sleep(5)
else:
await self.parse_statement(text)
if self.quit:
break
finally:
await self.api.client.disconnect()
print("Goodbye")
await asyncio.sleep(0)
# exit(self.exit_code)
def generate_prompt_text(self):
if self.indent_level == 0:
prompt_text = ">>> "
prompt_default = ""
else:
prompt_text = "..."
prompt_default = " " * self.indent_level * self.indent_size
return prompt_text, prompt_default
async def parse_statement(self, text):
if self.indent_level == 0:
skip_loop = self.parse_command(text)
if skip_loop:
return
if self.recording:
self.record_buffer += text + "\n"
# update current indent level
if self.indent_level > 0:
if len(text.strip()) == 0:
self.indent_level = 0
else:
current_indent = (len(text) -
len(text.lstrip())) // self.indent_size
self.indent_level = current_indent
# if we see a colon, begin buffer
if len(text) > 0 and text[-1] == ":":
self.indent_level += 1
if self.indent_level > 0:
self.buffer += text + "\n"
if self.indent_level == 0:
if len(self.buffer) > 0:
text = self.buffer + "\n" + text
self.buffer = ""
await self.execute_block(text)
async def execute_block(self, text):
try:
await self.aexec(text.strip())
except SystemExit as e:
self.quit = True
self.exit_code = e.code or 0
return
except:
self.print_traceback(traceback.format_exc())
async def aexec(self, code):
# Make an async function with the code and `exec` it
effective_globals = {**self.locals, **self.globals}
exec(
f"async def __ex(): " + "".join(f"\n {l}"
for l in code.split("\n")) +
"\n return locals()",
effective_globals,
self.locals,
)
# Get `__ex` from local variables, call it and return the result
self.func_locals = await self.locals["__ex"]()
for key in list(self.func_locals.keys()):
if key in self.globals:
del self.func_locals[key]
raise Exception(
f'Statement attempted to override global variable "{key}". This is not allowed.'
)
# patch func locals into locals
self.locals = {**self.locals, **self.func_locals}
def print_traceback(self, *args, **kwargs):
stringify = " ".join(str(e) for e in args)
print(
highlight(stringify, Python3TracebackLexer(), TerminalFormatter()))
def parse_command(self, text):
clean_text = text.strip()
if len(clean_text) == 0: # obviously there is no command to parse
return
if clean_text[0] == "#":
tokens = clean_text[1::].split()
if tokens[0] == "record":
if len(tokens) < 2:
print(
"Record command requires either 'stop' or a file name to record to."
)
return True
if tokens[1] == "stop":
self.recording = False
with open(self.record_buffer_name, "w") as f:
f.write(self.record_buffer)
self.record_buffer = ""
editor = os.getenv("EDITOR")
if editor == None or editor == "":
print("Opening {}".format(self.record_buffer_name))
webbrowser.open(
"file://" +
os.path.abspath(self.record_buffer_name))
else:
print("Opening {} via {}".format(
self.record_buffer_name, editor))
os.system("%s %s" % (editor, self.record_buffer_name))
return True
else:
if len(tokens) < 2:
print(
"Record command requires a file name to record to. e.g. #record a.py"
)
return True
self.record_buffer_name = tokens[1]
self.recording = True
return True
elif tokens[0] == "clear":
print(chr(27) + "[2J")
elif tokens[0] == "help":
print("\n".join((
"MSA Interpreter Help:",
" Availiable Commands:",
" # help: Show this help text",
" # record <file name>: Begin recording commands to a script.",
" # record stop: stop recording commands, save the script, and open to review.",
)))
return False
def init(self, mode, cli_config):
"""Initializes the supervisor.
Parameters
----------
mode : int
A msa.core.RunMode enum value to configure which modules should be started based on the
environment the system is being run in.
cli_config: Dict
A dictionary containing configuration options derived from the command line interface.
"""
# ### PLACEHOLDER - Load Configuration file here --
self.config_manager = ConfigManager(cli_config)
config = self.config_manager.get_config()
# Initialize logging
self.init_logging(config["logging"])
plugin_names = []
# ### Loading Modules
self.logger.info("Loading modules.")
# load builtin modules
self.logger.debug("Loading builtin modules.")
bultin_modules = load_builtin_modules()
self.logger.debug("Finished loading builtin modules.")
# load plugin modules
self.logger.debug("Loading plugin modules.")
plugin_modules = load_plugin_modules(plugin_names, mode)
self.logger.debug("Finished loading plugin modules.")
self.logger.info("Finished loading modules.")
self.loaded_modules = bultin_modules + plugin_modules
# ### Registering Handlers
self.logger.info("Registering handlers.")
# register event handlers
for module in self.loaded_modules:
self.logger.debug("Registering handlers for module msa.{}".format(
module.__name__))
for handler in module.handler_factories:
namespace = "{}.{}".format(module.__name__[4:],
handler.__name__)
full_namespace = "msa.{}".format(namespace)
self.logger.debug(
"Registering handler: msa.{}".format(namespace))
handler_logger = self.root_logger.getChild(namespace)
self.loggers[full_namespace] = handler_logger
event_queue = self.event_bus.create_event_queue()
module_config = config["module_config"].get(
full_namespace, None)
inited_handler = handler(self.loop, event_queue,
handler_logger, module_config)
self.initialized_event_handlers.append(inited_handler)
self.handler_lookup[handler] = inited_handler
self.logger.debug(
"Finished registering handler: {}".format(full_namespace))
self.logger.debug(
"Finished registering handlers for module {}".format(
module.__name__))
self.logger.info("Finished registering handlers.")
self.apply_granular_log_levels(
config["logging"]["granular_log_levels"])
class Supervisor:
"""The supervisor is responsible for managing the execution of the application and orchestrating the event system.
"""
def __init__(self):
if not os.environ.get("TEST"):
# block getting a loop if we are running unit tests
# helps suppress a warning.
self.loop = asyncio.new_event_loop()
self.event_bus = EventBus(self.loop)
self.event_queue = asyncio.Queue(self.loop)
self.config_manager = None
self.stop_loop = False
self.stop_main_coro = None
self.stop_future = None
self.loaded_modules = []
self.initialized_event_handlers = []
self.handler_lookup = {}
self.shutdown_callbacks = []
self.executor = ThreadPoolExecutor()
self.root_logger = None
self.logger = None
self.loggers = {}
def init_logging(self, logging_config):
"""
Initializes application logging, setting up the global log namespace, and the supervisor log namespace.
"""
self.root_logger = logging.getLogger("msa")
self.root_logger.setLevel(logging_config["global_log_level"])
mode = "w" if logging_config["truncate_log_file"] else "a"
file_handler = logging.FileHandler(logging_config["log_file_location"],
mode=mode)
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s')
file_handler.setFormatter(formatter)
self.root_logger.addHandler(file_handler)
self.logger = self.root_logger.getChild("core.supervisor")
self.loggers["core.supervisor"] = self.logger
def apply_granular_log_levels(self, granular_level_config):
"""
Applies the granular log levels configured in the conficuration file.
Parameters
----------
granular_level_config : List[Dict[String, String]]
A list of namespace to log level mappings to be applied.
"""
self.logger.info("Setting granular log levels.")
# order by length shortest namespaces will be the highest level, and should be applied first so that lower
# level rules may be applied without being overwritten.
granular_level_config = sorted(granular_level_config,
key=lambda e: len(e["namespace"]),
reverse=True)
for log_config in granular_level_config:
for namespace, logger in self.loggers.items():
if log_config["namespace"] in namespace:
effective_log_level = log_config.get(
"level", logger.getEffectiveLevel())
logger.setLevel(effective_log_level)
self.logger.info("Finished setting granular log levels.")
def init(self, mode, cli_config):
"""Initializes the supervisor.
Parameters
----------
mode : int
A msa.core.RunMode enum value to configure which modules should be started based on the
environment the system is being run in.
cli_config: Dict
A dictionary containing configuration options derived from the command line interface.
"""
# ### PLACEHOLDER - Load Configuration file here --
self.config_manager = ConfigManager(cli_config)
config = self.config_manager.get_config()
# Initialize logging
self.init_logging(config["logging"])
plugin_names = []
# ### Loading Modules
self.logger.info("Loading modules.")
# load builtin modules
self.logger.debug("Loading builtin modules.")
bultin_modules = load_builtin_modules()
self.logger.debug("Finished loading builtin modules.")
# load plugin modules
self.logger.debug("Loading plugin modules.")
plugin_modules = load_plugin_modules(plugin_names, mode)
self.logger.debug("Finished loading plugin modules.")
self.logger.info("Finished loading modules.")
self.loaded_modules = bultin_modules + plugin_modules
# ### Registering Handlers
self.logger.info("Registering handlers.")
# register event handlers
for module in self.loaded_modules:
self.logger.debug("Registering handlers for module msa.{}".format(
module.__name__))
for handler in module.handler_factories:
namespace = "{}.{}".format(module.__name__[4:],
handler.__name__)
full_namespace = "msa.{}".format(namespace)
self.logger.debug(
"Registering handler: msa.{}".format(namespace))
handler_logger = self.root_logger.getChild(namespace)
self.loggers[full_namespace] = handler_logger
event_queue = self.event_bus.create_event_queue()
module_config = config["module_config"].get(
full_namespace, None)
inited_handler = handler(self.loop, event_queue,
handler_logger, module_config)
self.initialized_event_handlers.append(inited_handler)
self.handler_lookup[handler] = inited_handler
self.logger.debug(
"Finished registering handler: {}".format(full_namespace))
self.logger.debug(
"Finished registering handlers for module {}".format(
module.__name__))
self.logger.info("Finished registering handlers.")
self.apply_granular_log_levels(
config["logging"]["granular_log_levels"])
def start(self, additional_coros=[]):
r"""Starts the supervisor.
Parameters
----------
additional_coros : List[Coroutines]
a list of other coroutines to be started. Acts as a hook for specialized
startup scenarios.
"""
self.logger.info("Starting main coroutine.")
try:
with suppress(asyncio.CancelledError):
self.logger.debug("Priming main coroutine")
primed_coro = self.main_coro(additional_coros)
self.logger.debug(
"Main coroutine primed, executing in the loop.")
self.loop.run_until_complete(primed_coro)
self.logger.debug("Finished running main coroutine.")
except KeyboardInterrupt:
self.info(
"Keyboard interrupt (Ctrl-C) encountered, beginning shutdown.")
print("Ctrl-C Pressed. Quitting...")
finally:
self.stop()
self.loop.run_until_complete(self.loop.shutdown_asyncgens())
self.logger.info("Stopping loop.")
self.loop.close()
self.logger.info("Exiting.")
sys.exit(0)
def stop(self):
"""Schedules the supervisor to stop, and exit the application."""
self.logger.info("Schedule the main coroutine to stop.")
self.stop_future = asyncio.ensure_future(self.exit())
async def exit(self):
"""Shuts down running tasks and stops the event loop, exiting the application."""
self.logger.info("Stopping handlers, and main coroutine.")
self.stop_loop = True
self.logger.debug("Shutting down executor threads.")
self.executor.shutdown()
self.logger.debug("Calling shutdown callbacks.")
for callback in self.shutdown_callbacks:
callback()
await asyncio.sleep(
0.5) # let most of the handlers finish their current loop
await asyncio.sleep(
0.5) # let most of the handlers finish their current loop
self.stop_main_coro = True
self.logger.debug("Cancel any remaining tasks.")
if sys.version_info[0] == 3 and sys.version_info[1] == 6:
pending = asyncio.Task.all_tasks()
current = asyncio.Task.current_task()
else:
pending = asyncio.all_tasks() # get all tasks
current = asyncio.current_task()
pending.remove(current) # except this task
pending.remove(self.main_coro_task)
for task in pending:
if not task.done():
with suppress(asyncio.CancelledError):
task.cancel()
await asyncio.sleep(0.01)
await task
def fire_event(self, new_event):
"""Fires an event to all event listeners.
Parameters
----------
new_event : `Event`
A new instance of a subclass of `Event` to be propagated to other event handlers.
"""
self.logger.debug("Fire event: {}".format(new_event))
def fire():
self.loop.create_task(self.event_bus.fire_event(new_event))
self.loop.call_soon(fire)
async def main_coro(self, additional_coros=[]):
"""The main coroutine that manages starting the handlers, and waiting for a shutdown signal.
Parameters
----------
additional_coros : List[Coroutines]
Additional coroutines to be run in the event loop.
"""
self.logger.debug("Main coroutine executing.")
if sys.version_info[0] == 3 and sys.version_info[1] == 6:
self.main_coro_task = asyncio.Task.current_task()
else:
self.main_coro_task = asyncio.current_task()
self.logger.debug("Main Coro: Call async init on handlers.")
init_coros = [
handler.init() for handler in self.initialized_event_handlers
]
await asyncio.gather(*init_coros)
self.logger.debug("Main Coro: Prime handler coroutines.")
primed_coros = [
handler.handle_wrapper()
for handler in self.initialized_event_handlers
]
self.logger.debug("Main Coro: Prime additional coroutines: {}".format(
len(additional_coros)))
if len(additional_coros) > 0:
primed_coros.extend(additional_coros)
futures = None
try:
self.logger.debug("Beginning handler execution.")
futures = await asyncio.gather(*primed_coros)
except Exception as err:
self.logger.eror(err, traceback.print_exc())
self.logger.debug("Main Coro: Sleep until shutdown is started.")
while not self.stop_main_coro:
await asyncio.sleep(0.5)
self.logger.debug("Main Coro: Wakeup for shutdown.")
if futures is not None:
for future in futures:
if future is not None:
with suppress(asyncio.CancelledError):
await future
# cancel and suppress exit future
if self.stop_future is not None:
asyncio.gather(self.stop_future)
self.logger.info("Main coro: finishing execution.")
print("\rGoodbye!\n")
def should_stop(self):
"""Indicates whether the supervisor is in the process is shutting down.
Used for signaling event_handlers to cancel rescheduling.
"""
return self.stop_loop
def get_handler(self, handler_type):
"""Returns the handler instance for a given type of handler. Used for unit tests.
Parameters
----------
- handler_type: A type of handler."""
return self.handler_lookup.get(handler_type)