def load_settings():
""" Load settings from seed from env """
settings_data = None
settings_seed = os.environ.get("CORE_CONFIG_SEED", None)
#
if not settings_seed or ":" not in settings_seed:
return None
#
settings_seed_tag = settings_seed[:settings_seed.find(":")]
settings_seed_data = settings_seed[len(settings_seed_tag) + 1:]
try:
seed = importlib.import_module(f"pylon.core.seeds.{settings_seed_tag}")
settings_data = seed.unseed(settings_seed_data)
except: # pylint: disable=W0702
log.exception("Failed to unseed settings")
#
if not settings_data:
return None
#
try:
settings = yaml.load(os.path.expandvars(settings_data), Loader=yaml.SafeLoader)
settings = config.config_substitution(settings, config.vault_secrets(settings))
except: # pylint: disable=W0702
log.exception("Failed to parse settings")
return None
#
return settings
def __init__(self, context):
self.context = context
#
events_rabbitmq = self.context.settings.get("events", dict()).get(
"rabbitmq", dict())
if events_rabbitmq:
try:
self.node = arbiter.EventNode(
host=events_rabbitmq.get("host"),
port=events_rabbitmq.get("port", 5672),
user=events_rabbitmq.get("user", ""),
password=events_rabbitmq.get("password", ""),
vhost=events_rabbitmq.get("vhost", "carrier"),
event_queue=events_rabbitmq.get("queue", "events"),
hmac_key=events_rabbitmq.get("hmac_key", None),
hmac_digest=events_rabbitmq.get("hmac_digest", "sha512"),
callback_workers=events_rabbitmq.get(
"callback_workers", 1),
)
self.node.start()
except: # pylint: disable=W0702
log.exception(
"Cannot make EventNode instance, using local events only")
self.node = arbiter.MockEventNode()
else:
self.node = arbiter.MockEventNode()
#
self.partials = dict()
def load_development_modules(context):
""" Load and enable platform modules in development mode """
#
if os.environ.get("WERKZEUG_RUN_MAIN") != "true":
log.info("Running in development mode before reloader is started. Skipping module loading")
return list()
log.info("Using module dir: %s", context.settings["development"]["modules"])
module_map = get_development_module_map(context)
log.info("Enabling module: Market")
try:
module_metadata, _ = module_map.pop('market')
enable_development_module('market', module_metadata, context=context)
log.info("Initialized module: Market")
module_map = get_development_module_map(context)
del module_map['market']
except: # pylint: disable=W0702
log.exception("Failed to initialize module: Market")
module_order = dependency.resolve_depencies(module_map)
log.debug("Module order: %s", module_order)
temporary_data_dirs = list()
for module_name in module_order:
log.info("Enabling module: %s", module_name)
try:
module_metadata, _ = module_map[module_name]
enable_development_module(module_name, module_metadata, context=context)
log.info("Initialized module: %s", module_name)
except: # pylint: disable=W0702
log.exception("Failed to initialize module: %s", module_name)
return temporary_data_dirs
def get_development_config(settings, name):
""" Get config from storage """
config_path = os.environ.get("PYLON_CONFIG_PATH", settings["development"]["config"])
try:
config_data = open(os.path.join(config_path, f"{name}.yml"), "rb").read()
yaml_data = yaml.load(os.path.expandvars(config_data), Loader=yaml.SafeLoader)
return config_substitution(yaml_data, vault_secrets(settings))
except: # pylint: disable=W0702
log.exception("Failed to get config for %s, assuming none", name)
return None
def get_config(settings, name):
""" Get config from storage """
minio = MinIOHelper.get_client(settings["storage"])
try:
config_data = minio.get_object(settings["storage"]["buckets"]["config"], f"{name}.yml").read() # pylint: disable=C0301
yaml_data = yaml.load(os.path.expandvars(config_data), Loader=yaml.SafeLoader)
return config_substitution(yaml_data, vault_secrets(settings))
except: # pylint: disable=W0702
log.exception("Failed to get config for %s, assuming none", name)
return None
def run_slot(self, slot, payload=None):
""" Run callbacks for slot """
result = list()
if slot not in self.callbacks:
return ""
for callback in self.callbacks[slot]:
try:
callback_func = getattr(self.context.rpc_manager.call,
callback)
callback_result = callback_func(slot, payload)
if callback_result is not None:
result.append(callback_result)
except: # pylint: disable=W0702
log.exception("Template slot callback exception")
return "\n".join(result)
def get_development_module_map(context) -> dict:
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_development_modules(context.settings):
log.info("Found module: %s", module_name)
#
module_path = os.path.join(context.settings["development"]["modules"], module_name)
metadata_path = os.path.join(module_path, "metadata.json")
#
try:
# Make loader for this module
module_loader = None
# Load module metadata
if not os.path.exists(metadata_path):
log.error("No module metadata, skipping")
continue
with open(metadata_path, "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
return module_map
def main(): # pylint: disable=R0912,R0914,R0915
""" Entry point """
# Register signal handling
signal.signal(signal.SIGTERM, signal_sigterm)
# Enable logging
enable_logging()
# Say hello
log.info("Starting plugin-based Galloper core")
# Make context holder
context = Context()
# Load settings from seed
log.info("Loading and parsing settings")
settings = load_settings()
if not settings:
log.error("Settings are empty or invalid. Exiting")
os._exit(1) # pylint: disable=W0212
context.settings = settings
# Save global node name
context.node_name = settings.get("server", dict()).get("name", socket.gethostname())
# Enable Loki logging if requested in config
enable_loki_logging(context)
# Register provider for template and resource loading from modules
pkg_resources.register_loader_type(module.DataModuleLoader, module.DataModuleProvider)
# Make ModuleManager instance
module_manager = module.ModuleManager(settings)
context.module_manager = module_manager
# Make EventManager instance
event_manager = event.EventManager(context)
context.event_manager = event_manager
# Make app instance
log.info("Creating Flask application")
app = flask.Flask("project")
api = Api(app, catch_all_404s=True)
if settings.get("server", dict()).get("proxy", False):
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
context.app = app
context.api = api
# Set application settings
app.config["CONTEXT"] = context
app.config.from_mapping(settings.get("application", dict()))
# Save global URL prefix to context. May merge with traefik rule in future
context.url_prefix = settings.get("server", dict()).get("path", "/")
while context.url_prefix.endswith("/"):
context.url_prefix = context.url_prefix[:-1]
# Enable server-side sessions
init_flask_sessions(context)
# Make RpcManager instance
rpc_manager = rpc.RpcManager(context)
context.rpc_manager = rpc_manager
# Make SlotManager instance
slot_manager = slot.SlotManager(context)
context.slot_manager = slot_manager
app.context_processor(slot.template_slot_processor(context))
# Load and initialize modules
if not CORE_DEVELOPMENT_MODE:
temporary_data_dirs = load_modules(context)
else:
temporary_data_dirs = load_development_modules(context)
# Register Traefik route via Redis KV
register_traefik_route(context)
# Run WSGI server
try:
if not CORE_DEVELOPMENT_MODE:
log.info("Starting WSGI server")
http_server = WSGIServer(
(
settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT)
),
app
)
http_server.serve_forever()
else:
log.info("Starting Flask server")
app.run(
host=settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
port=settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT),
debug=CORE_DEVELOPMENT_MODE, use_reloader=CORE_DEVELOPMENT_MODE,
)
finally:
log.info("WSGI server stopped")
# Unregister traefik route
unregister_traefik_route(context)
# De-init modules
for module_name in module_manager.modules:
_, _, module_obj = module_manager.get_module(module_name)
module_obj.deinit()
# Delete module data dirs
for directory in temporary_data_dirs:
log.info("Deleting temporary data directory: %s", directory)
try:
shutil.rmtree(directory)
except: # pylint: disable=W0702
log.exception("Failed to delete, skipping")
# Exit
log.info("Exiting")
def load_modules(context):
""" Load and enable platform modules """
#
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_modules(context.settings):
log.info("Found module: %s", module_name)
module_data = storage.get_module(context.settings, module_name)
if not module_data:
log.error("Failed to get module data, skipping")
continue
try:
# Make loader for this module
module_loader = module.DataModuleLoader(module_data)
# Load module metadata
if "metadata.json" not in module_loader.storage_files:
log.error("No module metadata, skipping")
continue
with module_loader.storage.open("metadata.json", "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
#
module_order = dependency.resolve_depencies(module_map)
log.debug("Module order: %s", module_order)
#
temporary_data_dirs = list()
#
for module_name in module_order:
log.info("Enabling module: %s", module_name)
try:
# Get module metadata and loader
module_metadata, module_loader = module_map[module_name]
log.info(
"Initializing module: %s [%s]",
module_metadata.get("name", "N/A"),
module_metadata.get("version", "N/A"),
)
# Extract module data if needed
if module_metadata.get("extract", False):
module_data_dir = tempfile.mkdtemp()
temporary_data_dirs.append(module_data_dir)
module_loader.storage.extractall(module_data_dir)
module_root_path = os.path.join(
module_data_dir, module_metadata.get("module").replace(".", os.path.sep)
)
else:
module_root_path = None
# Import module package
sys.meta_path.insert(0, module_loader)
importlib.invalidate_caches()
module_pkg = importlib.import_module(module_metadata.get("module"))
# Make module instance
module_obj = module_pkg.Module(
settings=storage.get_config(context.settings, module_name),
root_path=module_root_path,
context=context
)
# Initialize module
module_obj.init()
# Finally done
context.module_manager.add_module(
module_name, module_root_path, module_metadata, module_obj
)
log.info("Initialized module: %s", module_name)
except: # pylint: disable=W0702
log.exception("Failed to initialize module: %s", module_name)
#
return temporary_data_dirs
def __init__(self, stderr: str):
msg = f'Clone error {stderr}'
log.exception(msg)
super().__init__(msg)
def __init__(self, response: ClientResponse):
msg = f'Fetch error {response.status} on url {response.url}'
log.exception(msg)
super().__init__(msg)
