def load_settings(): """ Load settings from seed from env """ settings_data = None settings_seed = os.environ.get("CORE_CONFIG_SEED", None) # if not settings_seed or ":" not in settings_seed: return None # settings_seed_tag = settings_seed[:settings_seed.find(":")] settings_seed_data = settings_seed[len(settings_seed_tag) + 1:] try: seed = importlib.import_module(f"pylon.core.seeds.{settings_seed_tag}") settings_data = seed.unseed(settings_seed_data) except: # pylint: disable=W0702 log.exception("Failed to unseed settings") # if not settings_data: return None # try: settings = yaml.load(os.path.expandvars(settings_data), Loader=yaml.SafeLoader) settings = config.config_substitution(settings, config.vault_secrets(settings)) except: # pylint: disable=W0702 log.exception("Failed to parse settings") return None # return settings
def __init__(self, context): self.context = context # events_rabbitmq = self.context.settings.get("events", dict()).get( "rabbitmq", dict()) if events_rabbitmq: try: self.node = arbiter.EventNode( host=events_rabbitmq.get("host"), port=events_rabbitmq.get("port", 5672), user=events_rabbitmq.get("user", ""), password=events_rabbitmq.get("password", ""), vhost=events_rabbitmq.get("vhost", "carrier"), event_queue=events_rabbitmq.get("queue", "events"), hmac_key=events_rabbitmq.get("hmac_key", None), hmac_digest=events_rabbitmq.get("hmac_digest", "sha512"), callback_workers=events_rabbitmq.get( "callback_workers", 1), ) self.node.start() except: # pylint: disable=W0702 log.exception( "Cannot make EventNode instance, using local events only") self.node = arbiter.MockEventNode() else: self.node = arbiter.MockEventNode() # self.partials = dict()
def load_development_modules(context): """ Load and enable platform modules in development mode """ # if os.environ.get("WERKZEUG_RUN_MAIN") != "true": log.info("Running in development mode before reloader is started. Skipping module loading") return list() log.info("Using module dir: %s", context.settings["development"]["modules"]) module_map = get_development_module_map(context) log.info("Enabling module: Market") try: module_metadata, _ = module_map.pop('market') enable_development_module('market', module_metadata, context=context) log.info("Initialized module: Market") module_map = get_development_module_map(context) del module_map['market'] except: # pylint: disable=W0702 log.exception("Failed to initialize module: Market") module_order = dependency.resolve_depencies(module_map) log.debug("Module order: %s", module_order) temporary_data_dirs = list() for module_name in module_order: log.info("Enabling module: %s", module_name) try: module_metadata, _ = module_map[module_name] enable_development_module(module_name, module_metadata, context=context) log.info("Initialized module: %s", module_name) except: # pylint: disable=W0702 log.exception("Failed to initialize module: %s", module_name) return temporary_data_dirs
def get_development_config(settings, name): """ Get config from storage """ config_path = os.environ.get("PYLON_CONFIG_PATH", settings["development"]["config"]) try: config_data = open(os.path.join(config_path, f"{name}.yml"), "rb").read() yaml_data = yaml.load(os.path.expandvars(config_data), Loader=yaml.SafeLoader) return config_substitution(yaml_data, vault_secrets(settings)) except: # pylint: disable=W0702 log.exception("Failed to get config for %s, assuming none", name) return None
def get_config(settings, name): """ Get config from storage """ minio = MinIOHelper.get_client(settings["storage"]) try: config_data = minio.get_object(settings["storage"]["buckets"]["config"], f"{name}.yml").read() # pylint: disable=C0301 yaml_data = yaml.load(os.path.expandvars(config_data), Loader=yaml.SafeLoader) return config_substitution(yaml_data, vault_secrets(settings)) except: # pylint: disable=W0702 log.exception("Failed to get config for %s, assuming none", name) return None
def run_slot(self, slot, payload=None): """ Run callbacks for slot """ result = list() if slot not in self.callbacks: return "" for callback in self.callbacks[slot]: try: callback_func = getattr(self.context.rpc_manager.call, callback) callback_result = callback_func(slot, payload) if callback_result is not None: result.append(callback_result) except: # pylint: disable=W0702 log.exception("Template slot callback exception") return "\n".join(result)
def get_development_module_map(context) -> dict: module_map = dict() # module_name -> (metadata, loader) # for module_name in storage.list_development_modules(context.settings): log.info("Found module: %s", module_name) # module_path = os.path.join(context.settings["development"]["modules"], module_name) metadata_path = os.path.join(module_path, "metadata.json") # try: # Make loader for this module module_loader = None # Load module metadata if not os.path.exists(metadata_path): log.error("No module metadata, skipping") continue with open(metadata_path, "r") as file: module_metadata = json.load(file) # Add to module map module_map[module_name] = (module_metadata, module_loader) except: # pylint: disable=W0702 log.exception("Failed to prepare module: %s", module_name) return module_map
def main(): # pylint: disable=R0912,R0914,R0915 """ Entry point """ # Register signal handling signal.signal(signal.SIGTERM, signal_sigterm) # Enable logging enable_logging() # Say hello log.info("Starting plugin-based Galloper core") # Make context holder context = Context() # Load settings from seed log.info("Loading and parsing settings") settings = load_settings() if not settings: log.error("Settings are empty or invalid. Exiting") os._exit(1) # pylint: disable=W0212 context.settings = settings # Save global node name context.node_name = settings.get("server", dict()).get("name", socket.gethostname()) # Enable Loki logging if requested in config enable_loki_logging(context) # Register provider for template and resource loading from modules pkg_resources.register_loader_type(module.DataModuleLoader, module.DataModuleProvider) # Make ModuleManager instance module_manager = module.ModuleManager(settings) context.module_manager = module_manager # Make EventManager instance event_manager = event.EventManager(context) context.event_manager = event_manager # Make app instance log.info("Creating Flask application") app = flask.Flask("project") api = Api(app, catch_all_404s=True) if settings.get("server", dict()).get("proxy", False): app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1) context.app = app context.api = api # Set application settings app.config["CONTEXT"] = context app.config.from_mapping(settings.get("application", dict())) # Save global URL prefix to context. May merge with traefik rule in future context.url_prefix = settings.get("server", dict()).get("path", "/") while context.url_prefix.endswith("/"): context.url_prefix = context.url_prefix[:-1] # Enable server-side sessions init_flask_sessions(context) # Make RpcManager instance rpc_manager = rpc.RpcManager(context) context.rpc_manager = rpc_manager # Make SlotManager instance slot_manager = slot.SlotManager(context) context.slot_manager = slot_manager app.context_processor(slot.template_slot_processor(context)) # Load and initialize modules if not CORE_DEVELOPMENT_MODE: temporary_data_dirs = load_modules(context) else: temporary_data_dirs = load_development_modules(context) # Register Traefik route via Redis KV register_traefik_route(context) # Run WSGI server try: if not CORE_DEVELOPMENT_MODE: log.info("Starting WSGI server") http_server = WSGIServer( ( settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST), settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT) ), app ) http_server.serve_forever() else: log.info("Starting Flask server") app.run( host=settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST), port=settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT), debug=CORE_DEVELOPMENT_MODE, use_reloader=CORE_DEVELOPMENT_MODE, ) finally: log.info("WSGI server stopped") # Unregister traefik route unregister_traefik_route(context) # De-init modules for module_name in module_manager.modules: _, _, module_obj = module_manager.get_module(module_name) module_obj.deinit() # Delete module data dirs for directory in temporary_data_dirs: log.info("Deleting temporary data directory: %s", directory) try: shutil.rmtree(directory) except: # pylint: disable=W0702 log.exception("Failed to delete, skipping") # Exit log.info("Exiting")
def load_modules(context): """ Load and enable platform modules """ # module_map = dict() # module_name -> (metadata, loader) # for module_name in storage.list_modules(context.settings): log.info("Found module: %s", module_name) module_data = storage.get_module(context.settings, module_name) if not module_data: log.error("Failed to get module data, skipping") continue try: # Make loader for this module module_loader = module.DataModuleLoader(module_data) # Load module metadata if "metadata.json" not in module_loader.storage_files: log.error("No module metadata, skipping") continue with module_loader.storage.open("metadata.json", "r") as file: module_metadata = json.load(file) # Add to module map module_map[module_name] = (module_metadata, module_loader) except: # pylint: disable=W0702 log.exception("Failed to prepare module: %s", module_name) # module_order = dependency.resolve_depencies(module_map) log.debug("Module order: %s", module_order) # temporary_data_dirs = list() # for module_name in module_order: log.info("Enabling module: %s", module_name) try: # Get module metadata and loader module_metadata, module_loader = module_map[module_name] log.info( "Initializing module: %s [%s]", module_metadata.get("name", "N/A"), module_metadata.get("version", "N/A"), ) # Extract module data if needed if module_metadata.get("extract", False): module_data_dir = tempfile.mkdtemp() temporary_data_dirs.append(module_data_dir) module_loader.storage.extractall(module_data_dir) module_root_path = os.path.join( module_data_dir, module_metadata.get("module").replace(".", os.path.sep) ) else: module_root_path = None # Import module package sys.meta_path.insert(0, module_loader) importlib.invalidate_caches() module_pkg = importlib.import_module(module_metadata.get("module")) # Make module instance module_obj = module_pkg.Module( settings=storage.get_config(context.settings, module_name), root_path=module_root_path, context=context ) # Initialize module module_obj.init() # Finally done context.module_manager.add_module( module_name, module_root_path, module_metadata, module_obj ) log.info("Initialized module: %s", module_name) except: # pylint: disable=W0702 log.exception("Failed to initialize module: %s", module_name) # return temporary_data_dirs
def __init__(self, stderr: str): msg = f'Clone error {stderr}' log.exception(msg) super().__init__(msg)
def __init__(self, response: ClientResponse): msg = f'Fetch error {response.status} on url {response.url}' log.exception(msg) super().__init__(msg)