def auth(self): if "X-Forwarded-Uri" in request.headers: if request.headers["X-Forwarded-Uri"].startswith("/static") and \ any(request.headers["X-Forwarded-Uri"].endswith(res) for res in [".ico", ".js", ".css"]): return make_response("OK") # Check if need to login target = request.args.get("target") scope = request.args.get("scope") for header in ("X-Forwarded-Proto", "X-Forwarded-Host", "X-Forwarded-Port", "X-Forwarded-Uri"): if header in request.headers: session[header] = request.headers[header] if "Authorization" in request.headers: return self.handle_auth( auth_header=request.headers.get("Authorization", "")) if "X-Forwarded-Uri" in request.headers and "/api/v1" in "X-Forwarded-Uri": if "Referer" in request.headers and "/api/v1" not in "Referer": session["X-Forwarded-Uri"] = request.headers["Referer"] else: session["X-Forwarded-Uri"] = request.base_url if not session.get( "auth_attributes") or session["auth_attributes"]["exp"] < int( time()): return redirect(self.settings["login_handler_url"], 302) if not session.get("auth", False) and not self.settings["disable_auth"]: # Redirect to login return redirect( self.settings.get( "auth_redirect", f"{request.base_url}{request.script_root}/login")) if target is None: target = "raw" # Map auth response response = make_response("OK") try: self.context.rpc_manager.call_function_with_timeout( func='{prefix}{key}'.format( prefix=self.settings['rpc_manager']['prefix']['mappers'], key=target.lower()), timeout=int(self.settings['rpc_manager']['timeout']), response=response, scope=scope) except Empty: log.error(f'Cannot find mapper for auth_key {target}') return make_response("KO", 403) except (AttributeError, TypeError): from traceback import format_exc log.error(f"Failed to map auth data {format_exc()}") except NameError: return redirect(self.settings["login_default_redirect_url"]) return response
def register_traefik_route(context): """ Create Traefik route for this Pylon instance """ context.traefik_redis_keys = list() # if CORE_DEVELOPMENT_MODE and os.environ.get("WERKZEUG_RUN_MAIN") != "true": log.info("Running in development mode before reloader is started. Skipping registration") return # traefik_config = context.settings.get("traefik", dict()) if not traefik_config: log.error("Cannot register route: no traefik config") return # redis_config = traefik_config.get("redis", dict()) if not redis_config: log.error("Cannot register route: no redis config") return # local_hostname = socket.gethostname() local_port = context.settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT) # node_name = context.node_name # if "node_url" in traefik_config: node_url = traefik_config.get("node_url") elif "node_hostname" in traefik_config: node_url = f"http://{traefik_config.get('node_hostname')}:{local_port}" else: node_url = f"http://{local_hostname}:{local_port}" # log.info("Registering traefik route for node '%s'", node_name) # store = StrictRedis( host=redis_config.get("host", "localhost"), password=redis_config.get("password", None), ) # traefik_rootkey = traefik_config.get("rootkey", "traefik") traefik_rule = traefik_config.get("rule", "PathPrefix(`/`)") traefik_entrypoint = traefik_config.get("entrypoint", "http") # store.set(f"{traefik_rootkey}/http/routers/{node_name}/rule", traefik_rule) store.set(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0", traefik_entrypoint) store.set(f"{traefik_rootkey}/http/routers/{node_name}/service", f"{node_name}") store.set(f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url", node_url) # context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/rule") context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0") context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/service") context.traefik_redis_keys.append( f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url" )
def info(self, scope: str = '') -> dict: """ Map info data """ if scope not in self.mapper_settings: raise redirect(self.access_denied_endpoint) auth_info = super(JsonMapper, self).info(scope) result = {"raw": auth_info} try: for key, path in self.mapper_settings[scope].items(): result[key] = jsonpath_rw.parse(path).find(auth_info)[0].value except: # pylint: disable=W0702 from traceback import format_exc log.error(f"Failed to set scope data {format_exc()}") return result
def info(scope, auth_settings: dict = None): """ Map info data """ if scope not in auth_settings["mappers"]["json"]: raise redirect(auth_settings["endpoints"]["access_denied"]) auth_info = raw.info() result = {"raw": auth_info} try: for key, path in auth_settings["mappers"]["json"][scope].items(): result[key] = jsonpath_rw.parse(path).find(auth_info)[0].value except: # pylint: disable=W0702 from traceback import format_exc log.error(f"Failed to set scope data {format_exc()}") return result
def _on_unregister_slot_callback(self, context, event_name, event_payload): _ = context, event_name # for key in ["slot", "callback"]: if key not in event_payload: log.error("Invalid slot unregistration data, skipping") return # if event_payload["slot"] not in self.callbacks: return if event_payload["callback"] not in self.callbacks[ event_payload["slot"]]: return self.callbacks[event_payload["slot"]].remove(event_payload["callback"])
def auth(scope, response, auth_settings: dict = None): """ Map auth data """ if scope not in auth_settings["mappers"]["header"]: raise redirect(auth_settings["endpoints"]["access_denied"]) response = raw.auth(scope, response) # Set "raw" headers too auth_info = info(scope) if f"/{scope}" not in auth_info["auth_attributes"]["groups"]: raise NameError(f"User is not a memeber of {scope} group") try: for header, path in auth_settings["mappers"]["header"][scope].items(): response.headers[header] = jsonpath_rw.parse(path).find( auth_info)[0].value except: log.error("Failed to set scope headers") return response
def auth(self, response: Response, scope: str = '') -> Response: """ Map auth data """ if scope not in self.mapper_settings: raise redirect(self.access_denied_endpoint) response = super(HeaderMapper, self).auth(response, scope) # Set "raw" headers too auth_info = self.info(scope) if f"/{scope}" not in auth_info["auth_attributes"]["groups"]: raise NameError(f"User is not a memeber of {scope} group") try: for header, path in self.mapper_settings[scope].items(): response.headers[header] = jsonpath_rw.parse(path).find( auth_info)[0].value except Exception as e: log.error(f"Failed to set scope headers: {str(e)}") return response
async def download_plugin_zip(url: str, plugin: Plugin) -> None: async with ClientSession() as client: async with client.get(url) as response: if response.ok: tmp_bytes = BytesIO(await response.read()) zip_file = ZipFile(tmp_bytes) zip_file.extractall(plugin.directory) src = Path(plugin.directory, zip_file.namelist()[0]) try: src.rename(plugin.path) except (FileExistsError, OSError): shutil.copytree(src, plugin.path, dirs_exist_ok=True) shutil.rmtree(src) plugin.reload_metadata() else: log.error('Response failed with status %s', response.status)
def _on_register_slot_callback(self, context, event_name, event_payload): _ = context, event_name # for key in ["slot", "callback"]: if key not in event_payload: log.error("Invalid slot registration data, skipping") return # log.debug("New slot callback: %s - %s", event_payload["slot"], event_payload["callback"]) # if event_payload["slot"] not in self.callbacks: self.callbacks[event_payload["slot"]] = list() if event_payload["callback"] not in self.callbacks[ event_payload["slot"]]: self.callbacks[event_payload["slot"]].append( event_payload["callback"])
def resolve_depencies(module_map): """ Resolve depencies """ # Check required depencies for module_name, module_data in module_map.items(): for dependency in module_data[0].get("depends_on", list()): if dependency not in module_map: log.error("Dependency %s not present (required by %s)", dependency, module_name) raise RuntimeError("Required dependency not present") # Walk modules module_order = list() visited_modules = set() for module_name in module_map: if module_name not in module_order: _walk_module_depencies(module_name, module_map, module_order, visited_modules) # Return correct order return module_order
def _walk_module_depencies(module_name, module_map, module_order, visited_modules): # Collect depencies depencies = list() depencies.extend(module_map[module_name][0].get("depends_on", list())) for optional_dependency in module_map[module_name][0].get( "init_after", list()): if optional_dependency in module_map: depencies.append(optional_dependency) # Resolve visited_modules.add(module_name) for dependency in depencies: if dependency not in module_order: if dependency in visited_modules: log.error("Circular dependency (%s <-> %s)", dependency, module_name) raise RuntimeError("Circular dependency present") _walk_module_depencies(dependency, module_map, module_order, visited_modules) # Add to resolved order module_order.append(module_name)
def check_auth(auth_header: str, *, rpc_prefix: str, rpc_timeout: int, rpc_manager: RpcManager) -> Response: try: auth_key, auth_value = auth_header.strip().split(" ") except ValueError: return make_response("KO", 401) if check_auth_token(auth_header=auth_header): return make_response("OK", 200) try: auth_result = rpc_manager.call_function_with_timeout( func='{prefix}{key}'.format(prefix=rpc_prefix, key=auth_key.lower()), timeout=rpc_timeout, auth_value=auth_value) except Empty: log.error(f'Cannot find handler for auth_key {auth_key.lower()}') return make_response("KO", 401) except binascii.Error: log.error('Incorrect auth data received') return make_response('KO', 403) return make_response(*auth_result)
def get_development_module_map(context) -> dict: module_map = dict() # module_name -> (metadata, loader) # for module_name in storage.list_development_modules(context.settings): log.info("Found module: %s", module_name) # module_path = os.path.join(context.settings["development"]["modules"], module_name) metadata_path = os.path.join(module_path, "metadata.json") # try: # Make loader for this module module_loader = None # Load module metadata if not os.path.exists(metadata_path): log.error("No module metadata, skipping") continue with open(metadata_path, "r") as file: module_metadata = json.load(file) # Add to module map module_map[module_name] = (module_metadata, module_loader) except: # pylint: disable=W0702 log.exception("Failed to prepare module: %s", module_name) return module_map
def unregister_traefik_route(context): """ Delete Traefik route for this Pylon instance """ # traefik_config = context.settings.get("traefik", dict()) if not traefik_config: log.error("Cannot unregister route: no traefik config") return # redis_config = traefik_config.get("redis", dict()) if not redis_config: log.error("Cannot unregister route: no redis config") return # log.info("Unregistering traefik route for node '%s'", context.node_name) # store = StrictRedis( host=redis_config.get("host", "localhost"), password=redis_config.get("password", None), ) # while context.traefik_redis_keys: key = context.traefik_redis_keys.pop() store.delete(key)
def main(): # pylint: disable=R0912,R0914,R0915 """ Entry point """ # Register signal handling signal.signal(signal.SIGTERM, signal_sigterm) # Enable logging enable_logging() # Say hello log.info("Starting plugin-based Galloper core") # Make context holder context = Context() # Load settings from seed log.info("Loading and parsing settings") settings = load_settings() if not settings: log.error("Settings are empty or invalid. Exiting") os._exit(1) # pylint: disable=W0212 context.settings = settings # Save global node name context.node_name = settings.get("server", dict()).get("name", socket.gethostname()) # Enable Loki logging if requested in config enable_loki_logging(context) # Register provider for template and resource loading from modules pkg_resources.register_loader_type(module.DataModuleLoader, module.DataModuleProvider) # Make ModuleManager instance module_manager = module.ModuleManager(settings) context.module_manager = module_manager # Make EventManager instance event_manager = event.EventManager(context) context.event_manager = event_manager # Make app instance log.info("Creating Flask application") app = flask.Flask("project") api = Api(app, catch_all_404s=True) if settings.get("server", dict()).get("proxy", False): app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1) context.app = app context.api = api # Set application settings app.config["CONTEXT"] = context app.config.from_mapping(settings.get("application", dict())) # Save global URL prefix to context. May merge with traefik rule in future context.url_prefix = settings.get("server", dict()).get("path", "/") while context.url_prefix.endswith("/"): context.url_prefix = context.url_prefix[:-1] # Enable server-side sessions init_flask_sessions(context) # Make RpcManager instance rpc_manager = rpc.RpcManager(context) context.rpc_manager = rpc_manager # Make SlotManager instance slot_manager = slot.SlotManager(context) context.slot_manager = slot_manager app.context_processor(slot.template_slot_processor(context)) # Load and initialize modules if not CORE_DEVELOPMENT_MODE: temporary_data_dirs = load_modules(context) else: temporary_data_dirs = load_development_modules(context) # Register Traefik route via Redis KV register_traefik_route(context) # Run WSGI server try: if not CORE_DEVELOPMENT_MODE: log.info("Starting WSGI server") http_server = WSGIServer( ( settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST), settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT) ), app ) http_server.serve_forever() else: log.info("Starting Flask server") app.run( host=settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST), port=settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT), debug=CORE_DEVELOPMENT_MODE, use_reloader=CORE_DEVELOPMENT_MODE, ) finally: log.info("WSGI server stopped") # Unregister traefik route unregister_traefik_route(context) # De-init modules for module_name in module_manager.modules: _, _, module_obj = module_manager.get_module(module_name) module_obj.deinit() # Delete module data dirs for directory in temporary_data_dirs: log.info("Deleting temporary data directory: %s", directory) try: shutil.rmtree(directory) except: # pylint: disable=W0702 log.exception("Failed to delete, skipping") # Exit log.info("Exiting")
def load_modules(context): """ Load and enable platform modules """ # module_map = dict() # module_name -> (metadata, loader) # for module_name in storage.list_modules(context.settings): log.info("Found module: %s", module_name) module_data = storage.get_module(context.settings, module_name) if not module_data: log.error("Failed to get module data, skipping") continue try: # Make loader for this module module_loader = module.DataModuleLoader(module_data) # Load module metadata if "metadata.json" not in module_loader.storage_files: log.error("No module metadata, skipping") continue with module_loader.storage.open("metadata.json", "r") as file: module_metadata = json.load(file) # Add to module map module_map[module_name] = (module_metadata, module_loader) except: # pylint: disable=W0702 log.exception("Failed to prepare module: %s", module_name) # module_order = dependency.resolve_depencies(module_map) log.debug("Module order: %s", module_order) # temporary_data_dirs = list() # for module_name in module_order: log.info("Enabling module: %s", module_name) try: # Get module metadata and loader module_metadata, module_loader = module_map[module_name] log.info( "Initializing module: %s [%s]", module_metadata.get("name", "N/A"), module_metadata.get("version", "N/A"), ) # Extract module data if needed if module_metadata.get("extract", False): module_data_dir = tempfile.mkdtemp() temporary_data_dirs.append(module_data_dir) module_loader.storage.extractall(module_data_dir) module_root_path = os.path.join( module_data_dir, module_metadata.get("module").replace(".", os.path.sep) ) else: module_root_path = None # Import module package sys.meta_path.insert(0, module_loader) importlib.invalidate_caches() module_pkg = importlib.import_module(module_metadata.get("module")) # Make module instance module_obj = module_pkg.Module( settings=storage.get_config(context.settings, module_name), root_path=module_root_path, context=context ) # Initialize module module_obj.init() # Finally done context.module_manager.add_module( module_name, module_root_path, module_metadata, module_obj ) log.info("Initialized module: %s", module_name) except: # pylint: disable=W0702 log.exception("Failed to initialize module: %s", module_name) # return temporary_data_dirs