def auth(self):
if "X-Forwarded-Uri" in request.headers:
if request.headers["X-Forwarded-Uri"].startswith("/static") and \
any(request.headers["X-Forwarded-Uri"].endswith(res) for res in [".ico", ".js", ".css"]):
return make_response("OK")
# Check if need to login
target = request.args.get("target")
scope = request.args.get("scope")
for header in ("X-Forwarded-Proto", "X-Forwarded-Host",
"X-Forwarded-Port", "X-Forwarded-Uri"):
if header in request.headers:
session[header] = request.headers[header]
if "Authorization" in request.headers:
return self.handle_auth(
auth_header=request.headers.get("Authorization", ""))
if "X-Forwarded-Uri" in request.headers and "/api/v1" in "X-Forwarded-Uri":
if "Referer" in request.headers and "/api/v1" not in "Referer":
session["X-Forwarded-Uri"] = request.headers["Referer"]
else:
session["X-Forwarded-Uri"] = request.base_url
if not session.get(
"auth_attributes") or session["auth_attributes"]["exp"] < int(
time()):
return redirect(self.settings["login_handler_url"], 302)
if not session.get("auth",
False) and not self.settings["disable_auth"]:
# Redirect to login
return redirect(
self.settings.get(
"auth_redirect",
f"{request.base_url}{request.script_root}/login"))
if target is None:
target = "raw"
# Map auth response
response = make_response("OK")
try:
self.context.rpc_manager.call_function_with_timeout(
func='{prefix}{key}'.format(
prefix=self.settings['rpc_manager']['prefix']['mappers'],
key=target.lower()),
timeout=int(self.settings['rpc_manager']['timeout']),
response=response,
scope=scope)
except Empty:
log.error(f'Cannot find mapper for auth_key {target}')
return make_response("KO", 403)
except (AttributeError, TypeError):
from traceback import format_exc
log.error(f"Failed to map auth data {format_exc()}")
except NameError:
return redirect(self.settings["login_default_redirect_url"])
return response
def register_traefik_route(context):
""" Create Traefik route for this Pylon instance """
context.traefik_redis_keys = list()
#
if CORE_DEVELOPMENT_MODE and os.environ.get("WERKZEUG_RUN_MAIN") != "true":
log.info("Running in development mode before reloader is started. Skipping registration")
return
#
traefik_config = context.settings.get("traefik", dict())
if not traefik_config:
log.error("Cannot register route: no traefik config")
return
#
redis_config = traefik_config.get("redis", dict())
if not redis_config:
log.error("Cannot register route: no redis config")
return
#
local_hostname = socket.gethostname()
local_port = context.settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT)
#
node_name = context.node_name
#
if "node_url" in traefik_config:
node_url = traefik_config.get("node_url")
elif "node_hostname" in traefik_config:
node_url = f"http://{traefik_config.get('node_hostname')}:{local_port}"
else:
node_url = f"http://{local_hostname}:{local_port}"
#
log.info("Registering traefik route for node '%s'", node_name)
#
store = StrictRedis(
host=redis_config.get("host", "localhost"),
password=redis_config.get("password", None),
)
#
traefik_rootkey = traefik_config.get("rootkey", "traefik")
traefik_rule = traefik_config.get("rule", "PathPrefix(`/`)")
traefik_entrypoint = traefik_config.get("entrypoint", "http")
#
store.set(f"{traefik_rootkey}/http/routers/{node_name}/rule", traefik_rule)
store.set(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0", traefik_entrypoint)
store.set(f"{traefik_rootkey}/http/routers/{node_name}/service", f"{node_name}")
store.set(f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url", node_url)
#
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/rule")
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0")
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/service")
context.traefik_redis_keys.append(
f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url"
)
def info(self, scope: str = '') -> dict:
""" Map info data """
if scope not in self.mapper_settings:
raise redirect(self.access_denied_endpoint)
auth_info = super(JsonMapper, self).info(scope)
result = {"raw": auth_info}
try:
for key, path in self.mapper_settings[scope].items():
result[key] = jsonpath_rw.parse(path).find(auth_info)[0].value
except: # pylint: disable=W0702
from traceback import format_exc
log.error(f"Failed to set scope data {format_exc()}")
return result
def info(scope, auth_settings: dict = None):
""" Map info data """
if scope not in auth_settings["mappers"]["json"]:
raise redirect(auth_settings["endpoints"]["access_denied"])
auth_info = raw.info()
result = {"raw": auth_info}
try:
for key, path in auth_settings["mappers"]["json"][scope].items():
result[key] = jsonpath_rw.parse(path).find(auth_info)[0].value
except: # pylint: disable=W0702
from traceback import format_exc
log.error(f"Failed to set scope data {format_exc()}")
return result
def _on_unregister_slot_callback(self, context, event_name, event_payload):
_ = context, event_name
#
for key in ["slot", "callback"]:
if key not in event_payload:
log.error("Invalid slot unregistration data, skipping")
return
#
if event_payload["slot"] not in self.callbacks:
return
if event_payload["callback"] not in self.callbacks[
event_payload["slot"]]:
return
self.callbacks[event_payload["slot"]].remove(event_payload["callback"])
def auth(scope, response, auth_settings: dict = None):
""" Map auth data """
if scope not in auth_settings["mappers"]["header"]:
raise redirect(auth_settings["endpoints"]["access_denied"])
response = raw.auth(scope, response) # Set "raw" headers too
auth_info = info(scope)
if f"/{scope}" not in auth_info["auth_attributes"]["groups"]:
raise NameError(f"User is not a memeber of {scope} group")
try:
for header, path in auth_settings["mappers"]["header"][scope].items():
response.headers[header] = jsonpath_rw.parse(path).find(
auth_info)[0].value
except:
log.error("Failed to set scope headers")
return response
def auth(self, response: Response, scope: str = '') -> Response:
""" Map auth data """
if scope not in self.mapper_settings:
raise redirect(self.access_denied_endpoint)
response = super(HeaderMapper,
self).auth(response, scope) # Set "raw" headers too
auth_info = self.info(scope)
if f"/{scope}" not in auth_info["auth_attributes"]["groups"]:
raise NameError(f"User is not a memeber of {scope} group")
try:
for header, path in self.mapper_settings[scope].items():
response.headers[header] = jsonpath_rw.parse(path).find(
auth_info)[0].value
except Exception as e:
log.error(f"Failed to set scope headers: {str(e)}")
return response
async def download_plugin_zip(url: str, plugin: Plugin) -> None:
async with ClientSession() as client:
async with client.get(url) as response:
if response.ok:
tmp_bytes = BytesIO(await response.read())
zip_file = ZipFile(tmp_bytes)
zip_file.extractall(plugin.directory)
src = Path(plugin.directory, zip_file.namelist()[0])
try:
src.rename(plugin.path)
except (FileExistsError, OSError):
shutil.copytree(src, plugin.path, dirs_exist_ok=True)
shutil.rmtree(src)
plugin.reload_metadata()
else:
log.error('Response failed with status %s',
response.status)
def _on_register_slot_callback(self, context, event_name, event_payload):
_ = context, event_name
#
for key in ["slot", "callback"]:
if key not in event_payload:
log.error("Invalid slot registration data, skipping")
return
#
log.debug("New slot callback: %s - %s", event_payload["slot"],
event_payload["callback"])
#
if event_payload["slot"] not in self.callbacks:
self.callbacks[event_payload["slot"]] = list()
if event_payload["callback"] not in self.callbacks[
event_payload["slot"]]:
self.callbacks[event_payload["slot"]].append(
event_payload["callback"])
def resolve_depencies(module_map):
""" Resolve depencies """
# Check required depencies
for module_name, module_data in module_map.items():
for dependency in module_data[0].get("depends_on", list()):
if dependency not in module_map:
log.error("Dependency %s not present (required by %s)",
dependency, module_name)
raise RuntimeError("Required dependency not present")
# Walk modules
module_order = list()
visited_modules = set()
for module_name in module_map:
if module_name not in module_order:
_walk_module_depencies(module_name, module_map, module_order,
visited_modules)
# Return correct order
return module_order
def _walk_module_depencies(module_name, module_map, module_order,
visited_modules):
# Collect depencies
depencies = list()
depencies.extend(module_map[module_name][0].get("depends_on", list()))
for optional_dependency in module_map[module_name][0].get(
"init_after", list()):
if optional_dependency in module_map:
depencies.append(optional_dependency)
# Resolve
visited_modules.add(module_name)
for dependency in depencies:
if dependency not in module_order:
if dependency in visited_modules:
log.error("Circular dependency (%s <-> %s)", dependency,
module_name)
raise RuntimeError("Circular dependency present")
_walk_module_depencies(dependency, module_map, module_order,
visited_modules)
# Add to resolved order
module_order.append(module_name)
def check_auth(auth_header: str, *, rpc_prefix: str, rpc_timeout: int,
rpc_manager: RpcManager) -> Response:
try:
auth_key, auth_value = auth_header.strip().split(" ")
except ValueError:
return make_response("KO", 401)
if check_auth_token(auth_header=auth_header):
return make_response("OK", 200)
try:
auth_result = rpc_manager.call_function_with_timeout(
func='{prefix}{key}'.format(prefix=rpc_prefix,
key=auth_key.lower()),
timeout=rpc_timeout,
auth_value=auth_value)
except Empty:
log.error(f'Cannot find handler for auth_key {auth_key.lower()}')
return make_response("KO", 401)
except binascii.Error:
log.error('Incorrect auth data received')
return make_response('KO', 403)
return make_response(*auth_result)
def get_development_module_map(context) -> dict:
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_development_modules(context.settings):
log.info("Found module: %s", module_name)
#
module_path = os.path.join(context.settings["development"]["modules"], module_name)
metadata_path = os.path.join(module_path, "metadata.json")
#
try:
# Make loader for this module
module_loader = None
# Load module metadata
if not os.path.exists(metadata_path):
log.error("No module metadata, skipping")
continue
with open(metadata_path, "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
return module_map
def unregister_traefik_route(context):
""" Delete Traefik route for this Pylon instance """
#
traefik_config = context.settings.get("traefik", dict())
if not traefik_config:
log.error("Cannot unregister route: no traefik config")
return
#
redis_config = traefik_config.get("redis", dict())
if not redis_config:
log.error("Cannot unregister route: no redis config")
return
#
log.info("Unregistering traefik route for node '%s'", context.node_name)
#
store = StrictRedis(
host=redis_config.get("host", "localhost"),
password=redis_config.get("password", None),
)
#
while context.traefik_redis_keys:
key = context.traefik_redis_keys.pop()
store.delete(key)
def main(): # pylint: disable=R0912,R0914,R0915
""" Entry point """
# Register signal handling
signal.signal(signal.SIGTERM, signal_sigterm)
# Enable logging
enable_logging()
# Say hello
log.info("Starting plugin-based Galloper core")
# Make context holder
context = Context()
# Load settings from seed
log.info("Loading and parsing settings")
settings = load_settings()
if not settings:
log.error("Settings are empty or invalid. Exiting")
os._exit(1) # pylint: disable=W0212
context.settings = settings
# Save global node name
context.node_name = settings.get("server", dict()).get("name", socket.gethostname())
# Enable Loki logging if requested in config
enable_loki_logging(context)
# Register provider for template and resource loading from modules
pkg_resources.register_loader_type(module.DataModuleLoader, module.DataModuleProvider)
# Make ModuleManager instance
module_manager = module.ModuleManager(settings)
context.module_manager = module_manager
# Make EventManager instance
event_manager = event.EventManager(context)
context.event_manager = event_manager
# Make app instance
log.info("Creating Flask application")
app = flask.Flask("project")
api = Api(app, catch_all_404s=True)
if settings.get("server", dict()).get("proxy", False):
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
context.app = app
context.api = api
# Set application settings
app.config["CONTEXT"] = context
app.config.from_mapping(settings.get("application", dict()))
# Save global URL prefix to context. May merge with traefik rule in future
context.url_prefix = settings.get("server", dict()).get("path", "/")
while context.url_prefix.endswith("/"):
context.url_prefix = context.url_prefix[:-1]
# Enable server-side sessions
init_flask_sessions(context)
# Make RpcManager instance
rpc_manager = rpc.RpcManager(context)
context.rpc_manager = rpc_manager
# Make SlotManager instance
slot_manager = slot.SlotManager(context)
context.slot_manager = slot_manager
app.context_processor(slot.template_slot_processor(context))
# Load and initialize modules
if not CORE_DEVELOPMENT_MODE:
temporary_data_dirs = load_modules(context)
else:
temporary_data_dirs = load_development_modules(context)
# Register Traefik route via Redis KV
register_traefik_route(context)
# Run WSGI server
try:
if not CORE_DEVELOPMENT_MODE:
log.info("Starting WSGI server")
http_server = WSGIServer(
(
settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT)
),
app
)
http_server.serve_forever()
else:
log.info("Starting Flask server")
app.run(
host=settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
port=settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT),
debug=CORE_DEVELOPMENT_MODE, use_reloader=CORE_DEVELOPMENT_MODE,
)
finally:
log.info("WSGI server stopped")
# Unregister traefik route
unregister_traefik_route(context)
# De-init modules
for module_name in module_manager.modules:
_, _, module_obj = module_manager.get_module(module_name)
module_obj.deinit()
# Delete module data dirs
for directory in temporary_data_dirs:
log.info("Deleting temporary data directory: %s", directory)
try:
shutil.rmtree(directory)
except: # pylint: disable=W0702
log.exception("Failed to delete, skipping")
# Exit
log.info("Exiting")
def load_modules(context):
""" Load and enable platform modules """
#
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_modules(context.settings):
log.info("Found module: %s", module_name)
module_data = storage.get_module(context.settings, module_name)
if not module_data:
log.error("Failed to get module data, skipping")
continue
try:
# Make loader for this module
module_loader = module.DataModuleLoader(module_data)
# Load module metadata
if "metadata.json" not in module_loader.storage_files:
log.error("No module metadata, skipping")
continue
with module_loader.storage.open("metadata.json", "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
#
module_order = dependency.resolve_depencies(module_map)
log.debug("Module order: %s", module_order)
#
temporary_data_dirs = list()
#
for module_name in module_order:
log.info("Enabling module: %s", module_name)
try:
# Get module metadata and loader
module_metadata, module_loader = module_map[module_name]
log.info(
"Initializing module: %s [%s]",
module_metadata.get("name", "N/A"),
module_metadata.get("version", "N/A"),
)
# Extract module data if needed
if module_metadata.get("extract", False):
module_data_dir = tempfile.mkdtemp()
temporary_data_dirs.append(module_data_dir)
module_loader.storage.extractall(module_data_dir)
module_root_path = os.path.join(
module_data_dir, module_metadata.get("module").replace(".", os.path.sep)
)
else:
module_root_path = None
# Import module package
sys.meta_path.insert(0, module_loader)
importlib.invalidate_caches()
module_pkg = importlib.import_module(module_metadata.get("module"))
# Make module instance
module_obj = module_pkg.Module(
settings=storage.get_config(context.settings, module_name),
root_path=module_root_path,
context=context
)
# Initialize module
module_obj.init()
# Finally done
context.module_manager.add_module(
module_name, module_root_path, module_metadata, module_obj
)
log.info("Initialized module: %s", module_name)
except: # pylint: disable=W0702
log.exception("Failed to initialize module: %s", module_name)
#
return temporary_data_dirs
