def enable_development_module(module_name, module_metadata, context):
# Get module metadata and loader
log.info(
"Initializing module: %s [%s]",
module_metadata.get("name", "N/A"),
module_metadata.get("version", "N/A"),
)
# Extract module data if needed
module_data_dir = os.path.join(context.settings["development"]["modules"], module_name)
module_root_path = os.path.join(
module_data_dir, module_metadata.get("module").replace(".", os.path.sep)
)
# Import module package
sys.path.insert(1, module_data_dir)
importlib.invalidate_caches()
module_pkg = importlib.import_module(module_metadata.get("module"))
# Make module instance
module_obj = module_pkg.Module(
settings=storage.get_development_config(context.settings, module_name),
root_path=module_root_path,
context=context
)
# Initialize module
module_obj.init()
# Finally done
context.module_manager.add_module(
module_name, module_root_path, module_metadata, module_obj
)
def init(self):
""" Init module """
log.info('Initializing module auth_oidc')
_, _, root_module = self.context.module_manager.get_module("auth_root")
root_settings = root_module.settings
self.rpc_prefix = root_settings['rpc_manager']['prefix']['mappers']
mappers = dict()
mappers['raw'] = RawMapper(
info_endpoint=self.settings['endpoints']['info'])
mappers['header'] = HeaderMapper(
info_endpoint=self.settings['endpoints']['info'],
mapper_settings=self.settings['header'],
access_denied_endpoint=root_settings['endpoints']['access_denied'])
mappers['json'] = JsonMapper(
info_endpoint=self.settings['endpoints']['info'],
mapper_settings=self.settings['json'],
access_denied_endpoint=root_settings['endpoints']['access_denied'])
# rpc_manager
for mapper_name, mapper_instance in mappers.items():
self.context.rpc_manager.register_function(
func=mapper_instance.auth,
name=f'{self.rpc_prefix}{mapper_name}')
log.debug(
f'Auth mapper {str(mapper_name)} registered in rpc_manager under name {self.rpc_prefix}{mapper_name}'
)
def init(self):
""" Init module """
log.info('Initializing module auth_root')
self.rpc_prefix = self.settings['rpc_manager']['prefix']['root']
bp = flask.Blueprint( # pylint: disable=C0103
'auth_root',
'plugins.auth_root',
root_path=self.root_path,
url_prefix=
f'{self.context.url_prefix}/{self.settings["endpoints"]["root"]}/')
bp.add_url_rule('/auth', 'auth', self.auth)
bp.add_url_rule('/me', 'me', self.me, methods=['GET'])
bp.add_url_rule('/token', 'token', self.token)
bp.add_url_rule('/login', 'login', self.login)
bp.add_url_rule('/logout', 'logout', self.logout)
# Register in app
self.context.app.register_blueprint(bp)
# rpc_manager
self.context.rpc_manager.register_function(
push_kwargs(
rpc_manager=self.context.rpc_manager,
rpc_prefix=self.rpc_prefix,
rpc_timeout=int(
self.settings['rpc_manager']['timeout']))(check_auth),
name=f'{self.rpc_prefix}check_auth')
def clone_plugin_dulwich(self, plugin: Plugin):
if not self.git_manager:
raise AttributeError('Git manager not set')
log.info('Plugin clone called %s', plugin)
try:
# git_settings = GitSettings(market_data=self.market_data[plugin.name])
market_data = self.market_data[plugin.name]
except KeyError:
raise CloneError(f'No market data available for plugin {plugin}')
try:
# git_settings = GitSettings(market_data=self.market_data[plugin.name])
auth_data = self.git_config[plugin.name]
except KeyError:
log.info(f'No auth data found for plugin {plugin}. Using default')
try:
auth_data = self.git_config['default']
except KeyError:
raise KeyError('No default config found!')
self.git_manager.clone(source=market_data['source']['url'],
target=plugin.path,
branch=market_data['source'].get(
'branch', 'master'),
depth=market_data['source'].get('depth'),
auth_args_override={
k: v
for k, v in auth_data.items()
if v is not None
})
plugin.reload_metadata()
self.plugins_to_download.add(plugin)
def register_traefik_route(context):
""" Create Traefik route for this Pylon instance """
context.traefik_redis_keys = list()
#
if CORE_DEVELOPMENT_MODE and os.environ.get("WERKZEUG_RUN_MAIN") != "true":
log.info("Running in development mode before reloader is started. Skipping registration")
return
#
traefik_config = context.settings.get("traefik", dict())
if not traefik_config:
log.error("Cannot register route: no traefik config")
return
#
redis_config = traefik_config.get("redis", dict())
if not redis_config:
log.error("Cannot register route: no redis config")
return
#
local_hostname = socket.gethostname()
local_port = context.settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT)
#
node_name = context.node_name
#
if "node_url" in traefik_config:
node_url = traefik_config.get("node_url")
elif "node_hostname" in traefik_config:
node_url = f"http://{traefik_config.get('node_hostname')}:{local_port}"
else:
node_url = f"http://{local_hostname}:{local_port}"
#
log.info("Registering traefik route for node '%s'", node_name)
#
store = StrictRedis(
host=redis_config.get("host", "localhost"),
password=redis_config.get("password", None),
)
#
traefik_rootkey = traefik_config.get("rootkey", "traefik")
traefik_rule = traefik_config.get("rule", "PathPrefix(`/`)")
traefik_entrypoint = traefik_config.get("entrypoint", "http")
#
store.set(f"{traefik_rootkey}/http/routers/{node_name}/rule", traefik_rule)
store.set(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0", traefik_entrypoint)
store.set(f"{traefik_rootkey}/http/routers/{node_name}/service", f"{node_name}")
store.set(f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url", node_url)
#
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/rule")
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/entrypoints/0")
context.traefik_redis_keys.append(f"{traefik_rootkey}/http/routers/{node_name}/service")
context.traefik_redis_keys.append(
f"{traefik_rootkey}/http/services/{node_name}/loadbalancer/servers/0/url"
)
def check_requirements(self):
add_entries(self.plugin_list)
req_status = MappingProxyType({
'safe': defaultdict(list),
'attention': defaultdict(list),
'conflict': defaultdict(list)
})
pending_requirements = defaultdict(list)
for plugin in map(Plugin, self.plugin_list):
log.info('Checking requirements for %s', plugin)
update_pending_requirements(plugin, pending_requirements,
req_status)
resolve_version_conflicts(pending_requirements, req_status)
return req_status
def init(self):
""" Init module """
log.info('Initializing module Market')
Plugin.directory = Path(
self.context.settings['development']['modules'])
me = Plugin(self.name)
me._register_in_global()
self.check_updates()
plugins_to_download = self.plugin_list
try:
plugins_to_download.extend(self.settings['preordered_plugins'])
except (KeyError, TypeError):
...
try:
plugins_to_download.extend(getenv('PREORDERED_PLUGINS').split(','))
except AttributeError:
...
# self.download_plugins(set(plugins_to_download))
GitManagerMixin.git_manager = self.context.git_manager
GitManagerMixin.git_config = self.settings['git_config']
self.clone_plugins(set(plugins_to_download))
req_status = self.check_requirements()
if req_status['attention']:
if self.settings['requirements']['raise_on_attention']:
raise VersionConflict(req_status['attention'])
else:
for i in req_status['attention'].values():
for plugin, requirement in map(
lambda y: (y['plugin'], y['requirement']), i):
plugin.installer(package=requirement)
if req_status['conflict']:
raise VersionConflict(req_status['conflict'])
for i in req_status['safe'].values():
for plugin, requirement in map(
lambda y: (y['plugin'], y['requirement']), i):
plugin.installer(package=requirement)
self.copy_configs()
def init_flask_sessions(context):
""" Enable third-party server-side session storage """
redis_config = context.settings.get("sessions", dict()).get("redis", dict())
#
if redis_config:
session_store = RedisStore(
StrictRedis(
host=redis_config.get("host", "localhost"),
password=redis_config.get("password", None),
)
)
session_prefix = context.settings.get("sessions", dict()).get("prefix", None)
if session_prefix:
session_store = PrefixDecorator(session_prefix, session_store)
log.info("Using redis for session storage")
else:
session_store = DictStore()
log.info("Using memory for session storage")
#
KVSessionExtension(session_store, context.app)
def get_development_module_map(context) -> dict:
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_development_modules(context.settings):
log.info("Found module: %s", module_name)
#
module_path = os.path.join(context.settings["development"]["modules"], module_name)
metadata_path = os.path.join(module_path, "metadata.json")
#
try:
# Make loader for this module
module_loader = None
# Load module metadata
if not os.path.exists(metadata_path):
log.error("No module metadata, skipping")
continue
with open(metadata_path, "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
return module_map
def unregister_traefik_route(context):
""" Delete Traefik route for this Pylon instance """
#
traefik_config = context.settings.get("traefik", dict())
if not traefik_config:
log.error("Cannot unregister route: no traefik config")
return
#
redis_config = traefik_config.get("redis", dict())
if not redis_config:
log.error("Cannot unregister route: no redis config")
return
#
log.info("Unregistering traefik route for node '%s'", context.node_name)
#
store = StrictRedis(
host=redis_config.get("host", "localhost"),
password=redis_config.get("password", None),
)
#
while context.traefik_redis_keys:
key = context.traefik_redis_keys.pop()
store.delete(key)
def deinit(self): # pylint: disable=R0201
""" De-init module """
log.info('De-initializing module Market')
def init(self):
""" Init module """
log.info('Initializing module auth_manager')
_, _, root_module = self.context.module_manager.get_module("auth_root")
root_settings = root_module.settings
self.rpc_prefix = root_settings['rpc_manager']['prefix']['manager']
url_prefix = f'{self.context.url_prefix}/{self.settings["endpoints"]["root"]}/'
BaseResource.set_settings(
self.settings,
rpc_prefix=self.rpc_prefix,
rpc_prefix_root=root_settings['rpc_manager']['prefix']['root'])
BaseResource.set_rpc_manager(self.context.rpc_manager)
add_resource_to_api(self.context.api, UserAPI, f'/user/<string:realm>',
f'/user/<string:realm>/<string:user_id>')
add_resource_to_api(self.context.api, GroupAPI,
f'/group/<string:realm>',
f'/group/<string:realm>/<string:group_id>')
add_resource_to_api(self.context.api,
MembershipAPI,
f'/membership/<string:realm>',
methods=['PUT', 'POST'])
add_resource_to_api(self.context.api,
SubgroupAPI,
f'/subgroup/<string:realm>',
methods=['POST'])
# rpc_manager
# token
self.context.rpc_manager.register_function(
push_kwargs(
url=self.settings['keycloak_urls']['token'],
creds=AuthCreds(
username=self.settings['token_credentials']['username'],
password=self.settings['token_credentials']['password']))(
get_token),
name=f'{self.rpc_prefix}get_token')
# get functions
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['user'])(get_users),
name=f'{self.rpc_prefix}get_users')
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['group'])(get_groups),
name=f'{self.rpc_prefix}get_groups')
# put functions
self.context.rpc_manager.register_function(
put_entity, name=f'{self.rpc_prefix}put_entity')
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['user'])(put_entity),
name=f'{self.rpc_prefix}put_user')
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['group'])(put_entity),
name=f'{self.rpc_prefix}put_group')
# post functions
self.context.rpc_manager.register_function(
post_entity, name=f'{self.rpc_prefix}post_entity')
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['user'])(post_entity),
name=f'{self.rpc_prefix}post_user')
self.context.rpc_manager.register_function(
push_kwargs(
base_url=self.settings['keycloak_urls']['group'])(post_group),
name=f'{self.rpc_prefix}post_group')
# delete functions
self.context.rpc_manager.register_function(
delete_entity, name=f'{self.rpc_prefix}delete_entity')
self.context.rpc_manager.register_function(
push_kwargs(base_url=self.settings['keycloak_urls']['user'])(
delete_entity),
name=f'{self.rpc_prefix}delete_user')
self.context.rpc_manager.register_function(
push_kwargs(base_url=self.settings['keycloak_urls']['group'])(
delete_entity),
name=f'{self.rpc_prefix}delete_group')
# group membership functions
self.context.rpc_manager.register_function(
push_kwargs(user_url=self.settings['keycloak_urls']['user'])(
add_users_to_groups),
name=f'{self.rpc_prefix}add_users_to_groups')
self.context.rpc_manager.register_function(
push_kwargs(user_url=self.settings['keycloak_urls']['user'])(
expel_users_from_groups),
name=f'{self.rpc_prefix}expel_users_from_groups')
# subgroup functions
self.context.rpc_manager.register_function(
push_kwargs(base_url=self.settings['keycloak_urls']['group'])(
add_subgroup),
name=f'{self.rpc_prefix}add_subgroup')
# blueprint endpoints
bp = flask.Blueprint('auth_manager',
'plugins.auth_manager',
root_path=self.root_path,
url_prefix=url_prefix)
bp.add_url_rule('/clear_token',
'clear_token',
self.clear_token,
methods=['GET'])
# Register in app
self.context.app.register_blueprint(bp)
def deinit(self): # pylint: disable=R0201
""" De-init module """
log.info('De-initializing module auth_manager')
def main(): # pylint: disable=R0912,R0914,R0915
""" Entry point """
# Register signal handling
signal.signal(signal.SIGTERM, signal_sigterm)
# Enable logging
enable_logging()
# Say hello
log.info("Starting plugin-based Galloper core")
# Make context holder
context = Context()
# Load settings from seed
log.info("Loading and parsing settings")
settings = load_settings()
if not settings:
log.error("Settings are empty or invalid. Exiting")
os._exit(1) # pylint: disable=W0212
context.settings = settings
# Save global node name
context.node_name = settings.get("server", dict()).get("name", socket.gethostname())
# Enable Loki logging if requested in config
enable_loki_logging(context)
# Register provider for template and resource loading from modules
pkg_resources.register_loader_type(module.DataModuleLoader, module.DataModuleProvider)
# Make ModuleManager instance
module_manager = module.ModuleManager(settings)
context.module_manager = module_manager
# Make EventManager instance
event_manager = event.EventManager(context)
context.event_manager = event_manager
# Make app instance
log.info("Creating Flask application")
app = flask.Flask("project")
api = Api(app, catch_all_404s=True)
if settings.get("server", dict()).get("proxy", False):
app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
context.app = app
context.api = api
# Set application settings
app.config["CONTEXT"] = context
app.config.from_mapping(settings.get("application", dict()))
# Save global URL prefix to context. May merge with traefik rule in future
context.url_prefix = settings.get("server", dict()).get("path", "/")
while context.url_prefix.endswith("/"):
context.url_prefix = context.url_prefix[:-1]
# Enable server-side sessions
init_flask_sessions(context)
# Make RpcManager instance
rpc_manager = rpc.RpcManager(context)
context.rpc_manager = rpc_manager
# Make SlotManager instance
slot_manager = slot.SlotManager(context)
context.slot_manager = slot_manager
app.context_processor(slot.template_slot_processor(context))
# Load and initialize modules
if not CORE_DEVELOPMENT_MODE:
temporary_data_dirs = load_modules(context)
else:
temporary_data_dirs = load_development_modules(context)
# Register Traefik route via Redis KV
register_traefik_route(context)
# Run WSGI server
try:
if not CORE_DEVELOPMENT_MODE:
log.info("Starting WSGI server")
http_server = WSGIServer(
(
settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT)
),
app
)
http_server.serve_forever()
else:
log.info("Starting Flask server")
app.run(
host=settings.get("server", dict()).get("host", constants.SERVER_DEFAULT_HOST),
port=settings.get("server", dict()).get("port", constants.SERVER_DEFAULT_PORT),
debug=CORE_DEVELOPMENT_MODE, use_reloader=CORE_DEVELOPMENT_MODE,
)
finally:
log.info("WSGI server stopped")
# Unregister traefik route
unregister_traefik_route(context)
# De-init modules
for module_name in module_manager.modules:
_, _, module_obj = module_manager.get_module(module_name)
module_obj.deinit()
# Delete module data dirs
for directory in temporary_data_dirs:
log.info("Deleting temporary data directory: %s", directory)
try:
shutil.rmtree(directory)
except: # pylint: disable=W0702
log.exception("Failed to delete, skipping")
# Exit
log.info("Exiting")
def load_development_modules(context):
""" Load and enable platform modules in development mode """
#
if os.environ.get("WERKZEUG_RUN_MAIN") != "true":
log.info("Running in development mode before reloader is started. Skipping module loading")
return list()
log.info("Using module dir: %s", context.settings["development"]["modules"])
module_map = get_development_module_map(context)
log.info("Enabling module: Market")
try:
module_metadata, _ = module_map.pop('market')
enable_development_module('market', module_metadata, context=context)
log.info("Initialized module: Market")
module_map = get_development_module_map(context)
del module_map['market']
except: # pylint: disable=W0702
log.exception("Failed to initialize module: Market")
module_order = dependency.resolve_depencies(module_map)
log.debug("Module order: %s", module_order)
temporary_data_dirs = list()
for module_name in module_order:
log.info("Enabling module: %s", module_name)
try:
module_metadata, _ = module_map[module_name]
enable_development_module(module_name, module_metadata, context=context)
log.info("Initialized module: %s", module_name)
except: # pylint: disable=W0702
log.exception("Failed to initialize module: %s", module_name)
return temporary_data_dirs
def load_modules(context):
""" Load and enable platform modules """
#
module_map = dict() # module_name -> (metadata, loader)
#
for module_name in storage.list_modules(context.settings):
log.info("Found module: %s", module_name)
module_data = storage.get_module(context.settings, module_name)
if not module_data:
log.error("Failed to get module data, skipping")
continue
try:
# Make loader for this module
module_loader = module.DataModuleLoader(module_data)
# Load module metadata
if "metadata.json" not in module_loader.storage_files:
log.error("No module metadata, skipping")
continue
with module_loader.storage.open("metadata.json", "r") as file:
module_metadata = json.load(file)
# Add to module map
module_map[module_name] = (module_metadata, module_loader)
except: # pylint: disable=W0702
log.exception("Failed to prepare module: %s", module_name)
#
module_order = dependency.resolve_depencies(module_map)
log.debug("Module order: %s", module_order)
#
temporary_data_dirs = list()
#
for module_name in module_order:
log.info("Enabling module: %s", module_name)
try:
# Get module metadata and loader
module_metadata, module_loader = module_map[module_name]
log.info(
"Initializing module: %s [%s]",
module_metadata.get("name", "N/A"),
module_metadata.get("version", "N/A"),
)
# Extract module data if needed
if module_metadata.get("extract", False):
module_data_dir = tempfile.mkdtemp()
temporary_data_dirs.append(module_data_dir)
module_loader.storage.extractall(module_data_dir)
module_root_path = os.path.join(
module_data_dir, module_metadata.get("module").replace(".", os.path.sep)
)
else:
module_root_path = None
# Import module package
sys.meta_path.insert(0, module_loader)
importlib.invalidate_caches()
module_pkg = importlib.import_module(module_metadata.get("module"))
# Make module instance
module_obj = module_pkg.Module(
settings=storage.get_config(context.settings, module_name),
root_path=module_root_path,
context=context
)
# Initialize module
module_obj.init()
# Finally done
context.module_manager.add_module(
module_name, module_root_path, module_metadata, module_obj
)
log.info("Initialized module: %s", module_name)
except: # pylint: disable=W0702
log.exception("Failed to initialize module: %s", module_name)
#
return temporary_data_dirs
async def clone_plugin_subprocess(self, plugin: Plugin):
log.info('Plugin clone called %s', plugin)
self.tasks.append(
asyncio.create_task(self.clone_github_repo(plugin),
name=f'Task_clone_repo_{plugin.name}'))
self.plugins_to_download.add(plugin)