def _environment(config_module) -> None:
"""Validates ENVIRONMENT if specified.
ENVIRONMENT must be a dictionary of Strings to Strings. Otherwise it is invalid.
"""
env = getattr(config_module, ConfigKeys.ENVIRONMENT.name, None)
if env is None:
return
if not isinstance(env, dict):
raise PluginValidationError(
f"Invalid {ConfigKeys.ENVIRONMENT.name} entry '{env}': if present it must "
f"be a dict"
)
for key, value in env.items():
if not isinstance(key, str):
raise PluginValidationError(
f"Invalid {ConfigKeys.ENVIRONMENT.name} key '{key}': must be a string"
)
if not isinstance(value, str):
raise PluginValidationError(
f"Invalid {ConfigKeys.ENVIRONMENT.name} value '{value}': must be a string"
)
if key.startswith("BG_"):
raise PluginValidationError(
f"Invalid {ConfigKeys.ENVIRONMENT.name} key '{key}': Can't specify an "
f"environment variable with a 'BG_' prefix as it can mess with "
f"internal Beer-garden machinery. Sorry about that :/"
)
def _args(config_module) -> None:
args = getattr(config_module, ConfigKeys.PLUGIN_ARGS.name, None)
if args is None:
return
if isinstance(args, list):
ConfigLoader._individual_args(args)
elif isinstance(args, dict):
instances = getattr(config_module, ConfigKeys.INSTANCES.name, None)
for instance_name, instance_args in args.items():
if instances is not None and instance_name not in instances:
raise PluginValidationError(
f"{ConfigKeys.PLUGIN_ARGS.name} contains key '{instance_name}' "
f"but that instance is not in {ConfigKeys.INSTANCES.name}"
)
ConfigLoader._individual_args(instance_args)
if instances:
for instance_name in instances:
if instance_name not in args.keys():
raise PluginValidationError(
f"{ConfigKeys.INSTANCES.name} contains '{instance_name}' "
f"but that instance is not in {ConfigKeys.PLUGIN_ARGS.name}"
)
else:
raise PluginValidationError(
f"Invalid {ConfigKeys.PLUGIN_ARGS.name} '{args}': must be a list or dict"
)
def _create_runners(self, plugin_path: Path) -> List[ProcessRunner]:
"""Create and start ProcessRunners for a particular directory
It will use the validator to validate the config.
Args:
plugin_path: The path of the plugin
Returns:
Newly created runner dictionary
"""
config_file = plugin_path / CONFIG_NAME
if not plugin_path:
raise PluginValidationError(f"Plugin path {plugin_path} does not exist")
if not plugin_path.is_dir():
raise PluginValidationError(f"Plugin path {plugin_path} is not a directory")
if not config_file.exists():
raise PluginValidationError(f"Config file {config_file} does not exist")
if not config_file.is_file():
raise PluginValidationError(f"Config file {config_file} is not a file")
try:
plugin_config = ConfigLoader.load(config_file)
except PluginValidationError as ex:
self.logger.error(f"Error loading config for plugin at {plugin_path}: {ex}")
return []
new_runners = []
for instance_name in plugin_config["INSTANCES"]:
runner_id = "".join([choice(string.ascii_letters) for _ in range(10)])
capture_streams = plugin_config.get("CAPTURE_STREAMS")
process_args = self._process_args(plugin_config, instance_name)
process_env = self._environment(
plugin_config, instance_name, plugin_path, runner_id
)
new_runners.append(
ProcessRunner(
runner_id=runner_id,
process_args=process_args,
process_cwd=plugin_path,
process_env=process_env,
capture_streams=capture_streams,
)
)
self._runners += new_runners
for runner in new_runners:
self.logger.debug(f"Starting runner {runner}")
runner.start()
return new_runners
def _individual_args(args) -> None:
"""Validates an individual PLUGIN_ARGS entry"""
if args is None:
return
if not isinstance(args, list):
raise PluginValidationError(
f"Invalid {ConfigKeys.PLUGIN_ARGS.name} entry '{args}': must be a list"
)
for arg in args:
if not isinstance(arg, str):
raise PluginValidationError(
f"Invalid plugin argument '{arg}': must be a string")
def _entry_point(config_module, path: Path) -> None:
"""Validates a plugin's entry point.
An entry point is considered valid if the config has an entry with key
PLUGIN_ENTRY and the value is a path to either a file or the name of a runnable
Python module.
"""
entry_point = getattr(config_module, ConfigKeys.PLUGIN_ENTRY.name, None)
if not entry_point:
return
if (path / entry_point).is_file():
return
entry_parts = entry_point.split(" ")
pkg = entry_parts[1] if entry_parts[0] == "-m" else entry_parts[0]
pkg_path = path / pkg
if (
pkg_path.is_dir()
and (pkg_path / "__init__.py").is_file()
and (pkg_path / "__main__.py").is_file()
):
return
raise PluginValidationError(
f"{ConfigKeys.PLUGIN_ENTRY.name} '{entry_point}' must be a Python file or a "
f"runnable package"
)
def _instances(config_module) -> None:
instances = getattr(config_module, ConfigKeys.INSTANCES.name, None)
if instances is not None and not isinstance(instances, list):
raise PluginValidationError(
f"Invalid {ConfigKeys.INSTANCES.name} entry '{instances}': if present"
" it must be a list")
def _is_valid_plugin_path(path: Optional[Path], known_paths: List[str],
the_logger: logging.Logger) -> bool:
try:
if path is None:
raise PluginValidationError("malformed plugin path, NoneType")
path_parts = path.parts
if len(path_parts) == 0:
raise PluginValidationError("empty path")
if path_parts[-1].startswith("."):
raise PluginValidationError("hidden file")
if not path.exists():
raise PluginValidationError("does not exist")
if not path.is_dir():
raise PluginValidationError("not a directory")
config_file = path / CONFIG_NAME
if not config_file.exists():
raise PluginValidationError(
f"config file {config_file} does not exist")
if not config_file.is_file():
raise PluginValidationError(
f"config file {config_file} is not a file")
return path not in known_paths
except PluginValidationError as plugin_error:
the_logger.warning("Not loading plugin at %s: %s" %
(str(path), str(plugin_error)))
return False
def _normalize(instances, args, max_instances):
"""Normalize the config
Will reconcile the different ways instances and arguments can be specified as
well as determine the correct MAX_INSTANCE value
"""
if isinstance(instances, list) and isinstance(args, dict):
# Fully specified, nothing to translate
pass
elif instances is None and args is None:
instances = ["default"]
args = {"default": None}
elif args is None:
args = {}
for instance_name in instances:
args[instance_name] = None
elif instances is None:
if isinstance(args, list):
instances = ["default"]
args = {"default": args}
elif isinstance(args, dict):
instances = list(args.keys())
else:
raise ValueError(
f"PLUGIN_ARGS must be list or dict, found {type(args)}"
)
elif isinstance(args, list):
temp_args = {}
for instance_name in instances:
temp_args[instance_name] = args
args = temp_args
else:
raise PluginValidationError("Invalid INSTANCES and PLUGIN_ARGS combination")
if max_instances is None:
max_instances = -1
return {
"INSTANCES": instances,
"PLUGIN_ARGS": args,
"MAX_INSTANCES": max_instances,
}
def _process_args(plugin_config, instance_name):
if plugin_config.get("INTERPRETER_PATH"):
process_args = [plugin_config.get("INTERPRETER_PATH")]
else:
process_args = [sys.executable]
if plugin_config.get("PLUGIN_ENTRY"):
process_args += plugin_config["PLUGIN_ENTRY"].split(" ")
elif plugin_config.get("NAME"):
process_args += ["-m", plugin_config["NAME"]]
else:
raise PluginValidationError("Can't generate process args")
plugin_args = plugin_config["PLUGIN_ARGS"].get(instance_name)
if plugin_args:
process_args += plugin_args
return process_args
def _process_args(plugin_config: Dict[str, Any], instance_name: str):
interp_path = plugin_config.get("INTERPRETER_PATH")
process_args = [interp_path
] if interp_path is not None else [sys.executable]
plugin_entry = plugin_config.get("PLUGIN_ENTRY")
if plugin_entry is not None:
process_args += plugin_entry.split(" ")
else:
plugin_name = plugin_config.get("NAME")
if plugin_name is not None:
process_args += ["-m", plugin_name]
else:
raise PluginValidationError("Can't generate process args")
plugin_args = plugin_config["PLUGIN_ARGS"].get(instance_name)
if plugin_args is not None:
process_args += plugin_args
return process_args