def get_running_modules(self):
"""Returns dict of running modules (includes container objects and
Docker labels)."""
utils.check_daemon(self._ctx.docker_client)
containers = self._ctx.docker_client.containers.list(
filters={"label": RESOURCE_LABEL})
if not containers:
return {}
# Remove Presto container since it isn't a module
for i, container in enumerate(containers):
if container.name == "presto":
del containers[i]
names = []
label_sets = []
for i, container in enumerate(containers):
label_set = {}
for k, v in container.labels.items():
if "catalog-" in v:
names.append(v.lower().strip().replace("catalog-", ""))
elif "security-" in v:
names.append(v.lower().strip().replace("security-", ""))
else:
continue
label_set[k] = v
if not label_set and container.name != "presto":
raise err.UserError(
f"Missing Minipresto labels for container '{container.name}'.",
f"Check this module's 'docker-compose.yml' file and ensure you are "
f"following the documentation on labels.",
)
label_sets.append(label_set)
running = {}
for name, label_set, container in zip(names, label_sets, containers):
if not isinstance(self.data.get(name), dict):
raise err.UserError(
f"Module '{name}' is running, but it is not found "
f"in the library. Was it deleted, or are you pointing "
f"Minipresto to the wrong location?")
if not running.get(name, False):
running[name] = self.data[name]
if not running.get("labels", False):
running[name]["labels"] = label_set
if not running.get(name).get("containers", False):
running[name]["containers"] = [container]
else:
running[name]["containers"].append(container)
return running
def cli(ctx, modules, json_format, running):
"""Version command for Minipresto."""
utils.check_lib(ctx)
ctx.logger.log("Printing module metadata...")
if modules and not running:
for module in modules:
module_dict = ctx.modules.data.get(module, {})
if not module_dict:
raise err.UserError(
f"Invalid module: {module}",
"Ensure the module you're referencing is in the Minipresto library.",
)
log_info(module, module_dict, json_format)
else:
if running:
for module_key, module_dict in ctx.modules.get_running_modules(
).items():
for i, container in enumerate(module_dict.get(
"containers", {})):
module_dict["containers"][i] = {
"id": container.short_id,
"name": container.name,
"labels": container.labels,
}
log_info(module_key, module_dict, json_format)
else:
for module_key, module_dict in ctx.modules.data.items():
log_info(module_key, module_dict, json_format)
def _parse_library_env(self):
"""Parses the Minipresto library's root `minipresto.env` file. All config from
this file is added to the 'MODULES' section of the environment
dictionary since this file explicitly defines the versions of the module
services."""
env_file = os.path.join(self._ctx.minipresto_lib_dir, "minipresto.env")
if not os.path.isfile(env_file):
raise err.UserError(
f"Library 'minipresto.env' file does not exist at path: {env_file}",
f"Are you pointing to a valid library, and is the minipresto.env file "
f"present in that library?",
)
# Check if modules section was added from Minipresto config file parsing
section = self.env.get("MODULES", None)
if not isinstance(section, dict):
self.env["MODULES"] = {}
with open(env_file, "r") as f:
for env_var in f:
env_var = utils.parse_key_value_pair(env_var,
err_type=err.UserError)
if env_var is None:
continue
# Skip if the key exists in any section
if self.get_var(env_var[0], False):
continue
self.env["MODULES"][env_var[0]] = env_var[1]
def validate_name(ctx, name):
"""Validates the chosen filename for correct input."""
for char in name:
if all((char != "_", char != "-", not char.isalnum())):
raise err.UserError(
f"Illegal character found in provided filename: '{char}'. ",
f"Alphanumeric, hyphens, and underscores are allowed. "
f"Rename and retry.",
)
def check_daemon(docker_client):
"""Checks if the Docker daemon is running. If an exception is thrown, it is
handled."""
try:
docker_client.ping()
except Exception as e:
raise err.UserError(
f"Error when pinging the Docker server. Is the Docker daemon running?\n"
f"Error from Docker: {str(e)}",
"You may need to initialize your Docker daemon.",
)
def minipresto_lib_dir(self):
"""The directory of the Minipresto library. The directory can be
determined in four ways (this is the order of precedence):
1. Passing `LIB_PATH` to the CLI's `--env` option sets the library
directory for the current command.
2. The `minipresto.cfg` file's `LIB_PATH` variable sets the library
directory if present.
3. The path `~/.minipresto/lib/` is used as the default lib path if the
`LIB_PATH` var is not found.
4. As a last resort, Minipresto will check to see if the library exists
in relation to the positioning of the `components.py` file and
assumes the project is being run out of a cloned repository."""
lib_dir = ""
try:
# Try to get `LIB_path` var - handle exception if `env` attribute is
# not yet set
lib_dir = self.env.get_var("LIB_PATH", "")
except:
pass
if not lib_dir and os.path.isdir(
os.path.join(self.minipresto_user_dir, "lib")):
lib_dir = os.path.join(self.minipresto_user_dir, "lib")
elif not lib_dir: # Use repo root, fail if this doesn't exist
lib_dir = Path(os.path.abspath(__file__)).resolve().parents[2]
lib_dir = os.path.join(lib_dir, "lib")
if not os.path.isdir(lib_dir) or not os.path.isfile(
os.path.join(lib_dir, "minipresto.env")):
raise err.UserError(
"You must provide a path to a compatible Minipresto library.",
f"You can point to a Minipresto library a few different "
f"ways:\n(1) You can set the 'LIB_PATH' variable in your "
f"Minipresto config via the command 'minipresto config'--this "
f"should be placed under the '[CLI]' section.\n(2) You can "
f"pass in 'LIB_PATH' as an environment variable for the current "
f"command, e.g. 'minipresto -e LIB_PATH=<path/to/lib> ...'\n"
f"(3) If the above variable is not found, Minipresto will check "
f"if '~/.minipresto/lib/' is a valid directory.\n(4) "
f"If you are running Minipresto out of a cloned repo, the library "
f"path will be automatically detected without the need to perform "
f"any of the above.",
)
return lib_dir
def cli(ctx, modules, no_rollback, docker_native):
"""Provision command for Minipresto. If the resulting docker-compose command
is unsuccessful, the function exits with a non-zero status code."""
utils.check_daemon(ctx.docker_client)
utils.check_lib(ctx)
modules = append_running_modules(modules)
check_compatibility(modules)
if not modules:
ctx.logger.log(
f"No catalog or security options received. Provisioning "
f"standalone Presto container...")
else:
for module in modules:
if not ctx.modules.data.get(module, False):
raise err.UserError(
f"Invalid module: '{module}'. It was not found "
f"in the Minipresto library at {ctx.minipresto_lib_dir}")
try:
cmd_chunk = chunk(modules)
# Module env variables shared with compose should be from the modules
# section of environment variables and any extra variables provided by the
# user that didn't fit into any other section
compose_env = ctx.env.get_section("MODULES")
compose_env.update(ctx.env.get_section("EXTRA"))
compose_cmd = build_command(docker_native, compose_env, cmd_chunk)
ctx.cmd_executor.execute_commands(compose_cmd, environment=compose_env)
initialize_containers()
containers_to_restart = execute_bootstraps(modules)
containers_to_restart = append_user_config(containers_to_restart)
check_dup_configs()
restart_containers(containers_to_restart)
ctx.logger.log(f"Environment provisioning complete.")
except Exception as e:
rollback_provision(no_rollback)
utils.handle_exception(e)
def check_compatibility(ctx, modules=[]):
"""Checks if any of the provided modules are mutually exclusive of each
other. If they are, a user error is raised."""
for module in modules:
incompatible = ctx.modules.data.get(module,
{}).get("incompatible_modules", [])
if not incompatible:
continue
for module_inner in modules:
if (module_inner in incompatible) or (incompatible[0] == "*"
and len(modules) > 1):
raise err.UserError(
f"Incompatible modules detected. Tried to provision module "
f"'{module_inner}', but found that the module is incompatible "
f"with module '{module}'. Incompatible modules listed for module "
f"'{module}' are: {incompatible}",
f"You can see which modules are incompatible with this module by "
f"running 'minipresto modules -m {module}'",
)
def cli(ctx, modules, name, directory, force, no_scrub):
"""Snapshot command for Minipresto."""
# The snapshot temp files are saved in ~/.minipresto/snapshots/<name>
# regardless of the directory provided. The artifact (tarball) will go
# to either the default directory or the user-provided directory.
utils.check_lib(ctx)
if directory and not os.path.isdir(directory):
raise err.UserError(
f"Cannot save snapshot in nonexistent directory: {directory}",
"Pick any directory that exists, and this will work.",
)
if not directory:
directory = os.path.join(ctx.minipresto_lib_dir, "snapshots")
validate_name(name)
check_exists(name, directory, force)
if modules:
ctx.logger.log(f"Creating snapshot of specified modules...")
snapshot_runner(name, no_scrub, False, modules, directory)
else:
modules = ctx.modules.get_running_modules()
if not modules:
ctx.logger.log(
f"No running Minipresto modules to snapshot. Snapshotting "
f"Presto resources only.",
level=ctx.logger.verbose,
)
else:
ctx.logger.log(f"Creating snapshot of active environment...")
snapshot_runner(name, no_scrub, True, list(modules.keys()), directory)
check_complete(name, directory)
ctx.logger.log(
f"Snapshot complete and saved at path: {os.path.join(directory, name)}.tar.gz"
)
def execute_container_bootstrap(ctx,
bootstrap="",
container_name="",
yaml_file=""):
"""Executes a single bootstrap inside a container. If the
`/opt/minipresto/bootstrap_status.txt` file has the same checksum as the
bootstrap script that is about to be executed, the boostrap script is
skipped.
Returns `False` if the script is not executed and `True` if it is."""
if any((not bootstrap, not container_name, not yaml_file)):
raise utils.handle_missing_param(list(locals().keys()))
bootstrap_file = os.path.join(os.path.dirname(yaml_file), "resources",
"bootstrap", bootstrap)
if not os.path.isfile(bootstrap_file):
raise err.UserError(
f"Bootstrap file does not exist at location: {bootstrap_file}",
"Check this module in the library to ensure the bootstrap script is present.",
)
# Add executable permissions to bootstrap
st = os.stat(bootstrap_file)
os.chmod(
bootstrap_file,
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
)
bootstrap_checksum = hashlib.md5(open(bootstrap_file,
"rb").read()).hexdigest()
container = ctx.docker_client.containers.get(container_name)
# Check if this script has already been executed
output = ctx.cmd_executor.execute_commands(
"cat /opt/minipresto/bootstrap_status.txt",
suppress_output=True,
container=container,
trigger_error=False,
)
if f"{bootstrap_checksum}" in output[0].get("output", ""):
ctx.logger.log(
f"Bootstrap already executed in container '{container_name}'. Skipping.",
level=ctx.logger.verbose,
)
return False
ctx.logger.log(
f"Executing bootstrap script in container '{container_name}'...",
level=ctx.logger.verbose,
)
ctx.cmd_executor.execute_commands(
f"docker cp {bootstrap_file} {container_name}:/tmp/")
# Record executed file checksum
ctx.cmd_executor.execute_commands(
f"/tmp/{os.path.basename(bootstrap_file)}",
f'bash -c "echo {bootstrap_checksum} >> /opt/minipresto/bootstrap_status.txt"',
container=container,
)
ctx.logger.log(
f"Successfully executed bootstrap script in container '{container_name}'.",
level=ctx.logger.verbose,
)
return True
def _load_modules(self):
"""Loads module data during instantiation."""
self._ctx.logger.log("Loading modules...",
level=self._ctx.logger.verbose)
modules_dir = os.path.join(self._ctx.minipresto_lib_dir, MODULE_ROOT)
if not os.path.isdir(modules_dir):
raise err.MiniprestoError(
f"Path is not a directory: {modules_dir}. "
f"Are you pointing to a compatible Minipresto library?")
# Loop through both catalog and security modules
sections = [
os.path.join(modules_dir, MODULE_CATALOG),
os.path.join(modules_dir, MODULE_SECURITY),
]
for section_dir in sections:
for _dir in os.listdir(section_dir):
module_dir = os.path.join(section_dir, _dir)
if not os.path.isdir(module_dir):
self._ctx.logger.log(
f"Skipping file (expected a directory, not a file) "
f"at path: {module_dir}",
level=self._ctx.logger.verbose,
)
continue
# List inner-module files
module_files = os.listdir(module_dir)
yaml_basename = f"{os.path.basename(module_dir)}.yml"
if not yaml_basename in module_files:
raise err.UserError(
f"Missing Docker Compose file in module directory {_dir}. "
f"Expected file to be present: {yaml_basename}",
hint_msg=
"Check this module in your library to ensure it is properly constructed.",
)
# Module dir and YAML exist, add to modules
module_name = os.path.basename(module_dir)
self.data[module_name] = {}
self.data[module_name]["type"] = os.path.basename(section_dir)
self.data[module_name]["module_dir"] = module_dir
# Add YAML file path
yaml_file = os.path.join(module_dir, yaml_basename)
self.data[module_name]["yaml_file"] = yaml_file
# Add YAML dict
with open(yaml_file) as f:
self.data[module_name]["yaml_dict"] = yaml.load(
f, Loader=yaml.FullLoader)
# Get metadata.json if present
json_basename = "metadata.json"
json_file = os.path.join(module_dir, json_basename)
metadata = {}
if os.path.isfile(json_file):
with open(json_file) as f:
metadata = json.load(f)
else:
self._ctx.logger.log(
f"No JSON metadata file for module '{module_name}'. "
f"Will not load metadata for module.",
level=self._ctx.logger.verbose,
)
self.data[module_name]["description"] = metadata.get(
"description", "No module description provided.")
self.data[module_name]["incompatible_modules"] = metadata.get(
"incompatible_modules", [])