def check_enterprise(ctx, modules=[]):
"""Checks if any of the provided modules are Starburst Enterprise features.
If they are, we check that a pointer to an SEP license is provided."""
ctx.logger.log(
"Checking for SEP license for enterprise modules...",
level=ctx.logger.verbose,
)
for module in modules:
enterprise = ctx.modules.data.get(module, {}).get("enterprise", False)
if enterprise:
yaml_path = os.path.join(ctx.minitrino_lib_dir, "docker-compose.yml")
with open(yaml_path) as f:
yaml_file = yaml.load(f, Loader=yaml.FullLoader)
if (
not yaml_file.get("services", False)
.get("trino", False)
.get("volumes", False)
):
raise err.UserError(
f"Module {module} requires a Starburst license. "
f"The license volume in the library's docker-compose.yml "
f"file must be uncommented at: {yaml_path}"
)
if not ctx.env.get_var("STARBURST_LIC_PATH", False):
raise err.UserError(
f"Module {module} requires a Starburst license. "
f"You must provide a path to a Starburst license via the "
f"STARBURST_LIC_PATH environment variable"
)
def get_running_modules(self):
"""Returns dict of running modules (includes container objects and
Docker labels)."""
utils.check_daemon(self._ctx.docker_client)
containers = self._ctx.docker_client.containers.list(
filters={"label": RESOURCE_LABEL}
)
if not containers:
return {}
# Remove Trino container since it isn't a module
for i, container in enumerate(containers):
if container.name == "trino":
del containers[i]
names = []
label_sets = []
for i, container in enumerate(containers):
label_set = {}
for k, v in container.labels.items():
if "catalog-" in v:
names.append(v.lower().strip().replace("catalog-", ""))
elif "security-" in v:
names.append(v.lower().strip().replace("security-", ""))
else:
continue
label_set[k] = v
if not label_set and container.name != "trino":
raise err.UserError(
f"Missing Minitrino labels for container '{container.name}'.",
f"Check this module's 'docker-compose.yml' file and ensure you are "
f"following the documentation on labels.",
)
label_sets.append(label_set)
running = {}
for name, label_set, container in zip(names, label_sets, containers):
if not isinstance(self.data.get(name), dict):
raise err.UserError(
f"Module '{name}' is running, but it is not found "
f"in the library. Was it deleted, or are you pointing "
f"Minitrino to the wrong location?"
)
if not running.get(name, False):
running[name] = self.data[name]
if not running.get("labels", False):
running[name]["labels"] = label_set
if not running.get(name).get("containers", False):
running[name]["containers"] = [container]
else:
running[name]["containers"].append(container)
return running
def check_starburst_ver(ctx):
"""Checks if a proper Starburst version is provided."""
starburst_ver = ctx.env.get_var("STARBURST_VER", "")
error_msg = (f"Provided Starburst version '{starburst_ver}' is invalid. "
f"The provided version must be 354-e or higher.")
try:
starburst_ver_int = int(starburst_ver[0:3])
if starburst_ver_int < 354 or "-e" not in starburst_ver:
raise err.UserError(error_msg)
except:
raise err.UserError(error_msg)
def cli(ctx, modules, json_format, running):
"""Version command for Minitrino."""
utils.check_lib(ctx)
ctx.logger.log("Printing module metadata...")
if modules and not running:
for module in modules:
module_dict = ctx.modules.data.get(module, {})
if not module_dict:
raise err.UserError(
f"Invalid module: {module}",
"Ensure the module you're referencing is in the Minitrino library.",
)
log_info(module, module_dict, json_format)
else:
if running:
for module_key, module_dict in ctx.modules.get_running_modules(
).items():
for i, container in enumerate(module_dict.get(
"containers", {})):
module_dict["containers"][i] = {
"id": container.short_id,
"name": container.name,
"labels": container.labels,
}
log_info(module_key, module_dict, json_format)
else:
for module_key, module_dict in ctx.modules.data.items():
log_info(module_key, module_dict, json_format)
def _parse_library_env(self):
"""Parses the Minitrino library's root `minitrino.env` file. All config from
this file is added to the 'MODULES' section of the environment
dictionary since this file explicitly defines the versions of the module
services."""
env_file = os.path.join(self._ctx.minitrino_lib_dir, "minitrino.env")
if not os.path.isfile(env_file):
raise err.UserError(
f"Library 'minitrino.env' file does not exist at path: {env_file}",
f"Are you pointing to a valid library, and is the minitrino.env file "
f"present in that library?",
)
# Check if modules section was added from Minitrino config file parsing
section = self.env.get("MODULES", None)
if not isinstance(section, dict):
self.env["MODULES"] = {}
with open(env_file, "r") as f:
for env_var in f:
env_var = utils.parse_key_value_pair(env_var, err_type=err.UserError)
if env_var is None:
continue
# Skip if the key exists in any section
if self.get_var(env_var[0], False):
continue
self.env["MODULES"][env_var[0]] = env_var[1]
def validate_name(ctx, name):
"""Validates the chosen filename for correct input."""
for char in name:
if all((char != "_", char != "-", not char.isalnum())):
raise err.UserError(
f"Illegal character found in provided filename: '{char}'. ",
f"Alphanumeric, hyphens, and underscores are allowed. "
f"Rename and retry.",
)
def check_daemon(docker_client):
"""Checks if the Docker daemon is running. If an exception is thrown, it is
handled."""
try:
docker_client.ping()
except Exception as e:
raise err.UserError(
f"Error when pinging the Docker server. Is the Docker daemon running?\n"
f"Error from Docker: {str(e)}",
"You may need to initialize your Docker daemon.",
)
def cli(ctx, modules, no_rollback, docker_native):
"""Provision command for Minitrino. If the resulting docker-compose command
is unsuccessful, the function exits with a non-zero status code."""
utils.check_daemon(ctx.docker_client)
utils.check_lib(ctx)
utils.check_starburst_ver(ctx)
modules = append_running_modules(modules)
check_compatibility(modules)
check_enterprise(modules)
if not modules:
ctx.logger.log(
f"No catalog or security options received. Provisioning "
f"standalone Trino container..."
)
else:
for module in modules:
if not ctx.modules.data.get(module, False):
raise err.UserError(
f"Invalid module: '{module}'. It was not found "
f"in the Minitrino library at {ctx.minitrino_lib_dir}"
)
try:
cmd_chunk = chunk(modules)
# Module env variables shared with compose should be from the modules
# section of environment variables and any extra variables provided by the
# user that didn't fit into any other section
compose_env = ctx.env.get_section("MODULES")
compose_env.update(ctx.env.get_section("EXTRA"))
compose_cmd = build_command(docker_native, compose_env, cmd_chunk)
ctx.cmd_executor.execute_commands(compose_cmd, environment=compose_env)
initialize_containers()
containers_to_restart = execute_bootstraps(modules)
containers_to_restart = append_user_config(containers_to_restart)
check_dup_configs()
restart_containers(containers_to_restart)
ctx.logger.log(f"Environment provisioning complete.")
except Exception as e:
rollback_provision(no_rollback)
utils.handle_exception(e)
def minitrino_lib_dir(self):
"""The directory of the Minitrino library. The directory can be
determined in four ways (this is the order of precedence):
1. Passing `LIB_PATH` to the CLI's `--env` option sets the library
directory for the current command.
2. The `minitrino.cfg` file's `LIB_PATH` variable sets the library
directory if present.
3. The path `~/.minitrino/lib/` is used as the default lib path if the
`LIB_PATH` var is not found.
4. As a last resort, Minitrino will check to see if the library exists
in relation to the positioning of the `components.py` file and
assumes the project is being run out of a cloned repository."""
lib_dir = ""
try:
# Try to get `LIB_path` var - handle exception if `env` attribute is
# not yet set
lib_dir = self.env.get_var("LIB_PATH", "")
except:
pass
if not lib_dir and os.path.isdir(os.path.join(self.minitrino_user_dir, "lib")):
lib_dir = os.path.join(self.minitrino_user_dir, "lib")
elif not lib_dir: # Use repo root, fail if this doesn't exist
lib_dir = Path(os.path.abspath(__file__)).resolve().parents[2]
lib_dir = os.path.join(lib_dir, "lib")
if not os.path.isdir(lib_dir) or not os.path.isfile(
os.path.join(lib_dir, "minitrino.env")
):
raise err.UserError(
"You must provide a path to a compatible Minitrino library.",
f"You can point to a Minitrino library a few different "
f"ways:\n(1) You can set the 'LIB_PATH' variable in your "
f"Minitrino config via the command 'minitrino config'--this "
f"should be placed under the '[CLI]' section.\n(2) You can "
f"pass in 'LIB_PATH' as an environment variable for the current "
f"command, e.g. 'minitrino -e LIB_PATH=<path/to/lib> ...'\n"
f"(3) If the above variable is not found, Minitrino will check "
f"if '~/.minitrino/lib/' is a valid directory.\n(4) "
f"If you are running Minitrino out of a cloned repo, the library "
f"path will be automatically detected without the need to perform "
f"any of the above.",
)
return lib_dir
def check_compatibility(ctx, modules=[]):
"""Checks if any of the provided modules are mutually exclusive of each
other. If they are, a user error is raised."""
for module in modules:
incompatible = ctx.modules.data.get(module, {}).get("incompatibleModules", [])
if not incompatible:
continue
for module_inner in modules:
if (module_inner in incompatible) or (
incompatible[0] == "*" and len(modules) > 1
):
raise err.UserError(
f"Incompatible modules detected. Tried to provision module "
f"'{module_inner}', but found that the module is incompatible "
f"with module '{module}'. Incompatible modules listed for module "
f"'{module}' are: {incompatible}",
f"You can see which modules are incompatible with this module by "
f"running 'minitrino modules -m {module}'",
)
def cli(ctx, modules, name, directory, force, no_scrub):
"""Snapshot command for Minitrino."""
# The snapshot temp files are saved in ~/.minitrino/snapshots/<name>
# regardless of the directory provided. The artifact (tarball) will go
# to either the default directory or the user-provided directory.
utils.check_lib(ctx)
if directory and not os.path.isdir(directory):
raise err.UserError(
f"Cannot save snapshot in nonexistent directory: {directory}",
"Pick any directory that exists, and this will work.",
)
if not directory:
directory = os.path.join(ctx.minitrino_lib_dir, "snapshots")
validate_name(name)
check_exists(name, directory, force)
if modules:
ctx.logger.log(f"Creating snapshot of specified modules...")
snapshot_runner(name, no_scrub, False, modules, directory)
else:
modules = ctx.modules.get_running_modules()
if not modules:
ctx.logger.log(
f"No running Minitrino modules to snapshot. Snapshotting "
f"Trino resources only.",
level=ctx.logger.verbose,
)
else:
ctx.logger.log(f"Creating snapshot of active environment...")
snapshot_runner(name, no_scrub, True, list(modules.keys()), directory)
check_complete(name, directory)
ctx.logger.log(
f"Snapshot complete and saved at path: {os.path.join(directory, name)}.tar.gz"
)
def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file=""):
"""Executes a single bootstrap inside a container. If the
`/opt/minitrino/bootstrap_status.txt` file has the same checksum as the
bootstrap script that is about to be executed, the boostrap script is
skipped.
Returns `False` if the script is not executed and `True` if it is."""
if any((not bootstrap, not container_name, not yaml_file)):
raise utils.handle_missing_param(list(locals().keys()))
bootstrap_file = os.path.join(
os.path.dirname(yaml_file), "resources", "bootstrap", bootstrap
)
if not os.path.isfile(bootstrap_file):
raise err.UserError(
f"Bootstrap file does not exist at location: {bootstrap_file}",
"Check this module in the library to ensure the bootstrap script is present.",
)
# Add executable permissions to bootstrap
st = os.stat(bootstrap_file)
os.chmod(
bootstrap_file,
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
)
bootstrap_checksum = hashlib.md5(open(bootstrap_file, "rb").read()).hexdigest()
container = ctx.docker_client.containers.get(container_name)
# Check if this script has already been executed
output = ctx.cmd_executor.execute_commands(
"cat /opt/minitrino/bootstrap_status.txt",
suppress_output=True,
container=container,
trigger_error=False,
)
if f"{bootstrap_checksum}" in output[0].get("output", ""):
ctx.logger.log(
f"Bootstrap already executed in container '{container_name}'. Skipping.",
level=ctx.logger.verbose,
)
return False
ctx.logger.log(
f"Executing bootstrap script in container '{container_name}'...",
level=ctx.logger.verbose,
)
ctx.cmd_executor.execute_commands(
f"docker cp {bootstrap_file} {container_name}:/tmp/"
)
# Record executed file checksum
ctx.cmd_executor.execute_commands(
f"/tmp/{os.path.basename(bootstrap_file)}",
f'bash -c "echo {bootstrap_checksum} >> /opt/minitrino/bootstrap_status.txt"',
container=container,
)
ctx.logger.log(
f"Successfully executed bootstrap script in container '{container_name}'.",
level=ctx.logger.verbose,
)
return True
def _load_modules(self):
"""Loads module data during instantiation."""
self._ctx.logger.log("Loading modules...", level=self._ctx.logger.verbose)
modules_dir = os.path.join(self._ctx.minitrino_lib_dir, MODULE_ROOT)
if not os.path.isdir(modules_dir):
raise err.MinitrinoError(
f"Path is not a directory: {modules_dir}. "
f"Are you pointing to a compatible Minitrino library?"
)
# Loop through both catalog and security modules
sections = [
os.path.join(modules_dir, MODULE_CATALOG),
os.path.join(modules_dir, MODULE_SECURITY),
]
for section_dir in sections:
for _dir in os.listdir(section_dir):
module_dir = os.path.join(section_dir, _dir)
if not os.path.isdir(module_dir):
self._ctx.logger.log(
f"Skipping file (expected a directory, not a file) "
f"at path: {module_dir}",
level=self._ctx.logger.verbose,
)
continue
# List inner-module files
module_files = os.listdir(module_dir)
yaml_basename = f"{os.path.basename(module_dir)}.yml"
if not yaml_basename in module_files:
raise err.UserError(
f"Missing Docker Compose file in module directory {_dir}. "
f"Expected file to be present: {yaml_basename}",
hint_msg="Check this module in your library to ensure it is properly constructed.",
)
# Module dir and YAML exist, add to modules
module_name = os.path.basename(module_dir)
self.data[module_name] = {}
self.data[module_name]["type"] = os.path.basename(section_dir)
self.data[module_name]["module_dir"] = module_dir
# Add YAML file path
yaml_file = os.path.join(module_dir, yaml_basename)
self.data[module_name]["yaml_file"] = yaml_file
# Add YAML dict
with open(yaml_file) as f:
self.data[module_name]["yaml_dict"] = yaml.load(
f, Loader=yaml.FullLoader
)
# Get metadata.json if present
json_basename = "metadata.json"
json_file = os.path.join(module_dir, json_basename)
metadata = {}
if os.path.isfile(json_file):
with open(json_file) as f:
metadata = json.load(f)
else:
self._ctx.logger.log(
f"No JSON metadata file for module '{module_name}'. "
f"Will not load metadata for module.",
level=self._ctx.logger.verbose,
)
for k, v in metadata.items():
self.data[module_name][k] = v