def check_enterprise(ctx, modules=[]): """Checks if any of the provided modules are Starburst Enterprise features. If they are, we check that a pointer to an SEP license is provided.""" ctx.logger.log( "Checking for SEP license for enterprise modules...", level=ctx.logger.verbose, ) for module in modules: enterprise = ctx.modules.data.get(module, {}).get("enterprise", False) if enterprise: yaml_path = os.path.join(ctx.minitrino_lib_dir, "docker-compose.yml") with open(yaml_path) as f: yaml_file = yaml.load(f, Loader=yaml.FullLoader) if ( not yaml_file.get("services", False) .get("trino", False) .get("volumes", False) ): raise err.UserError( f"Module {module} requires a Starburst license. " f"The license volume in the library's docker-compose.yml " f"file must be uncommented at: {yaml_path}" ) if not ctx.env.get_var("STARBURST_LIC_PATH", False): raise err.UserError( f"Module {module} requires a Starburst license. " f"You must provide a path to a Starburst license via the " f"STARBURST_LIC_PATH environment variable" )
def get_running_modules(self): """Returns dict of running modules (includes container objects and Docker labels).""" utils.check_daemon(self._ctx.docker_client) containers = self._ctx.docker_client.containers.list( filters={"label": RESOURCE_LABEL} ) if not containers: return {} # Remove Trino container since it isn't a module for i, container in enumerate(containers): if container.name == "trino": del containers[i] names = [] label_sets = [] for i, container in enumerate(containers): label_set = {} for k, v in container.labels.items(): if "catalog-" in v: names.append(v.lower().strip().replace("catalog-", "")) elif "security-" in v: names.append(v.lower().strip().replace("security-", "")) else: continue label_set[k] = v if not label_set and container.name != "trino": raise err.UserError( f"Missing Minitrino labels for container '{container.name}'.", f"Check this module's 'docker-compose.yml' file and ensure you are " f"following the documentation on labels.", ) label_sets.append(label_set) running = {} for name, label_set, container in zip(names, label_sets, containers): if not isinstance(self.data.get(name), dict): raise err.UserError( f"Module '{name}' is running, but it is not found " f"in the library. Was it deleted, or are you pointing " f"Minitrino to the wrong location?" ) if not running.get(name, False): running[name] = self.data[name] if not running.get("labels", False): running[name]["labels"] = label_set if not running.get(name).get("containers", False): running[name]["containers"] = [container] else: running[name]["containers"].append(container) return running
def check_starburst_ver(ctx): """Checks if a proper Starburst version is provided.""" starburst_ver = ctx.env.get_var("STARBURST_VER", "") error_msg = (f"Provided Starburst version '{starburst_ver}' is invalid. " f"The provided version must be 354-e or higher.") try: starburst_ver_int = int(starburst_ver[0:3]) if starburst_ver_int < 354 or "-e" not in starburst_ver: raise err.UserError(error_msg) except: raise err.UserError(error_msg)
def cli(ctx, modules, json_format, running): """Version command for Minitrino.""" utils.check_lib(ctx) ctx.logger.log("Printing module metadata...") if modules and not running: for module in modules: module_dict = ctx.modules.data.get(module, {}) if not module_dict: raise err.UserError( f"Invalid module: {module}", "Ensure the module you're referencing is in the Minitrino library.", ) log_info(module, module_dict, json_format) else: if running: for module_key, module_dict in ctx.modules.get_running_modules( ).items(): for i, container in enumerate(module_dict.get( "containers", {})): module_dict["containers"][i] = { "id": container.short_id, "name": container.name, "labels": container.labels, } log_info(module_key, module_dict, json_format) else: for module_key, module_dict in ctx.modules.data.items(): log_info(module_key, module_dict, json_format)
def _parse_library_env(self): """Parses the Minitrino library's root `minitrino.env` file. All config from this file is added to the 'MODULES' section of the environment dictionary since this file explicitly defines the versions of the module services.""" env_file = os.path.join(self._ctx.minitrino_lib_dir, "minitrino.env") if not os.path.isfile(env_file): raise err.UserError( f"Library 'minitrino.env' file does not exist at path: {env_file}", f"Are you pointing to a valid library, and is the minitrino.env file " f"present in that library?", ) # Check if modules section was added from Minitrino config file parsing section = self.env.get("MODULES", None) if not isinstance(section, dict): self.env["MODULES"] = {} with open(env_file, "r") as f: for env_var in f: env_var = utils.parse_key_value_pair(env_var, err_type=err.UserError) if env_var is None: continue # Skip if the key exists in any section if self.get_var(env_var[0], False): continue self.env["MODULES"][env_var[0]] = env_var[1]
def validate_name(ctx, name): """Validates the chosen filename for correct input.""" for char in name: if all((char != "_", char != "-", not char.isalnum())): raise err.UserError( f"Illegal character found in provided filename: '{char}'. ", f"Alphanumeric, hyphens, and underscores are allowed. " f"Rename and retry.", )
def check_daemon(docker_client): """Checks if the Docker daemon is running. If an exception is thrown, it is handled.""" try: docker_client.ping() except Exception as e: raise err.UserError( f"Error when pinging the Docker server. Is the Docker daemon running?\n" f"Error from Docker: {str(e)}", "You may need to initialize your Docker daemon.", )
def cli(ctx, modules, no_rollback, docker_native): """Provision command for Minitrino. If the resulting docker-compose command is unsuccessful, the function exits with a non-zero status code.""" utils.check_daemon(ctx.docker_client) utils.check_lib(ctx) utils.check_starburst_ver(ctx) modules = append_running_modules(modules) check_compatibility(modules) check_enterprise(modules) if not modules: ctx.logger.log( f"No catalog or security options received. Provisioning " f"standalone Trino container..." ) else: for module in modules: if not ctx.modules.data.get(module, False): raise err.UserError( f"Invalid module: '{module}'. It was not found " f"in the Minitrino library at {ctx.minitrino_lib_dir}" ) try: cmd_chunk = chunk(modules) # Module env variables shared with compose should be from the modules # section of environment variables and any extra variables provided by the # user that didn't fit into any other section compose_env = ctx.env.get_section("MODULES") compose_env.update(ctx.env.get_section("EXTRA")) compose_cmd = build_command(docker_native, compose_env, cmd_chunk) ctx.cmd_executor.execute_commands(compose_cmd, environment=compose_env) initialize_containers() containers_to_restart = execute_bootstraps(modules) containers_to_restart = append_user_config(containers_to_restart) check_dup_configs() restart_containers(containers_to_restart) ctx.logger.log(f"Environment provisioning complete.") except Exception as e: rollback_provision(no_rollback) utils.handle_exception(e)
def minitrino_lib_dir(self): """The directory of the Minitrino library. The directory can be determined in four ways (this is the order of precedence): 1. Passing `LIB_PATH` to the CLI's `--env` option sets the library directory for the current command. 2. The `minitrino.cfg` file's `LIB_PATH` variable sets the library directory if present. 3. The path `~/.minitrino/lib/` is used as the default lib path if the `LIB_PATH` var is not found. 4. As a last resort, Minitrino will check to see if the library exists in relation to the positioning of the `components.py` file and assumes the project is being run out of a cloned repository.""" lib_dir = "" try: # Try to get `LIB_path` var - handle exception if `env` attribute is # not yet set lib_dir = self.env.get_var("LIB_PATH", "") except: pass if not lib_dir and os.path.isdir(os.path.join(self.minitrino_user_dir, "lib")): lib_dir = os.path.join(self.minitrino_user_dir, "lib") elif not lib_dir: # Use repo root, fail if this doesn't exist lib_dir = Path(os.path.abspath(__file__)).resolve().parents[2] lib_dir = os.path.join(lib_dir, "lib") if not os.path.isdir(lib_dir) or not os.path.isfile( os.path.join(lib_dir, "minitrino.env") ): raise err.UserError( "You must provide a path to a compatible Minitrino library.", f"You can point to a Minitrino library a few different " f"ways:\n(1) You can set the 'LIB_PATH' variable in your " f"Minitrino config via the command 'minitrino config'--this " f"should be placed under the '[CLI]' section.\n(2) You can " f"pass in 'LIB_PATH' as an environment variable for the current " f"command, e.g. 'minitrino -e LIB_PATH=<path/to/lib> ...'\n" f"(3) If the above variable is not found, Minitrino will check " f"if '~/.minitrino/lib/' is a valid directory.\n(4) " f"If you are running Minitrino out of a cloned repo, the library " f"path will be automatically detected without the need to perform " f"any of the above.", ) return lib_dir
def check_compatibility(ctx, modules=[]): """Checks if any of the provided modules are mutually exclusive of each other. If they are, a user error is raised.""" for module in modules: incompatible = ctx.modules.data.get(module, {}).get("incompatibleModules", []) if not incompatible: continue for module_inner in modules: if (module_inner in incompatible) or ( incompatible[0] == "*" and len(modules) > 1 ): raise err.UserError( f"Incompatible modules detected. Tried to provision module " f"'{module_inner}', but found that the module is incompatible " f"with module '{module}'. Incompatible modules listed for module " f"'{module}' are: {incompatible}", f"You can see which modules are incompatible with this module by " f"running 'minitrino modules -m {module}'", )
def cli(ctx, modules, name, directory, force, no_scrub): """Snapshot command for Minitrino.""" # The snapshot temp files are saved in ~/.minitrino/snapshots/<name> # regardless of the directory provided. The artifact (tarball) will go # to either the default directory or the user-provided directory. utils.check_lib(ctx) if directory and not os.path.isdir(directory): raise err.UserError( f"Cannot save snapshot in nonexistent directory: {directory}", "Pick any directory that exists, and this will work.", ) if not directory: directory = os.path.join(ctx.minitrino_lib_dir, "snapshots") validate_name(name) check_exists(name, directory, force) if modules: ctx.logger.log(f"Creating snapshot of specified modules...") snapshot_runner(name, no_scrub, False, modules, directory) else: modules = ctx.modules.get_running_modules() if not modules: ctx.logger.log( f"No running Minitrino modules to snapshot. Snapshotting " f"Trino resources only.", level=ctx.logger.verbose, ) else: ctx.logger.log(f"Creating snapshot of active environment...") snapshot_runner(name, no_scrub, True, list(modules.keys()), directory) check_complete(name, directory) ctx.logger.log( f"Snapshot complete and saved at path: {os.path.join(directory, name)}.tar.gz" )
def execute_container_bootstrap(ctx, bootstrap="", container_name="", yaml_file=""): """Executes a single bootstrap inside a container. If the `/opt/minitrino/bootstrap_status.txt` file has the same checksum as the bootstrap script that is about to be executed, the boostrap script is skipped. Returns `False` if the script is not executed and `True` if it is.""" if any((not bootstrap, not container_name, not yaml_file)): raise utils.handle_missing_param(list(locals().keys())) bootstrap_file = os.path.join( os.path.dirname(yaml_file), "resources", "bootstrap", bootstrap ) if not os.path.isfile(bootstrap_file): raise err.UserError( f"Bootstrap file does not exist at location: {bootstrap_file}", "Check this module in the library to ensure the bootstrap script is present.", ) # Add executable permissions to bootstrap st = os.stat(bootstrap_file) os.chmod( bootstrap_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, ) bootstrap_checksum = hashlib.md5(open(bootstrap_file, "rb").read()).hexdigest() container = ctx.docker_client.containers.get(container_name) # Check if this script has already been executed output = ctx.cmd_executor.execute_commands( "cat /opt/minitrino/bootstrap_status.txt", suppress_output=True, container=container, trigger_error=False, ) if f"{bootstrap_checksum}" in output[0].get("output", ""): ctx.logger.log( f"Bootstrap already executed in container '{container_name}'. Skipping.", level=ctx.logger.verbose, ) return False ctx.logger.log( f"Executing bootstrap script in container '{container_name}'...", level=ctx.logger.verbose, ) ctx.cmd_executor.execute_commands( f"docker cp {bootstrap_file} {container_name}:/tmp/" ) # Record executed file checksum ctx.cmd_executor.execute_commands( f"/tmp/{os.path.basename(bootstrap_file)}", f'bash -c "echo {bootstrap_checksum} >> /opt/minitrino/bootstrap_status.txt"', container=container, ) ctx.logger.log( f"Successfully executed bootstrap script in container '{container_name}'.", level=ctx.logger.verbose, ) return True
def _load_modules(self): """Loads module data during instantiation.""" self._ctx.logger.log("Loading modules...", level=self._ctx.logger.verbose) modules_dir = os.path.join(self._ctx.minitrino_lib_dir, MODULE_ROOT) if not os.path.isdir(modules_dir): raise err.MinitrinoError( f"Path is not a directory: {modules_dir}. " f"Are you pointing to a compatible Minitrino library?" ) # Loop through both catalog and security modules sections = [ os.path.join(modules_dir, MODULE_CATALOG), os.path.join(modules_dir, MODULE_SECURITY), ] for section_dir in sections: for _dir in os.listdir(section_dir): module_dir = os.path.join(section_dir, _dir) if not os.path.isdir(module_dir): self._ctx.logger.log( f"Skipping file (expected a directory, not a file) " f"at path: {module_dir}", level=self._ctx.logger.verbose, ) continue # List inner-module files module_files = os.listdir(module_dir) yaml_basename = f"{os.path.basename(module_dir)}.yml" if not yaml_basename in module_files: raise err.UserError( f"Missing Docker Compose file in module directory {_dir}. " f"Expected file to be present: {yaml_basename}", hint_msg="Check this module in your library to ensure it is properly constructed.", ) # Module dir and YAML exist, add to modules module_name = os.path.basename(module_dir) self.data[module_name] = {} self.data[module_name]["type"] = os.path.basename(section_dir) self.data[module_name]["module_dir"] = module_dir # Add YAML file path yaml_file = os.path.join(module_dir, yaml_basename) self.data[module_name]["yaml_file"] = yaml_file # Add YAML dict with open(yaml_file) as f: self.data[module_name]["yaml_dict"] = yaml.load( f, Loader=yaml.FullLoader ) # Get metadata.json if present json_basename = "metadata.json" json_file = os.path.join(module_dir, json_basename) metadata = {} if os.path.isfile(json_file): with open(json_file) as f: metadata = json.load(f) else: self._ctx.logger.log( f"No JSON metadata file for module '{module_name}'. " f"Will not load metadata for module.", level=self._ctx.logger.verbose, ) for k, v in metadata.items(): self.data[module_name][k] = v