def initialize_containers(ctx):
"""Initializes each container with /opt/minipresto/bootstrap_status.txt."""
containers = ctx.docker_client.containers.list(
filters={"label": RESOURCE_LABEL})
for container in containers:
output = ctx.cmd_executor.execute_commands(
"cat /opt/minipresto/bootstrap_status.txt",
suppress_output=True,
container=container,
trigger_error=False,
)
output_string = output[0].get("output", "").strip()
if "no such file or directory" in output_string.lower():
ctx.cmd_executor.execute_commands(
"mkdir -p /opt/minipresto/",
"touch /opt/minipresto/bootstrap_status.txt",
container=container,
)
elif output[0].get("return_code", None) == 0:
continue
else:
raise err.MiniprestoError(
f"Command failed.\n"
f"Output: {output_string}\n"
f"Exit code: {output[0].get('return_code', None)}")
def check_complete(ctx, name, directory):
"""Checks if the snapshot completed. If detected as incomplete, exists with
a non-zero status code."""
snapshot_file = os.path.join(directory, f"{name}.tar.gz")
if not os.path.isfile(snapshot_file):
raise err.MiniprestoError(
f"Snapshot tarball failed to write to {snapshot_file}")
def download_and_extract(ctx, version=""):
github_uri = f"https://github.com/jefflester/minipresto/archive/{version}.tar.gz"
tarball = os.path.join(ctx.minipresto_user_dir, f"{version}.tar.gz")
file_basename = f"minipresto-{version}" # filename after unpacking
lib_dir = os.path.join(ctx.minipresto_user_dir, file_basename, "lib")
try:
# Download the release tarball
cmd = f"curl -fsSL {github_uri} > {tarball}"
ctx.cmd_executor.execute_commands(cmd)
if not os.path.isfile(tarball):
raise err.MiniprestoError(
f"Failed to download Minipresto library ({tarball} not found)."
)
# Unpack tarball and copy lib
ctx.logger.log(
f"Unpacking tarball at {tarball} and copying library...",
level=ctx.logger.verbose,
)
ctx.cmd_executor.execute_commands(
f"tar -xzvf {tarball} -C {ctx.minipresto_user_dir}",
f"mv {lib_dir} {ctx.minipresto_user_dir}",
)
# Check that the library is present
lib_dir = os.path.join(ctx.minipresto_user_dir, "lib")
if not os.path.isdir(lib_dir):
raise err.MiniprestoError(
f"Library failed to install (not found at {lib_dir})")
# Cleanup
cleanup(tarball, file_basename)
except Exception as e:
cleanup(tarball, file_basename, False)
raise err.MiniprestoError(str(e))
def _execute_in_shell(self, command="", **kwargs):
"""Executes a command in the host shell."""
self._ctx.logger.log(
f"Executing command in shell:\n{command}",
level=self._ctx.logger.verbose,
)
process = subprocess.Popen(
command,
shell=True,
env=kwargs.get("environment", {}),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
if not kwargs.get("suppress_output", False):
# Stream the output of the executed command line-by-line.
# `universal_newlines=True` ensures output is generated as a string,
# so there is no need to decode bytes. The only cleansing we need to
# do is to run the string through the `_strip_ansi()` function.
started_stream = False
while True:
output_line = process.stdout.readline()
if output_line == "" and process.poll() is not None:
break
output_line = self._strip_ansi(output_line)
if not started_stream:
self._ctx.logger.log("Command Output:",
level=self._ctx.logger.verbose)
started_stream = True
self._ctx.logger.log(output_line,
level=self._ctx.logger.verbose,
stream=True)
output, _ = process.communicate() # Get full output (stdout + stderr)
if process.returncode != 0 and kwargs.get("trigger_error", True):
raise err.MiniprestoError(
f"Failed to execute shell command:\n{command}\n"
f"Exit code: {process.returncode}")
return {
"command": command,
"output": self._strip_ansi(output),
"return_code": process.returncode,
}
def _parse_user_env(self):
"""Parses user-provided environment variables for the current
command."""
if not self._ctx._user_env:
return
user_env_dict = {}
for env_var in self._ctx._user_env:
env_var = utils.parse_key_value_pair(env_var,
err_type=err.UserError)
user_env_dict[env_var[0]] = env_var[1]
# Loop through user-provided environment variables and check for matches
# in each section dict. If the variable key is in the section dict, it
# needs to be identified and replaced with the user's `--env` value,
# effectively overriding the original value.
#
# We build a new dict to prevent issues with changing dict size while
# iterating.
#
# Any variable keys that do not match with an existing key will be added
# to the section dict "EXTRA".
new_dict = {}
for section_k, section_v in self.env.items():
if not isinstance(section_v, dict):
raise err.MiniprestoError(
f"Invalid environment dictionary. Expected nested dictionaries. "
f"Received dictionary:\n"
f"{json.dumps(self.env, indent=2)}")
delete_keys = []
for user_k, user_v in user_env_dict.items():
if user_k in section_v:
if new_dict.get(section_k, None) is None:
new_dict[section_k] = section_v
new_dict[section_k][user_k] = user_v
delete_keys.append(user_k)
else:
new_dict[section_k] = section_v
for delete_key in delete_keys:
del user_env_dict[delete_key]
if user_env_dict:
self.env["EXTRA"] = {}
for k, v in user_env_dict.items():
self.env["EXTRA"][k] = v
def restart_containers(ctx, containers_to_restart=[]):
"""Restarts all the containers in the list."""
if containers_to_restart == []:
return
# Remove any duplicates
containers_to_restart = list(set(containers_to_restart))
for container in containers_to_restart:
try:
container = ctx.docker_client.containers.get(container)
ctx.logger.log(f"Restarting container '{container.name}'...",
level=ctx.logger.verbose)
container.restart()
except NotFound:
raise err.MiniprestoError(
f"Attempting to restart container '{container.name}', but the container was not found."
)
def handle_missing_param(params=[]):
"""Handles missing parameters required for function calls. This should be
used to signal a programmatic error, not a user error.
### Parameters
- `params`: List of parameter names that are required.
### Usage
```python
# All params are required
if not param:
raise handle_missing_param(list(locals().keys()))
# Two params are required
if not param:
raise handle_missing_param(["module", "path"])
```"""
if not params:
raise handle_missing_param(list(locals().keys()))
return err.MiniprestoError(f"Parameters {params} required to execute function.")
def log(self, *args, level=None, stream=False):
"""Logs messages to the user's console. Defaults to 'info' log level.
### Parameters
- `*args`: Messages to log.
- `level`: The level of the log message (info, warn, error, and
verbose).
- `stream`: If `True`, the logger will not apply a prefix to each line
streamed to the console."""
if not level:
level = self.info
# Skip verbose messages unless verbose mode is enabled
if not self._log_verbose and level == self.verbose:
return
for msg in args:
# Ensure the message can be a string
try:
msg = str(msg)
except:
raise err.MiniprestoError(
f"A string is required for {self.log.__name__}."
)
msgs = msg.replace("\r", "\n").split("\n")
# Log each message
for i, msg in enumerate(msgs):
msg = self._format(msg)
if not msg:
continue
if stream or i > 0:
msg_prefix = DEFAULT_INDENT
else:
msg_prefix = style(
level.get("prefix", ""),
fg=level.get("prefix_color", ""),
bold=True,
)
echo(f"{msg_prefix}{msg}")
def execute_bootstraps(ctx, modules=[]):
"""Executes bootstrap script for each container that has one––bootstrap
scripts will only execute once the container is fully running to prevent
conflicts with procedures executing as part of the container's entrypoint.
After each script executes, the relevant container is added to a restart
list.
Returns a list of containers names which had bootstrap scripts executed
inside of them."""
services = []
for module in modules:
yaml_file = ctx.modules.data.get(module, {}).get("yaml_file", "")
module_services = (ctx.modules.data.get(module, {}).get(
"yaml_dict", {}).get("services", {}))
if not module_services:
raise err.MiniprestoError(
f"Invalid Docker Compose YAML file (no 'services' section found): {yaml_file}"
)
# Get all services defined in YAML file
for service_key, service_dict in module_services.items():
services.append([service_key, service_dict, yaml_file])
containers = []
# Get all container names for each service
for service in services:
bootstrap = service[1].get("environment",
{}).get("MINIPRESTO_BOOTSTRAP")
if bootstrap is None:
continue
container_name = service[1].get("container_name")
if container_name is None:
# If there is not container name, the service name becomes the name
# of the container
container_name = service[0]
if execute_container_bootstrap(bootstrap, container_name, service[2]):
containers.append(container_name)
return containers
def handle_exception(error=Exception, additional_msg="", skip_traceback=False):
"""Handles a single exception. Wrapped by `@exception_handler` decorator.
### Parameters
- `error`: The exception object.
- `additional_msg`: An additional message to log, if any. Can be useful if
handling a generic exception and you need to append a user-friendly
message to the log.
- `skip_traceback`: If `True`, the traceback will not be printed to the
user's terminal. Defaults to `True` for user errors, but it is `False`
otherwise."""
if not isinstance(error, Exception):
raise handle_missing_param(["error"])
if isinstance(error, err.UserError):
error_msg = error.msg
exit_code = error.exit_code
skip_traceback = True
elif isinstance(error, err.MiniprestoError):
error_msg = error.msg
exit_code = error.exit_code
elif isinstance(error, Exception):
error_msg = str(error)
exit_code = 1
else:
raise err.MiniprestoError(
f"Invalid type given to 'e' parameter of {handle_exception.__name__}. "
f"Expected an Exception type, but got type {type(error).__name__}"
)
logger = Logger()
logger.log(additional_msg, error_msg, level=logger.error)
if not skip_traceback:
echo() # Force a newline
echo(f"{traceback.format_exc()}", err=True)
sys.exit(exit_code)
def prompt_msg(self, msg="", input_type=str):
"""Logs a prompt message and returns the user's input.
### Parameters
- `msg`: The prompt message
- `input_type`: The object type to check the input for"""
if not msg:
raise handle_missing_param(["msg"])
try:
msg = str(msg)
except:
raise err.MiniprestoError(f"A string is required for {self.log.__name__}.")
msg = self._format(msg)
styled_prefix = style(
self.info.get("prefix", ""), fg=self.info.get("prefix_color", ""), bold=True
)
return prompt(
f"{styled_prefix}{msg}",
type=input_type,
)
def append_user_config(ctx, containers_to_restart=[]):
"""Appends Presto config from minipresto.cfg file. If the config is not
present, it is added. If it exists, it is replaced. If anything changes in
the Presto config, the Presto container is added to the restart list if it's
not already in the list."""
user_presto_config = ctx.env.get_var("CONFIG", "")
if user_presto_config:
user_presto_config = user_presto_config.strip().split("\n")
user_jvm_config = ctx.env.get_var("JVM_CONFIG", "")
if user_jvm_config:
user_jvm_config = user_jvm_config.strip().split("\n")
if not user_presto_config and not user_jvm_config:
return containers_to_restart
ctx.logger.log(
"Appending user-defined Presto config to Presto container config...",
level=ctx.logger.verbose,
)
presto_container = ctx.docker_client.containers.get("presto")
if not presto_container:
raise err.MiniprestoError(
f"Attempting to append Presto configuration in Presto container, "
f"but no running Presto container was found.")
ctx.logger.log(
"Checking Presto server status before updating configs...",
level=ctx.logger.verbose,
)
retry = 0
while retry <= 30:
logs = presto_container.logs().decode()
if "======== SERVER STARTED ========" in logs:
break
elif presto_container.status != "running":
raise err.MiniprestoError(
f"Presto container stopped running. Inspect the container logs if the "
f"container is still available. If the container was rolled back, rerun "
f"the command with the '--no-rollback' option, then inspect the logs."
)
else:
ctx.logger.log(
"Presto server has not started. Waiting one second and trying again...",
level=ctx.logger.verbose,
)
time.sleep(1)
retry += 1
current_configs = ctx.cmd_executor.execute_commands(
f"cat {ETC_PRESTO}/{PRESTO_CONFIG}",
f"cat {ETC_PRESTO}/{PRESTO_JVM_CONFIG}",
container=presto_container,
suppress_output=True,
)
current_presto_config = current_configs[0].get("output",
"").strip().split("\n")
current_jvm_config = current_configs[1].get("output",
"").strip().split("\n")
def append_configs(user_configs, current_configs, filename):
# If there is an overlapping config key, replace it with the user
# config. If there is not overlapping config key, append it to the
# current config list.
if filename == PRESTO_CONFIG:
for user_config in user_configs:
user_config = utils.parse_key_value_pair(
user_config, err_type=err.UserError)
if user_config is None:
continue
for i, current_config in enumerate(current_configs):
if current_config.startswith("#"):
continue
current_config = utils.parse_key_value_pair(
current_config, err_type=err.UserError)
if current_config is None:
continue
if user_config[0] == current_config[0]:
current_configs[i] = "=".join(user_config)
break
if (i + 1 == len(current_configs)
and not "=".join(user_config) in current_configs):
current_configs.append("=".join(user_config))
else:
for user_config in user_configs:
user_config = user_config.strip()
if not user_config:
continue
for i, current_config in enumerate(current_configs):
if current_config.startswith("#"):
continue
current_config = current_config.strip()
if not current_config:
continue
if user_config == current_config:
current_configs[i] = user_config
break
if (i + 1 == len(current_configs)
and not user_config in current_configs):
current_configs.append(user_config)
# Replace existing file with new values
ctx.cmd_executor.execute_commands(f"rm {ETC_PRESTO}/{filename}",
container=presto_container)
for current_config in current_configs:
append_config = (
f'bash -c "cat <<EOT >> {ETC_PRESTO}/{filename}\n{current_config}\nEOT"'
)
ctx.cmd_executor.execute_commands(append_config,
container=presto_container)
append_configs(user_presto_config, current_presto_config, PRESTO_CONFIG)
append_configs(user_jvm_config, current_jvm_config, PRESTO_JVM_CONFIG)
if not "presto" in containers_to_restart:
containers_to_restart.append("presto")
return containers_to_restart
def check_dup_configs(ctx):
"""Checks for duplicate configs in Presto config files (jvm.config and
config.properties). This is a safety check for modules that may improperly
modify these files.
Duplicates will only be registered for JVM config if the configs are
identical. For config.properties, duplicates will be registered if there are
multiple overlapping property keys."""
check_files = [PRESTO_CONFIG, PRESTO_JVM_CONFIG]
for check_file in check_files:
ctx.logger.log(
f"Checking Presto {check_file} for duplicate configs...",
level=ctx.logger.verbose,
)
container = ctx.docker_client.containers.get("presto")
output = ctx.cmd_executor.execute_commands(
f"cat {ETC_PRESTO}/{check_file}",
suppress_output=True,
container=container,
)
configs = output[0].get("output", "")
if not configs:
raise err.MiniprestoError(
f"Presto {check_file} file unable to be read from Presto container."
)
configs = configs.strip().split("\n")
configs.sort()
duplicates = []
if check_file == PRESTO_CONFIG:
for i, config in enumerate(configs):
if config.startswith("#"):
continue
config = utils.parse_key_value_pair(config,
err_type=err.UserError)
if config is None:
continue
if i + 1 != len(configs):
next_config = utils.parse_key_value_pair(
configs[i + 1], err_type=err.UserError)
if config[0] == next_config[0]:
duplicates.extend(
["=".join(config), "=".join(next_config)])
else:
next_config = [""]
if config[0] == next_config[0]:
duplicates.extend(
["=".join(config), "=".join(next_config)])
elif duplicates:
duplicates = set(duplicates)
duplicates_string = "\n".join(duplicates)
ctx.logger.log(
f"Duplicate Presto configuration properties detected in "
f"{check_file} file:\n{duplicates_string}",
level=ctx.logger.warn,
)
duplicates = []
else: # JVM config
for i, config in enumerate(configs):
config = config.strip()
if config.startswith("#") or not config:
continue
if i + 1 != len(configs):
next_config = configs[i + 1].strip()
else:
next_config = ""
if config == next_config:
duplicates.extend([config, next_config])
elif duplicates:
duplicates_string = "\n".join(duplicates)
ctx.logger.log(
f"Duplicate Presto configuration properties detected in "
f"{check_file} file:\n{duplicates_string}",
level=ctx.logger.warn,
)
duplicates = []
def _execute_in_container(self, command="", **kwargs):
"""Executes a command inside of a container through the Docker SDK
(similar to `docker exec`)."""
container = kwargs.get("container", None)
if container is None:
raise err.MiniprestoError(
f"Attempted to execute a command inside of a "
f"container, but a container object was not provided.")
self._ctx.logger.log(
f"Executing command in container '{container.name}':\n{command}",
level=self._ctx.logger.verbose,
)
# Create exec handler and execute the command
exec_handler = self._ctx.api_client.exec_create(
container.name,
cmd=command,
environment=kwargs.get("environment", {}),
privileged=True,
user=kwargs.get("docker_user", "root"),
)
# `output` is a generator that yields response chunks
output_generator = self._ctx.api_client.exec_start(exec_handler,
stream=True)
# Output from the generator is returned as bytes, so they need to be
# decoded to strings. Response chunks are not guaranteed to be full
# lines. A newline in the output chunk will trigger a log dump of the
# current `full_line` up to the first newline in the current chunk. The
# remainder of the chunk (if any) resets the `full_line` var, then log
# dumped when the next newline is received.
output = ""
full_line = ""
started_stream = False
for chunk in output_generator:
chunk = self._strip_ansi(chunk.decode())
output += chunk
chunk = chunk.split("\n", 1)
if len(chunk) > 1: # Indicates newline present
full_line += chunk[0]
if not kwargs.get("suppress_output", False):
if not started_stream:
self._ctx.logger.log("Command Output:",
level=self._ctx.logger.verbose)
started_stream = True
self._ctx.logger.log(full_line,
level=self._ctx.logger.verbose,
stream=True)
full_line = ""
if chunk[1]:
full_line = chunk[1]
else:
full_line += chunk[0]
# Catch lingering full line post-loop
if not kwargs.get("suppress_output", False) and full_line:
self._ctx.logger.log(full_line,
level=self._ctx.logger.verbose,
stream=True)
# Get the exit code
return_code = self._ctx.api_client.exec_inspect(
exec_handler["Id"]).get("ExitCode")
if return_code != 0 and kwargs.get("trigger_error", True):
raise err.MiniprestoError(
f"Failed to execute command in container '{container.name}':\n{command}\n"
f"Exit code: {return_code}")
return {
"command": command,
"output": output,
"return_code": return_code
}
def _load_modules(self):
"""Loads module data during instantiation."""
self._ctx.logger.log("Loading modules...",
level=self._ctx.logger.verbose)
modules_dir = os.path.join(self._ctx.minipresto_lib_dir, MODULE_ROOT)
if not os.path.isdir(modules_dir):
raise err.MiniprestoError(
f"Path is not a directory: {modules_dir}. "
f"Are you pointing to a compatible Minipresto library?")
# Loop through both catalog and security modules
sections = [
os.path.join(modules_dir, MODULE_CATALOG),
os.path.join(modules_dir, MODULE_SECURITY),
]
for section_dir in sections:
for _dir in os.listdir(section_dir):
module_dir = os.path.join(section_dir, _dir)
if not os.path.isdir(module_dir):
self._ctx.logger.log(
f"Skipping file (expected a directory, not a file) "
f"at path: {module_dir}",
level=self._ctx.logger.verbose,
)
continue
# List inner-module files
module_files = os.listdir(module_dir)
yaml_basename = f"{os.path.basename(module_dir)}.yml"
if not yaml_basename in module_files:
raise err.UserError(
f"Missing Docker Compose file in module directory {_dir}. "
f"Expected file to be present: {yaml_basename}",
hint_msg=
"Check this module in your library to ensure it is properly constructed.",
)
# Module dir and YAML exist, add to modules
module_name = os.path.basename(module_dir)
self.data[module_name] = {}
self.data[module_name]["type"] = os.path.basename(section_dir)
self.data[module_name]["module_dir"] = module_dir
# Add YAML file path
yaml_file = os.path.join(module_dir, yaml_basename)
self.data[module_name]["yaml_file"] = yaml_file
# Add YAML dict
with open(yaml_file) as f:
self.data[module_name]["yaml_dict"] = yaml.load(
f, Loader=yaml.FullLoader)
# Get metadata.json if present
json_basename = "metadata.json"
json_file = os.path.join(module_dir, json_basename)
metadata = {}
if os.path.isfile(json_file):
with open(json_file) as f:
metadata = json.load(f)
else:
self._ctx.logger.log(
f"No JSON metadata file for module '{module_name}'. "
f"Will not load metadata for module.",
level=self._ctx.logger.verbose,
)
self.data[module_name]["description"] = metadata.get(
"description", "No module description provided.")
self.data[module_name]["incompatible_modules"] = metadata.get(
"incompatible_modules", [])
