def append_configs(user_configs, current_configs, filename):
# If there is an overlapping config key, replace it with the user
# config. If there is not overlapping config key, append it to the
# current config list.
if filename == PRESTO_CONFIG:
for user_config in user_configs:
user_config = utils.parse_key_value_pair(
user_config, err_type=err.UserError)
if user_config is None:
continue
for i, current_config in enumerate(current_configs):
if current_config.startswith("#"):
continue
current_config = utils.parse_key_value_pair(
current_config, err_type=err.UserError)
if current_config is None:
continue
if user_config[0] == current_config[0]:
current_configs[i] = "=".join(user_config)
break
if (i + 1 == len(current_configs)
and not "=".join(user_config) in current_configs):
current_configs.append("=".join(user_config))
else:
for user_config in user_configs:
user_config = user_config.strip()
if not user_config:
continue
for i, current_config in enumerate(current_configs):
if current_config.startswith("#"):
continue
current_config = current_config.strip()
if not current_config:
continue
if user_config == current_config:
current_configs[i] = user_config
break
if (i + 1 == len(current_configs)
and not user_config in current_configs):
current_configs.append(user_config)
# Replace existing file with new values
ctx.cmd_executor.execute_commands(f"rm {ETC_PRESTO}/{filename}",
container=presto_container)
for current_config in current_configs:
append_config = (
f'bash -c "cat <<EOT >> {ETC_PRESTO}/{filename}\n{current_config}\nEOT"'
)
ctx.cmd_executor.execute_commands(append_config,
container=presto_container)
def _construct_environment(self, environment={}, container=None):
"""Merges provided environment dictionary with user's shell environment
variables. For shell execution, the host environment will be set to the
existing variables in the host environment. For container execution, the
host environment will be set to the container's existing environment
variables."""
# Remove conflicting keys from host environment; Minipresto environment
# variables take precendance
if not container:
host_environment = os.environ.copy()
else:
host_environment_list = self._ctx.api_client.inspect_container(
container.id)["Config"]["Env"]
host_environment = {}
for env_var in host_environment_list:
env_var = utils.parse_key_value_pair(env_var)
host_environment[env_var[0]] = env_var[1]
if environment:
delete_keys = []
for host_key, host_value in host_environment.items():
for key, value in environment.items():
if key == host_key:
delete_keys.append(host_key)
for delete_key in delete_keys:
del host_environment[delete_key]
# Merge environment argument with copy of existing environment
environment.update(host_environment)
return environment
def _parse_library_env(self):
"""Parses the Minipresto library's root `minipresto.env` file. All config from
this file is added to the 'MODULES' section of the environment
dictionary since this file explicitly defines the versions of the module
services."""
env_file = os.path.join(self._ctx.minipresto_lib_dir, "minipresto.env")
if not os.path.isfile(env_file):
raise err.UserError(
f"Library 'minipresto.env' file does not exist at path: {env_file}",
f"Are you pointing to a valid library, and is the minipresto.env file "
f"present in that library?",
)
# Check if modules section was added from Minipresto config file parsing
section = self.env.get("MODULES", None)
if not isinstance(section, dict):
self.env["MODULES"] = {}
with open(env_file, "r") as f:
for env_var in f:
env_var = utils.parse_key_value_pair(env_var,
err_type=err.UserError)
if env_var is None:
continue
# Skip if the key exists in any section
if self.get_var(env_var[0], False):
continue
self.env["MODULES"][env_var[0]] = env_var[1]
def scrub_line(ctx, line):
"""Scrubs a line from a snapshot config file. Returns the scrubbed line."""
# If the key has a substring that matches any of the scrub keys, we know
# it's an item whose value needs to be scrubbed
line = utils.parse_key_value_pair(line, err_type=err.UserError)
if any(item in line[0].lower() for item in SCRUB_KEYS):
line[1] = "*" * 20
return "=".join(line)
def _parse_user_env(self):
"""Parses user-provided environment variables for the current
command."""
if not self._ctx._user_env:
return
user_env_dict = {}
for env_var in self._ctx._user_env:
env_var = utils.parse_key_value_pair(env_var,
err_type=err.UserError)
user_env_dict[env_var[0]] = env_var[1]
# Loop through user-provided environment variables and check for matches
# in each section dict. If the variable key is in the section dict, it
# needs to be identified and replaced with the user's `--env` value,
# effectively overriding the original value.
#
# We build a new dict to prevent issues with changing dict size while
# iterating.
#
# Any variable keys that do not match with an existing key will be added
# to the section dict "EXTRA".
new_dict = {}
for section_k, section_v in self.env.items():
if not isinstance(section_v, dict):
raise err.MiniprestoError(
f"Invalid environment dictionary. Expected nested dictionaries. "
f"Received dictionary:\n"
f"{json.dumps(self.env, indent=2)}")
delete_keys = []
for user_k, user_v in user_env_dict.items():
if user_k in section_v:
if new_dict.get(section_k, None) is None:
new_dict[section_k] = section_v
new_dict[section_k][user_k] = user_v
delete_keys.append(user_k)
else:
new_dict[section_k] = section_v
for delete_key in delete_keys:
del user_env_dict[delete_key]
if user_env_dict:
self.env["EXTRA"] = {}
for k, v in user_env_dict.items():
self.env["EXTRA"][k] = v
def check_dup_configs(ctx):
"""Checks for duplicate configs in Presto config files (jvm.config and
config.properties). This is a safety check for modules that may improperly
modify these files.
Duplicates will only be registered for JVM config if the configs are
identical. For config.properties, duplicates will be registered if there are
multiple overlapping property keys."""
check_files = [PRESTO_CONFIG, PRESTO_JVM_CONFIG]
for check_file in check_files:
ctx.logger.log(
f"Checking Presto {check_file} for duplicate configs...",
level=ctx.logger.verbose,
)
container = ctx.docker_client.containers.get("presto")
output = ctx.cmd_executor.execute_commands(
f"cat {ETC_PRESTO}/{check_file}",
suppress_output=True,
container=container,
)
configs = output[0].get("output", "")
if not configs:
raise err.MiniprestoError(
f"Presto {check_file} file unable to be read from Presto container."
)
configs = configs.strip().split("\n")
configs.sort()
duplicates = []
if check_file == PRESTO_CONFIG:
for i, config in enumerate(configs):
if config.startswith("#"):
continue
config = utils.parse_key_value_pair(config,
err_type=err.UserError)
if config is None:
continue
if i + 1 != len(configs):
next_config = utils.parse_key_value_pair(
configs[i + 1], err_type=err.UserError)
if config[0] == next_config[0]:
duplicates.extend(
["=".join(config), "=".join(next_config)])
else:
next_config = [""]
if config[0] == next_config[0]:
duplicates.extend(
["=".join(config), "=".join(next_config)])
elif duplicates:
duplicates = set(duplicates)
duplicates_string = "\n".join(duplicates)
ctx.logger.log(
f"Duplicate Presto configuration properties detected in "
f"{check_file} file:\n{duplicates_string}",
level=ctx.logger.warn,
)
duplicates = []
else: # JVM config
for i, config in enumerate(configs):
config = config.strip()
if config.startswith("#") or not config:
continue
if i + 1 != len(configs):
next_config = configs[i + 1].strip()
else:
next_config = ""
if config == next_config:
duplicates.extend([config, next_config])
elif duplicates:
duplicates_string = "\n".join(duplicates)
ctx.logger.log(
f"Duplicate Presto configuration properties detected in "
f"{check_file} file:\n{duplicates_string}",
level=ctx.logger.warn,
)
duplicates = []