def append_configs(user_configs, current_configs, filename): # If there is an overlapping config key, replace it with the user # config. If there is not overlapping config key, append it to the # current config list. if filename == PRESTO_CONFIG: for user_config in user_configs: user_config = utils.parse_key_value_pair( user_config, err_type=err.UserError) if user_config is None: continue for i, current_config in enumerate(current_configs): if current_config.startswith("#"): continue current_config = utils.parse_key_value_pair( current_config, err_type=err.UserError) if current_config is None: continue if user_config[0] == current_config[0]: current_configs[i] = "=".join(user_config) break if (i + 1 == len(current_configs) and not "=".join(user_config) in current_configs): current_configs.append("=".join(user_config)) else: for user_config in user_configs: user_config = user_config.strip() if not user_config: continue for i, current_config in enumerate(current_configs): if current_config.startswith("#"): continue current_config = current_config.strip() if not current_config: continue if user_config == current_config: current_configs[i] = user_config break if (i + 1 == len(current_configs) and not user_config in current_configs): current_configs.append(user_config) # Replace existing file with new values ctx.cmd_executor.execute_commands(f"rm {ETC_PRESTO}/{filename}", container=presto_container) for current_config in current_configs: append_config = ( f'bash -c "cat <<EOT >> {ETC_PRESTO}/{filename}\n{current_config}\nEOT"' ) ctx.cmd_executor.execute_commands(append_config, container=presto_container)
def _construct_environment(self, environment={}, container=None): """Merges provided environment dictionary with user's shell environment variables. For shell execution, the host environment will be set to the existing variables in the host environment. For container execution, the host environment will be set to the container's existing environment variables.""" # Remove conflicting keys from host environment; Minipresto environment # variables take precendance if not container: host_environment = os.environ.copy() else: host_environment_list = self._ctx.api_client.inspect_container( container.id)["Config"]["Env"] host_environment = {} for env_var in host_environment_list: env_var = utils.parse_key_value_pair(env_var) host_environment[env_var[0]] = env_var[1] if environment: delete_keys = [] for host_key, host_value in host_environment.items(): for key, value in environment.items(): if key == host_key: delete_keys.append(host_key) for delete_key in delete_keys: del host_environment[delete_key] # Merge environment argument with copy of existing environment environment.update(host_environment) return environment
def _parse_library_env(self): """Parses the Minipresto library's root `minipresto.env` file. All config from this file is added to the 'MODULES' section of the environment dictionary since this file explicitly defines the versions of the module services.""" env_file = os.path.join(self._ctx.minipresto_lib_dir, "minipresto.env") if not os.path.isfile(env_file): raise err.UserError( f"Library 'minipresto.env' file does not exist at path: {env_file}", f"Are you pointing to a valid library, and is the minipresto.env file " f"present in that library?", ) # Check if modules section was added from Minipresto config file parsing section = self.env.get("MODULES", None) if not isinstance(section, dict): self.env["MODULES"] = {} with open(env_file, "r") as f: for env_var in f: env_var = utils.parse_key_value_pair(env_var, err_type=err.UserError) if env_var is None: continue # Skip if the key exists in any section if self.get_var(env_var[0], False): continue self.env["MODULES"][env_var[0]] = env_var[1]
def scrub_line(ctx, line): """Scrubs a line from a snapshot config file. Returns the scrubbed line.""" # If the key has a substring that matches any of the scrub keys, we know # it's an item whose value needs to be scrubbed line = utils.parse_key_value_pair(line, err_type=err.UserError) if any(item in line[0].lower() for item in SCRUB_KEYS): line[1] = "*" * 20 return "=".join(line)
def _parse_user_env(self): """Parses user-provided environment variables for the current command.""" if not self._ctx._user_env: return user_env_dict = {} for env_var in self._ctx._user_env: env_var = utils.parse_key_value_pair(env_var, err_type=err.UserError) user_env_dict[env_var[0]] = env_var[1] # Loop through user-provided environment variables and check for matches # in each section dict. If the variable key is in the section dict, it # needs to be identified and replaced with the user's `--env` value, # effectively overriding the original value. # # We build a new dict to prevent issues with changing dict size while # iterating. # # Any variable keys that do not match with an existing key will be added # to the section dict "EXTRA". new_dict = {} for section_k, section_v in self.env.items(): if not isinstance(section_v, dict): raise err.MiniprestoError( f"Invalid environment dictionary. Expected nested dictionaries. " f"Received dictionary:\n" f"{json.dumps(self.env, indent=2)}") delete_keys = [] for user_k, user_v in user_env_dict.items(): if user_k in section_v: if new_dict.get(section_k, None) is None: new_dict[section_k] = section_v new_dict[section_k][user_k] = user_v delete_keys.append(user_k) else: new_dict[section_k] = section_v for delete_key in delete_keys: del user_env_dict[delete_key] if user_env_dict: self.env["EXTRA"] = {} for k, v in user_env_dict.items(): self.env["EXTRA"][k] = v
def check_dup_configs(ctx): """Checks for duplicate configs in Presto config files (jvm.config and config.properties). This is a safety check for modules that may improperly modify these files. Duplicates will only be registered for JVM config if the configs are identical. For config.properties, duplicates will be registered if there are multiple overlapping property keys.""" check_files = [PRESTO_CONFIG, PRESTO_JVM_CONFIG] for check_file in check_files: ctx.logger.log( f"Checking Presto {check_file} for duplicate configs...", level=ctx.logger.verbose, ) container = ctx.docker_client.containers.get("presto") output = ctx.cmd_executor.execute_commands( f"cat {ETC_PRESTO}/{check_file}", suppress_output=True, container=container, ) configs = output[0].get("output", "") if not configs: raise err.MiniprestoError( f"Presto {check_file} file unable to be read from Presto container." ) configs = configs.strip().split("\n") configs.sort() duplicates = [] if check_file == PRESTO_CONFIG: for i, config in enumerate(configs): if config.startswith("#"): continue config = utils.parse_key_value_pair(config, err_type=err.UserError) if config is None: continue if i + 1 != len(configs): next_config = utils.parse_key_value_pair( configs[i + 1], err_type=err.UserError) if config[0] == next_config[0]: duplicates.extend( ["=".join(config), "=".join(next_config)]) else: next_config = [""] if config[0] == next_config[0]: duplicates.extend( ["=".join(config), "=".join(next_config)]) elif duplicates: duplicates = set(duplicates) duplicates_string = "\n".join(duplicates) ctx.logger.log( f"Duplicate Presto configuration properties detected in " f"{check_file} file:\n{duplicates_string}", level=ctx.logger.warn, ) duplicates = [] else: # JVM config for i, config in enumerate(configs): config = config.strip() if config.startswith("#") or not config: continue if i + 1 != len(configs): next_config = configs[i + 1].strip() else: next_config = "" if config == next_config: duplicates.extend([config, next_config]) elif duplicates: duplicates_string = "\n".join(duplicates) ctx.logger.log( f"Duplicate Presto configuration properties detected in " f"{check_file} file:\n{duplicates_string}", level=ctx.logger.warn, ) duplicates = []