def __generate_from_template(
self, template_file: Path, target_file: Path, variables: dict
):
"""
Generate configuration file from the specified template file using
the supplied variables.
Args:
template_file (Path): Template used to generate the file.
target_file (Path): Location to write the generated file to.
variables (dict): Variables used to populate the template.
"""
template = Template(
template_file.read_text(),
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
try:
output_text = template.render(variables)
target_file.write_text(output_text)
except Exception as e:
error_and_exit(
f"Could not generate file from template. The configuration file is likely missing a setting: {e}"
)
def override(ctx, template, force):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.CONFIGURE_TEMPLATE_OVERRIDE, force)
baseline_templates_dir = self.cli_configuration.baseline_templates_dir
template_file_path = Path(
os.path.join(baseline_templates_dir, template))
if not template_file_path.exists():
error_and_exit(f"Could not find template [{template}]")
override_file_path = (
cli_context.get_baseline_template_overrides_dir().joinpath(
template))
if override_file_path.exists():
if not force:
error_and_exit(
f"Override template already exists at [{override_file_path}]. Use --force to overwrite."
)
logger.info(
"Force flag provided. Overwriting existing override file.")
# Makes the override and sub folders if they do not exist
os.makedirs(override_file_path.parent, exist_ok=True)
shutil.copy2(template_file_path, override_file_path)
logger.info(
f"Copied template [{template}] to [{override_file_path}]")
def get(ctx, template):
baseline_templates_dir = self.cli_configuration.baseline_templates_dir
template_file_path = Path(os.path.join(baseline_templates_dir, template))
if not template_file_path.exists():
error_and_exit(f"Could not find template [{template}]")
print(template_file_path.read_text())
def __seed_configuration_dir(self):
"""Seed the raw configuration into the configuration directory"""
print_header("Seeding configuration directory ...")
logger.debug("Copying app configuration file ...")
seed_app_configuration_file = self.cli_configuration.seed_app_configuration_file
if not seed_app_configuration_file.is_file():
error_and_exit(
f"Seed file [{seed_app_configuration_file}] is not valid. Release is corrupt."
)
# Create the configuration directory and copy in the app config file
target_app_configuration_file = self.cli_context.get_app_configuration_file()
logger.debug(
"Copying app configuration file to [%s] ...", target_app_configuration_file
)
os.makedirs(target_app_configuration_file.parent, exist_ok=True)
shutil.copy2(seed_app_configuration_file, target_app_configuration_file)
stack_configuration_file = self.cli_configuration.stack_configuration_file
target_stack_configuration_file = (
self.cli_context.get_stack_configuration_file()
)
# Copy in the stack configuration file
if stack_configuration_file.is_file():
shutil.copy2(stack_configuration_file, target_stack_configuration_file)
# Create the configurable templates directory
logger.debug("Copying configurable templates ...")
configurable_templates_dir = self.cli_context.get_configurable_templates_dir()
configurable_templates_dir.mkdir(parents=True, exist_ok=True)
seed_configurable_templates_dir = (
self.cli_configuration.configurable_templates_dir
)
if seed_configurable_templates_dir is None:
logger.debug("No configurable templates directory defined")
return
if not seed_configurable_templates_dir.is_dir():
error_and_exit(
f"Seed templates directory [{seed_configurable_templates_dir}] is not valid. Release is corrupt."
)
# Copy each seed file to the configurable templates directory
for source_file in seed_configurable_templates_dir.glob("**/*"):
logger.debug(source_file)
relative_file = source_file.relative_to(seed_configurable_templates_dir)
target_file = configurable_templates_dir.joinpath(relative_file)
if source_file.is_dir():
logger.debug("Creating directory [%s] ...", target_file)
target_file.mkdir(parents=True, exist_ok=True)
else:
logger.debug("Copying seed file to [%s] ...", target_file)
shutil.copy2(source_file, target_file)
def initialise_configuration(self):
"""Initialises the configuration repository"""
if not self.config_repo.is_repo_on_master_branch():
error_and_exit(
"Cannot initialise configuration, repo is not on master branch."
)
# Populate with new configuration
self.__create_new_configuration_branch_and_files()
def checkout_new_branch_from_master(self, branch_name: str):
"""Checkout a new branch from the HEAD of master
Args:
branch_name (str): name of the new branch
"""
if self.does_branch_exist(branch_name):
error_and_exit(f"Cannot create new branch {branch_name}. Already exists.")
self.checkout_existing_branch("master")
self.repo.git.checkout("HEAD", b=branch_name)
def get(ctx, template):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.CONFIGURE_TEMPLATE_GET)
baseline_templates_dir = self.cli_configuration.baseline_templates_dir
template_file_path = Path(
os.path.join(baseline_templates_dir, template))
if not template_file_path.exists():
error_and_exit(f"Could not find template [{template}]")
print(template_file_path.read_text())
def _validate_service_names(
self, ctx: click.Context, param: click.Option, values: click.Tuple
):
"""Validates service names. Exits with error if any invalid service names are passed in.
Args:
ctx (click.Context): Current CLI context.
param (click.Option): The option parameter to validate.
values (click.Tuple): The values passed to the option, could be multiple.
"""
if not self.orchestrator.verify_service_names(ctx.obj, values):
error_and_exit("One or more service names were not found.")
return values
def __backup_and_create_new_directory(
self, source_dir: Path, additional_filename_descriptor: str = "backup"
) -> Path:
"""Backs up a directory to a tar gzipped file with the current datetimestamp,
deletes the existing directory, and creates a new empty directory in its place
Args:
source_dir (Path): Path to the directory to backup and delete
additional_filename_descriptor (str, optional): an additional identifier to put into
the new tgz filename. If not supplied, defaults to 'backup'.
"""
if os.path.exists(source_dir) and os.listdir(source_dir):
# The datetime is accurate to seconds (microseconds was overkill), and we remove
# colon (:) because `tar tvf` doesn't like filenames with colons
current_datetime = (
datetime.now().replace(microsecond=0).isoformat().replace(":", "")
)
# We have to do a replacement in case it has a slash in it, which causes the
# creation of the tar file to fail
clean_additional_filename_descriptor = (
additional_filename_descriptor.replace("/", "-")
)
basename = os.path.basename(source_dir)
output_filename = os.path.join(
os.path.dirname(source_dir),
f"{basename}_{clean_additional_filename_descriptor}_{current_datetime}.tgz",
)
# Create the backup
logger.debug(f"Backing up directory [{source_dir}] to [{output_filename}]")
with tarfile.open(output_filename, "w:gz") as tar:
tar.add(source_dir, arcname=os.path.basename(source_dir))
# Ensure the backup has been successfully created before deleting the existing generated configuration directory
if not os.path.exists(output_filename):
error_and_exit(
f"Current generated configuration directory backup failed. Could not write out file [{output_filename}]."
)
# Remove the existing directory
shutil.rmtree(source_dir, ignore_errors=True)
logger.debug(
f"Deleted previous generated configuration directory [{source_dir}]"
)
source_dir.mkdir(parents=True, exist_ok=True)
logger.debug(f"Created clean directory [{source_dir}]")
return source_dir
def _action_orchestrator(
self,
ctx: Context,
action: ServiceAction,
service_names: tuple[str, ...] = None,
):
"""Applies an action to service(s).
Args:
ctx (Context): Click Context for current CLI.
action (ServiceAction): action to apply to service(s), ie start, stop ...
service_names (tuple[str, ...], optional): The name(s) of the service(s) to effect. If not provided the action applies to all services.
"""
hooks = self.cli_configuration.hooks
if action == ServiceAction.START:
action_run_function = self.orchestrator.start
pre_hook = hooks.pre_start
post_hook = hooks.post_start
elif action == ServiceAction.SHUTDOWN:
action_run_function = self.orchestrator.shutdown
pre_hook = hooks.pre_shutdown
post_hook = hooks.post_shutdown
else:
error_and_exit(f"Unhandled action called: [{action.name}]")
pre_run_log_message = (
f"{action.name} "
+ (
", ".join(service_names)
if service_names is not None and len(service_names) > 0
else self.cli_configuration.app_name
)
+ " ..."
)
post_run_log_message = f"{action.name} command finished with code [%i]"
logger.debug(f"Running pre-{action.name} hook")
pre_hook(ctx)
logger.info(pre_run_log_message)
result = action_run_function(ctx.obj, service_names)
logger.debug(f"Running post-{action.name} hook")
post_hook(ctx, result)
logger.info(post_run_log_message, result.returncode)
sys.exit(result.returncode)
def apply_configuration_changes(self, message: str):
"""Applies the current configuration settings to templates to generate application files.
Args:
message (str): The message associated with the changes this applies.
"""
if self.config_repo.is_repo_on_master_branch():
error_and_exit("Cannot apply configuration, repo is on master branch.")
# Commit changes to the configuration repository
self.config_repo.commit_changes(message)
# Re-generated generated configuration
self.__regenerate_generated_configuration()
def set(ctx: Context,
type: str,
encrypted: bool,
setting: str,
value: str = None):
"""Set a configuration value, with specified type, and optional encryption.
If the 'value' isn't passed in, then the user will be prompted. This is
useful in the case where the value is sensitive and shouldn't be captured
in terminal history.
Note - appcli does not currently support encrypting non-string-typed values.
Args:
ctx (Context): Click Context for current CLI.
type (str): Transform the input value as type
encrypted (Bool, flag): flag to indicate if value should be encrypted
setting (str): setting to set
value (str, optional): value to assign to setting
"""
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.CONFIGURE_SET)
# Check if value was not provided
if value is None:
value = click.prompt("Please enter a value", type=str)
# Transform input value as type
transformed_value = StringTransformer.transform(value, type)
# We don't support encrypting non-string-typed values yet, so error and exit.
if encrypted and not isinstance(transformed_value, str):
error_and_exit(
"Cannot encrypt a non-string-typed value. Exiting without setting value."
)
# Set settings value
final_value = (encrypt_text(cli_context, transformed_value)
if encrypted else transformed_value)
configuration = ConfigurationManager(cli_context,
self.cli_configuration)
configuration.set_variable(setting, final_value)
logger.debug(
f"Successfully set variable [{setting}] to [{'### Encrypted Value ###' if encrypted else value}]."
)
def install(ctx, install_dir: Path):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.INSTALL)
logger.info("Generating installer script ...")
# Get the template from the appcli package
launcher_template = pkg_resources.read_text(
templates, INSTALLER_TEMPLATE_FILENAME)
logger.debug(f"Read template file [{INSTALLER_TEMPLATE_FILENAME}]")
environment: str = cli_context.environment
target_install_dir: Path = install_dir / environment
if cli_context.configuration_dir is None:
cli_context = cli_context._replace(
configuration_dir=target_install_dir / "conf")
if cli_context.data_dir is None:
cli_context = cli_context._replace(
data_dir=target_install_dir / "data")
if cli_context.backup_dir is None:
cli_context = cli_context._replace(
backup_dir=target_install_dir / "backup")
render_variables = {
"cli_context": cli_context,
"configuration": self.configuration,
"install_dir": f"{target_install_dir}",
}
logger.debug(
f"Rendering template with render variables: [{render_variables}]"
)
template = Template(
launcher_template,
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
try:
output_text = template.render(render_variables)
print(output_text)
except Exception as e:
error_and_exit(
f"Could not generate file from template. The configuration file is likely missing a setting: {e}"
)
def restore(self, ctx, backup_filename: Path):
"""Restore application data and configuration from the provided local backup `.tgz` file.
This will create a backup of the existing data and config, remove the contents `conf`, `data` and
`conf/.generated` and then extract the backup to the appropriate locations.
`conf`, `data` and `conf/.generated` are mapped into appcli which means we keep the folder but replace their
contents on restore.
Args:
backup_filename (string): The name of the file to use in restoring data. The path of the file will be pulled from `CliContext.obj.backup_dir`.
"""
cli_context: CliContext = ctx.obj
logger.info(
f"Initiating system restore with backup [{backup_filename}]")
# Check that the backup file exists.
backup_dir: Path = cli_context.backup_dir
backup_name: Path = Path(os.path.join(backup_dir, backup_filename))
if not backup_name.is_file():
error_and_exit(f"Backup file [{backup_name}] not found.")
# Perform a backup of the existing application config and data.
logger.debug("Backup existing application data and configuration")
restore_backup_name = self.backup(
ctx, allow_rolling_deletion=False
) # False ensures we don't accidentally delete our backup
logger.debug(
f"Backup(s) complete. Generated backups: [{restore_backup_name}]")
# Extract conf and data directories from the tar.
# This will overwrite the contents of each directory, anything not in the backup (such as files matching the exclude glob patterns) will be left alone.
try:
with tarfile.open(backup_name) as tar:
conf_dir: Path = cli_context.configuration_dir
tar.extractall(conf_dir,
members=self.__members(
tar, os.path.basename(conf_dir)))
data_dir: Path = cli_context.data_dir
tar.extractall(data_dir,
members=self.__members(
tar, os.path.basename(data_dir)))
except Exception as e:
logger.error(f"Failed to extract backup. Reason: {e}")
logger.info("Restore complete.")
def launcher(ctx):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.LAUNCHER)
logger.info("Generating launcher script ...")
# Get the template from the appcli package
launcher_template = pkg_resources.read_text(
templates, LAUNCHER_TEMPLATE_FILENAME)
logger.debug(f"Read template file [{LAUNCHER_TEMPLATE_FILENAME}]")
render_variables = {
"app_version":
os.environ.get("APP_VERSION", "latest"),
"app_name":
configuration.app_name.upper(),
"cli_context":
ctx.obj,
"configuration":
self.configuration,
"current_datetime":
f"{datetime.datetime.utcnow().isoformat()}+00:00", # Since we're using utcnow(), we specify the offset manually
}
logger.debug(
f"Rendering template with render variables: [{render_variables}]"
)
template = Template(
launcher_template,
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
try:
output_text = template.render(render_variables)
print(output_text)
except Exception as e:
error_and_exit(
f"Could not generate file from template. The configuration file is likely missing a setting: {e}"
)
def check_environment():
"""Confirm that mandatory environment variables and additional data directories are defined."""
ENV_VAR_CONFIG_DIR = f"{APP_NAME}_CONFIG_DIR"
ENV_VAR_GENERATED_CONFIG_DIR = f"{APP_NAME}_GENERATED_CONFIG_DIR"
ENV_VAR_DATA_DIR = f"{APP_NAME}_DATA_DIR"
ENV_VAR_ENVIRONMENT = f"{APP_NAME}_ENVIRONMENT"
launcher_set_mandatory_env_vars = [
ENV_VAR_CONFIG_DIR,
ENV_VAR_GENERATED_CONFIG_DIR,
ENV_VAR_DATA_DIR,
ENV_VAR_ENVIRONMENT,
]
launcher_env_vars_set = check_environment_variable_defined(
launcher_set_mandatory_env_vars,
"Mandatory environment variable [%s] not defined. This should be set within the script generated with the 'launcher' command.",
"Cannot run without all mandatory environment variables defined",
)
additional_env_vars_set = check_environment_variable_defined(
configuration.mandatory_additional_env_variables,
'Mandatory additional environment variable [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-env-var "%s"="<value>"',
"Cannot run without all mandatory additional environment variables defined",
)
additional_data_dir_env_vars_set = check_environment_variable_defined(
configuration.mandatory_additional_data_dirs,
'Mandatory additional data directory [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-data-dir "%s"="</path/to/dir>"',
"Cannot run without all mandatory additional data directories defined",
)
if not (
launcher_env_vars_set
and additional_env_vars_set
and additional_data_dir_env_vars_set
):
error_and_exit(
"Some mandatory environment variables weren't set. See error messages above."
)
logger.info("All required environment variables are set.")
def migrate_configuration(self):
"""Migrates the configuration version to the current application version"""
if self.config_repo.is_repo_on_master_branch():
error_and_exit("Cannot migrate, repo is on master branch.")
config_version: str = self.config_repo.get_repository_version()
app_version: str = self.cli_context.app_version
# If the configuration version matches the application version, no migration is required.
if config_version == app_version:
logger.info(
f"Migration not required. Config version [{config_version}] matches application version [{app_version}]"
)
return
logger.info(
f"Migrating configuration version [{config_version}] to match application version [{app_version}]"
)
app_version_branch: str = self.config_repo.generate_branch_name(app_version)
if self.config_repo.does_branch_exist(app_version_branch):
# If the branch already exists, then this version has previously been installed.
logger.warning(
f"Version [{app_version}] of this application was previously installed. Rolling back to previous configuration. Manual remediation may be required."
)
# Switch to that branch, no further migration steps will be taken. This is effectively a roll-back.
self.config_repo.checkout_existing_branch(app_version_branch)
return
# Get the stack-settings file contents if it exists
stack_config_file = self.cli_context.get_stack_configuration_file()
stack_settings_exists_pre_migration = stack_config_file.is_file()
current_stack_settings_variables = (
stack_config_file.read_text()
if stack_settings_exists_pre_migration
else None
)
# Migrate the current configuration variables
current_variables = self.__get_variables_manager().get_all_variables()
# Compare migrated config to the 'clean config' of the new version, and make sure all variables have been set and are the same type.
clean_new_version_variables = VariablesManager(
self.cli_configuration.seed_app_configuration_file
).get_all_variables()
migrated_variables = self.cli_configuration.hooks.migrate_variables(
self.cli_context,
current_variables,
config_version,
clean_new_version_variables,
)
if not self.cli_configuration.hooks.is_valid_variables(
self.cli_context, migrated_variables, clean_new_version_variables
):
error_and_exit(
"Migrated variables did not pass application-specific variables validation function."
)
baseline_template_overrides_dir = (
self.cli_context.get_baseline_template_overrides_dir()
)
override_backup_dir: Path = self.__backup_directory(
baseline_template_overrides_dir
)
configurable_templates_dir = self.cli_context.get_configurable_templates_dir()
configurable_templates_backup_dir = self.__backup_directory(
configurable_templates_dir
)
# Backup and remove the existing generated config dir since it's now out of date
self.__backup_and_create_new_generated_config_dir(config_version)
# Initialise the new configuration branch and directory with all new files
self.__create_new_configuration_branch_and_files()
self.__overwrite_directory(override_backup_dir, baseline_template_overrides_dir)
if self.__directory_is_not_empty(baseline_template_overrides_dir):
logger.warning(
f"Overrides directory [{baseline_template_overrides_dir}] is non-empty, please check for compatibility of overridden files"
)
self.__overwrite_directory(
configurable_templates_backup_dir, configurable_templates_dir
)
if self.__directory_is_not_empty(configurable_templates_dir):
logger.warning(
f"Configurable templates directory [{configurable_templates_dir}] is non-empty, please check for compatibility"
)
# Write out 'migrated' variables file
self.__get_variables_manager().set_all_variables(migrated_variables)
# If stack settings existed pre-migration, then replace the default with the existing settings
if stack_settings_exists_pre_migration:
logger.warning(
"Stack settings file was copied directly from previous version, please check for compatibility"
)
stack_config_file.write_text(current_stack_settings_variables)
# Commit the new variables file
self.config_repo.commit_changes(
f"Migrated variables file from version [{config_version}] to version [{app_version}]"
)
def check_docker_socket():
"""Check that the docker socket exists, and exit if it does not"""
if not os.path.exists("/var/run/docker.sock"):
error_msg = """Docker socket not present. Please launch with a mounted /var/run/docker.sock"""
error_and_exit(error_msg)
def cli(
ctx,
debug,
configuration_dir,
data_dir,
environment,
docker_credentials_file,
additional_data_dir,
additional_env_var,
backup_dir,
):
if debug:
logger.info("Enabling debug logging")
enable_debug_logging()
ctx.obj = CliContext(
configuration_dir=configuration_dir,
data_dir=data_dir,
additional_data_dirs=additional_data_dir,
additional_env_variables=additional_env_var,
environment=environment,
docker_credentials_file=docker_credentials_file,
subcommand_args=ctx.obj,
debug=debug,
app_name=APP_NAME,
app_version=APP_VERSION,
commands=default_commands,
backup_dir=backup_dir,
)
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
sys.exit(1)
# attempt to set desired environment
initialised_environment = {}
for k, v in desired_environment.items():
if v is None:
logger.warning("Environment variable [%s] has not been set", k)
else:
logger.debug("Exporting environment variable [%s]", k)
os.environ[k] = v
initialised_environment[k] = v
if len(initialised_environment) != len(desired_environment):
error_and_exit(
"Could not set desired environment. Please ensure specified environment variables are set."
)
# For the `installer`/`launcher` commands, no further output/checks required.
if ctx.invoked_subcommand in ("launcher", "install"):
# Don't execute this function any further, continue to run subcommand with the current CLI context
return
check_docker_socket()
check_environment()
# Table of configuration variables to print
table = [
["Configuration directory", f"{ctx.obj.configuration_dir}"],
[
"Generated Configuration directory",
f"{ctx.obj.get_generated_configuration_dir()}",
],
["Data directory", f"{ctx.obj.data_dir}"],
["Backup directory", f"{ctx.obj.backup_dir}"],
["Environment", f"{ctx.obj.environment}"],
]
# Print out the configuration values as an aligned table
logger.info(
"%s (version: %s) CLI running with:\n\n%s\n",
APP_NAME,
APP_VERSION,
tabulate(table, colalign=("right", )),
)
if additional_data_dir:
logger.info(
"Additional data directories:\n\n%s\n",
tabulate(
additional_data_dir,
headers=["Environment Variable", "Path"],
colalign=("right", ),
),
)
if additional_env_var:
logger.info(
"Additional environment variables:\n\n%s\n",
tabulate(
additional_env_var,
headers=["Environment Variable", "Value"],
colalign=("right", ),
),
)
