def __migrate(self, ctx):
cli_context: CliContext = ctx.obj
# Perform migration
ConfigurationManager(
cli_context, self.cli_configuration
).migrate_configuration()
logger.info("Migration complete.")
def __migrate(self, ctx):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.MIGRATE)
# Perform migration
ConfigurationManager(cli_context,
self.cli_configuration).migrate_configuration()
logger.info("Migration complete.")
def __save(self, variables: Dict):
"""Saves the supplied Dict of variables to the configuration file
Args:
variables (Dict): the variables to save
"""
full_path = self.configuration_file.absolute().as_posix()
logger.info(f"Saving configuration to [{full_path}] ...")
with open(full_path, "w") as config_file:
self.yaml.dump(variables, config_file)
def encrypt(ctx, text: str):
cli_context: CliContext = ctx.obj
key_file: Path = cli_context.get_key_file()
if not key_file.is_file():
logger.info("Creating encryption key at [%s]", key_file)
crypto.create_and_save_key(key_file)
cipher = Cipher(key_file)
result = cipher.encrypt(text)
print(result)
def backup(self, backup_filename: Path, key_file: Path):
"""
Execute the backup of this remote strategy.
Args:
backup_filename: str. The full Path of the backup to create. This lives in the backup folder.
key_file: Path. The path to the key file.
"""
logger.info(f"Initiating backup [{self.name}]")
if self.should_run():
self.strategy.backup(backup_filename, key_file)
def run(ctx, service_name, extra_args):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.TASK_RUN)
logger.info(
"Running task [%s] with args [%s] ...",
service_name,
extra_args,
)
result = self.orchestrator.task(ctx.obj, service_name, extra_args)
logger.info("Task service finished with code [%i]",
result.returncode)
sys.exit(result.returncode)
def _action_orchestrator(
self,
ctx: Context,
action: ServiceAction,
service_names: tuple[str, ...] = None,
):
"""Applies an action to service(s).
Args:
ctx (Context): Click Context for current CLI.
action (ServiceAction): action to apply to service(s), ie start, stop ...
service_names (tuple[str, ...], optional): The name(s) of the service(s) to effect. If not provided the action applies to all services.
"""
hooks = self.cli_configuration.hooks
if action == ServiceAction.START:
action_run_function = self.orchestrator.start
pre_hook = hooks.pre_start
post_hook = hooks.post_start
elif action == ServiceAction.SHUTDOWN:
action_run_function = self.orchestrator.shutdown
pre_hook = hooks.pre_shutdown
post_hook = hooks.post_shutdown
else:
error_and_exit(f"Unhandled action called: [{action.name}]")
pre_run_log_message = (
f"{action.name} "
+ (
", ".join(service_names)
if service_names is not None and len(service_names) > 0
else self.cli_configuration.app_name
)
+ " ..."
)
post_run_log_message = f"{action.name} command finished with code [%i]"
logger.debug(f"Running pre-{action.name} hook")
pre_hook(ctx)
logger.info(pre_run_log_message)
result = action_run_function(ctx.obj, service_names)
logger.debug(f"Running post-{action.name} hook")
post_hook(ctx, result)
logger.info(post_run_log_message, result.returncode)
sys.exit(result.returncode)
def restore(self, ctx, backup_filename: Path):
"""Restore application data and configuration from the provided local backup `.tgz` file.
This will create a backup of the existing data and config, remove the contents `conf`, `data` and
`conf/.generated` and then extract the backup to the appropriate locations.
`conf`, `data` and `conf/.generated` are mapped into appcli which means we keep the folder but replace their
contents on restore.
Args:
backup_filename (string): The name of the file to use in restoring data. The path of the file will be pulled from `CliContext.obj.backup_dir`.
"""
cli_context: CliContext = ctx.obj
logger.info(
f"Initiating system restore with backup [{backup_filename}]")
# Check that the backup file exists.
backup_dir: Path = cli_context.backup_dir
backup_name: Path = Path(os.path.join(backup_dir, backup_filename))
if not backup_name.is_file():
error_and_exit(f"Backup file [{backup_name}] not found.")
# Perform a backup of the existing application config and data.
logger.debug("Backup existing application data and configuration")
restore_backup_name = self.backup(
ctx, allow_rolling_deletion=False
) # False ensures we don't accidentally delete our backup
logger.debug(
f"Backup(s) complete. Generated backups: [{restore_backup_name}]")
# Extract conf and data directories from the tar.
# This will overwrite the contents of each directory, anything not in the backup (such as files matching the exclude glob patterns) will be left alone.
try:
with tarfile.open(backup_name) as tar:
conf_dir: Path = cli_context.configuration_dir
tar.extractall(conf_dir,
members=self.__members(
tar, os.path.basename(conf_dir)))
data_dir: Path = cli_context.data_dir
tar.extractall(data_dir,
members=self.__members(
tar, os.path.basename(data_dir)))
except Exception as e:
logger.error(f"Failed to extract backup. Reason: {e}")
logger.info("Restore complete.")
def install(ctx, install_dir: Path):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.INSTALL)
logger.info("Generating installer script ...")
# Get the template from the appcli package
launcher_template = pkg_resources.read_text(
templates, INSTALLER_TEMPLATE_FILENAME)
logger.debug(f"Read template file [{INSTALLER_TEMPLATE_FILENAME}]")
environment: str = cli_context.environment
target_install_dir: Path = install_dir / environment
if cli_context.configuration_dir is None:
cli_context = cli_context._replace(
configuration_dir=target_install_dir / "conf")
if cli_context.data_dir is None:
cli_context = cli_context._replace(
data_dir=target_install_dir / "data")
if cli_context.backup_dir is None:
cli_context = cli_context._replace(
backup_dir=target_install_dir / "backup")
render_variables = {
"cli_context": cli_context,
"configuration": self.configuration,
"install_dir": f"{target_install_dir}",
}
logger.debug(
f"Rendering template with render variables: [{render_variables}]"
)
template = Template(
launcher_template,
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
try:
output_text = template.render(render_variables)
print(output_text)
except Exception as e:
error_and_exit(
f"Could not generate file from template. The configuration file is likely missing a setting: {e}"
)
def configure_default(self, app_name):
"""Applies the default opinionated configuration to Keycloak
This does the following:
- Creates a realm named '<app_name>'
- For realm '<app_name>', creates a client with the name '<app_name>', which has an audience mapper to itself,
and redirect URIs of ["*"]
- For realm '<app_name>', creates a realm role '<app_name>-admin'
- For realm '<app_name>', creates a user 'test.user' with password 'password', and assigns the realm role
'<app_name>-admin'
"""
self.create_realm(app_name)
logger.info(f"Created realm [{app_name}]")
client_payload = {
"redirectUris": ["*"],
"protocolMappers": [{
"name": f"{app_name}-audience",
"protocol": "openid-connect",
"protocolMapper": "oidc-audience-mapper",
"consentRequired": "false",
"config": {
"included.client.audience": app_name,
"id.token.claim": "false",
"access.token.claim": "true",
},
}],
}
self.create_client(app_name, app_name, client_payload)
secret = self.get_client_secret(app_name, app_name)
logger.info(f"Created client [{app_name}] with secret [{secret}]")
realm_role = f"{app_name}-admin"
self.create_realm_role(app_name, realm_role)
logger.info(f"Created realm role [{realm_role}]")
username = "******"
self.create_user(app_name, username, "password", "Test", "User",
"*****@*****.**")
logger.info(
f"Created user [test.user] with password [password] in realm [{app_name}]"
)
self.assign_realm_role(app_name, username, realm_role)
logger.info(f"Assigned realm role [{realm_role}] to user [test.user]")
def view_backups(self, ctx):
"""Display a list of available backups that were found in the backup folder."""
cli_context: CliContext = ctx.obj
logger.info("Displaying all locally-available backups.")
backup_dir: Path = cli_context.backup_dir
files = [
os.path.join(path[len(str(backup_dir)):].lstrip("/"),
name) # noqa: E203
for path, subdirs, files in os.walk(backup_dir) for name in files
]
backup_dir_files = sorted(
files,
reverse=True,
)
for backup in backup_dir_files:
print(backup)
def __pre_configure_get_and_set_validation(self, cli_context: CliContext):
"""Ensures the system is in a valid state for 'configure get'.
Args:
cli_context (CliContext): The current CLI context.
"""
logger.info(
"Checking system configuration is valid before 'configure get' ..."
)
# Block if the config dir doesn't exist as there's nothing to get or set
must_succeed_checks = [confirm_config_dir_exists]
execute_validation_functions(
cli_context=cli_context,
must_succeed_checks=must_succeed_checks,
)
logger.info("System configuration is valid")
def start(ctx, force, service_name):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.SERVICE_START, force)
hooks = self.cli_configuration.hooks
logger.debug("Running pre-start hook")
hooks.pre_start(ctx)
logger.info("Starting %s ...", configuration.app_name)
result = self.orchestrator.start(ctx.obj, service_name)
logger.debug("Running post-start hook")
hooks.post_start(ctx, result)
logger.info("Start command finished with code [%i]",
result.returncode)
sys.exit(result.returncode)
def launcher(ctx):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.LAUNCHER)
logger.info("Generating launcher script ...")
# Get the template from the appcli package
launcher_template = pkg_resources.read_text(
templates, LAUNCHER_TEMPLATE_FILENAME)
logger.debug(f"Read template file [{LAUNCHER_TEMPLATE_FILENAME}]")
render_variables = {
"app_version":
os.environ.get("APP_VERSION", "latest"),
"app_name":
configuration.app_name.upper(),
"cli_context":
ctx.obj,
"configuration":
self.configuration,
"current_datetime":
f"{datetime.datetime.utcnow().isoformat()}+00:00", # Since we're using utcnow(), we specify the offset manually
}
logger.debug(
f"Rendering template with render variables: [{render_variables}]"
)
template = Template(
launcher_template,
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True,
)
try:
output_text = template.render(render_variables)
print(output_text)
except Exception as e:
error_and_exit(
f"Could not generate file from template. The configuration file is likely missing a setting: {e}"
)
def start(ctx, force, service_name):
hooks = self.cli_configuration.hooks
# TODO: run self.cli_configuration.hooks.is_valid_variables() to confirm variables are valid
logger.debug("Running pre-start hook")
hooks.pre_start(ctx)
cli_context: CliContext = ctx.obj
self.__pre_start_validation(cli_context, force=force)
logger.info("Starting %s ...", configuration.app_name)
result = self.orchestrator.start(ctx.obj, service_name)
logger.debug("Running post-start hook")
hooks.post_start(ctx, result)
logger.info("Start command finished with code [%i]",
result.returncode)
sys.exit(result.returncode)
def __rolling_backup_deletion(self, backup_dir: Path):
"""Delete old backups, will only keep the most recent backups.
The number of backups to keep is specified in the stack settings configuration file.
Note that the age of the backup is derived from the alphanumerical order of the backup filename.
This means that any supplementary files in the backup directory could have unintended consequences during
rolling deletion.
Backup files are intentionally named with a datetime stamp to enable age ordering.
Args:
backup_dir (Path): The directory that contains the backups.
"""
# If the backup limit is set to 0 then we never want to delete a backup.
if self.backup_limit == 0:
logger.debug("Backup limit is 0 - skipping rolling deletion.")
return
logger.info(
f"Removing old backups - retaining at least the last [{self.backup_limit}] backups ..."
)
# Get all files from our backup directory
backup_files = os.listdir(backup_dir)
# Sort the backups alphanumerically by filename. Note that this assumes all files in the backup dir are backup
# files that use a time-sortable naming convention.
backup_dir_files = sorted(
backup_files,
reverse=True,
)
# Get the backups to delete by taking our sorted list of backups and then delete from the appropriate index
# onward.
backups_to_delete = backup_dir_files[
self.
backup_limit: # noqa: E203 - Disable flake8 error on spaces before a `:`
]
for backup_to_delete in backups_to_delete:
backup_file: Path = Path(os.path.join(backup_dir,
backup_to_delete))
logger.info(f"Deleting backup file [{backup_file}]")
os.remove(backup_file)
def check_environment():
"""Confirm that mandatory environment variables and additional data directories are defined."""
ENV_VAR_CONFIG_DIR = f"{APP_NAME}_CONFIG_DIR"
ENV_VAR_GENERATED_CONFIG_DIR = f"{APP_NAME}_GENERATED_CONFIG_DIR"
ENV_VAR_DATA_DIR = f"{APP_NAME}_DATA_DIR"
ENV_VAR_ENVIRONMENT = f"{APP_NAME}_ENVIRONMENT"
launcher_set_mandatory_env_vars = [
ENV_VAR_CONFIG_DIR,
ENV_VAR_GENERATED_CONFIG_DIR,
ENV_VAR_DATA_DIR,
ENV_VAR_ENVIRONMENT,
]
launcher_env_vars_set = check_environment_variable_defined(
launcher_set_mandatory_env_vars,
"Mandatory environment variable [%s] not defined. This should be set within the script generated with the 'launcher' command.",
"Cannot run without all mandatory environment variables defined",
)
additional_env_vars_set = check_environment_variable_defined(
configuration.mandatory_additional_env_variables,
'Mandatory additional environment variable [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-env-var "%s"="<value>"',
"Cannot run without all mandatory additional environment variables defined",
)
additional_data_dir_env_vars_set = check_environment_variable_defined(
configuration.mandatory_additional_data_dirs,
'Mandatory additional data directory [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-data-dir "%s"="</path/to/dir>"',
"Cannot run without all mandatory additional data directories defined",
)
if not (
launcher_env_vars_set
and additional_env_vars_set
and additional_data_dir_env_vars_set
):
error_and_exit(
"Some mandatory environment variables weren't set. See error messages above."
)
logger.info("All required environment variables are set.")
def decrypt_file(encrypted_file: Path, key_file: Path) -> Path:
"""
Decrypts the specified file using the supplied key.
Args:
encrypted_file (Path): File to decrypt.
key_file (Path): Key to use for decryption.
Returns:
Path: Path to the decrypted file.
"""
if not key_file.is_file():
logger.info("No decryption key found. [%s] will not be decrypted.",
encrypted_file)
return encrypted_file
logger.debug("Decrypting file [%s] using [%s].", str(encrypted_file),
key_file)
decrypted_file: Path = Path(NamedTemporaryFile(delete=False).name)
crypto.decrypt_values_in_file(encrypted_file, decrypted_file, key_file)
return decrypted_file
def __pre_shutdown_validation(self,
cli_context: CliContext,
force: bool = False):
"""Ensures the system is in a valid state for shutdown.
Args:
cli_context (CliContext): The current CLI context.
force (bool, optional): If True, only warns on validation failures, rather than exiting.
"""
logger.info(
"Checking system configuration is valid before shutting down ...")
execute_validation_functions(
cli_context=cli_context,
must_succeed_checks=[
confirm_generated_config_dir_exists
], # Only block shuting down the system on the generated config not existing
force=force,
)
logger.info("System configuration is valid")
def patched_subprocess_run(docker_compose_command, capture_output=True, input=None):
# TODO: We should take advantage of the printed command to perform test validation
logger.info(f"PYTEST_PATCHED_DOCKER_COMPOSE_COMMAND=[{docker_compose_command}]")
if all([x in docker_compose_command for x in ["config", "--services"]]):
# patch for fetching the valid service names, used by the verify_service_names orchestrator.
valid_services = "\n".join(DOCKER_COMPOSE_SERVICES) + "\n"
return subprocess.CompletedProcess(
returncode=0, args=None, stdout=bytes(valid_services, "utf-8")
)
elif all([x in docker_compose_command for x in ["up", "-d"]]):
# patch for starting all services, used by the start orchestrator.
return subprocess.CompletedProcess(returncode=0, args=None)
elif all([x in docker_compose_command for x in ["down"]]):
# patch for shutting down all services, used by the shutdown orchestrator for shutting down all services.
return subprocess.CompletedProcess(returncode=0, args=None)
elif all([x in docker_compose_command for x in ["rm", "-fsv"]]):
# patch for shutting down all services, used by the shutdown orchestrator when given provided with specific services.
return subprocess.CompletedProcess(returncode=0, args=None)
else:
# patch for unknown command action
print("Unknown command: %s", docker_compose_command)
return subprocess.CompletedProcess(returncode=1, args=None)
def init(ctx):
print_header(
f"Seeding configuration directory for {self.app_name}")
cli_context: CliContext = ctx.obj
# Run pre-hooks
hooks = self.cli_configuration.hooks
logger.debug("Running pre-configure init hook")
hooks.pre_configure_init(ctx)
# Initialise configuration directory
logger.debug("Initialising configuration directory")
ConfigurationManager(
cli_context,
self.cli_configuration).initialise_configuration()
# Run post-hooks
logger.debug("Running post-configure init hook")
hooks.post_configure_init(ctx)
logger.info("Finished initialising configuration")
def apply(ctx, message, force):
cli_context: CliContext = ctx.obj
# TODO: run self.cli_configuration.hooks.is_valid_variables() to confirm variables are valid
# Run pre-hooks
hooks = self.cli_configuration.hooks
logger.debug("Running pre-configure apply hook")
hooks.pre_configure_apply(ctx)
# Apply changes
logger.debug("Applying configuration")
ConfigurationManager(
cli_context,
self.cli_configuration).apply_configuration_changes(
message, force=force)
# Run post-hooks
logger.debug("Running post-configure apply hook")
hooks.post_configure_apply(ctx)
logger.info("Finished applying configuration")
def override(ctx, template, force):
cli_context: CliContext = ctx.obj
baseline_templates_dir = self.cli_configuration.baseline_templates_dir
template_file_path = Path(os.path.join(baseline_templates_dir, template))
if not template_file_path.exists():
error_and_exit(f"Could not find template [{template}]")
override_file_path = (
cli_context.get_baseline_template_overrides_dir().joinpath(template)
)
if override_file_path.exists():
if not force:
error_and_exit(
f"Override template already exists at [{override_file_path}]. Use --force to overwrite."
)
logger.info("Force flag provided. Overwriting existing override file.")
# Makes the override and sub folders if they do not exist
os.makedirs(override_file_path.parent, exist_ok=True)
shutil.copy2(template_file_path, override_file_path)
logger.info(f"Copied template [{template}] to [{override_file_path}]")
def apply(ctx, message, force):
cli_context: CliContext = ctx.obj
# We require the '--force' option to allow forcibly applying and
# overwriting existing modified generated configuration.
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.CONFIGURE_APPLY, force)
# Run pre-hooks
hooks = self.cli_configuration.hooks
logger.debug("Running pre-configure apply hook")
hooks.pre_configure_apply(ctx)
# Apply changes
logger.debug("Applying configuration")
ConfigurationManager(
cli_context,
self.cli_configuration).apply_configuration_changes(message)
# Run post-hooks
logger.debug("Running post-configure apply hook")
hooks.post_configure_apply(ctx)
logger.info("Finished applying configuration")
def init(ctx):
cli_context: CliContext = ctx.obj
cli_context.get_configuration_dir_state().verify_command_allowed(
AppcliCommand.CONFIGURE_INIT)
print_header(
f"Seeding configuration directory for {self.app_name}")
# Run pre-hooks
hooks = self.cli_configuration.hooks
logger.debug("Running pre-configure init hook")
hooks.pre_configure_init(ctx)
# Initialise configuration directory
logger.debug("Initialising configuration directory")
ConfigurationManager(
cli_context,
self.cli_configuration).initialise_configuration()
# Run post-hooks
logger.debug("Running post-configure init hook")
hooks.post_configure_init(ctx)
logger.info("Finished initialising configuration")
def __regenerate_generated_configuration(self):
"""Generate the generated configuration files"""
print_header("Generating configuration files")
generated_configuration_dir = self.__backup_and_create_new_generated_config_dir(
self.config_repo.get_repository_version()
)
logger.info("Generating configuration from default templates")
self.__apply_templates_from_directory(
self.cli_configuration.baseline_templates_dir, generated_configuration_dir
)
logger.info("Generating configuration from override templates")
self.__apply_templates_from_directory(
self.cli_context.get_baseline_template_overrides_dir(),
generated_configuration_dir,
)
logger.info("Generating configuration from configurable templates")
self.__apply_templates_from_directory(
self.cli_context.get_configurable_templates_dir(),
generated_configuration_dir,
)
files_to_decrypt = self.cli_configuration.decrypt_generated_files
if len(files_to_decrypt) > 0:
self.__decrypt_generated_files(
self.cli_context.get_key_file(),
self.cli_context.get_generated_configuration_dir(),
files_to_decrypt,
)
# Copy the settings file that was used to generate the templates
self.__copy_settings_files_to_generated_dir()
# Generate the metadata file
self.__generate_configuration_metadata_file()
# By re-instantiating the 'GeneratedConfigurationGitRepository', we put
# the generated config repo under version control.
generated_config_repo = GeneratedConfigurationGitRepository(
self.cli_context.get_generated_configuration_dir()
)
logger.debug(
f"Generated configuration at [{generated_config_repo.get_repo_path()}]"
)
def migrate_configuration(self):
"""Migrates the configuration version to the current application version"""
if self.config_repo.is_repo_on_master_branch():
error_and_exit("Cannot migrate, repo is on master branch.")
config_version: str = self.config_repo.get_repository_version()
app_version: str = self.cli_context.app_version
# If the configuration version matches the application version, no migration is required.
if config_version == app_version:
logger.info(
f"Migration not required. Config version [{config_version}] matches application version [{app_version}]"
)
return
logger.info(
f"Migrating configuration version [{config_version}] to match application version [{app_version}]"
)
app_version_branch: str = self.config_repo.generate_branch_name(app_version)
if self.config_repo.does_branch_exist(app_version_branch):
# If the branch already exists, then this version has previously been installed.
logger.warning(
f"Version [{app_version}] of this application was previously installed. Rolling back to previous configuration. Manual remediation may be required."
)
# Switch to that branch, no further migration steps will be taken. This is effectively a roll-back.
self.config_repo.checkout_existing_branch(app_version_branch)
return
# Get the stack-settings file contents if it exists
stack_config_file = self.cli_context.get_stack_configuration_file()
stack_settings_exists_pre_migration = stack_config_file.is_file()
current_stack_settings_variables = (
stack_config_file.read_text()
if stack_settings_exists_pre_migration
else None
)
# Migrate the current configuration variables
current_variables = self.__get_variables_manager().get_all_variables()
# Compare migrated config to the 'clean config' of the new version, and make sure all variables have been set and are the same type.
clean_new_version_variables = VariablesManager(
self.cli_configuration.seed_app_configuration_file
).get_all_variables()
migrated_variables = self.cli_configuration.hooks.migrate_variables(
self.cli_context,
current_variables,
config_version,
clean_new_version_variables,
)
if not self.cli_configuration.hooks.is_valid_variables(
self.cli_context, migrated_variables, clean_new_version_variables
):
error_and_exit(
"Migrated variables did not pass application-specific variables validation function."
)
baseline_template_overrides_dir = (
self.cli_context.get_baseline_template_overrides_dir()
)
override_backup_dir: Path = self.__backup_directory(
baseline_template_overrides_dir
)
configurable_templates_dir = self.cli_context.get_configurable_templates_dir()
configurable_templates_backup_dir = self.__backup_directory(
configurable_templates_dir
)
# Backup and remove the existing generated config dir since it's now out of date
self.__backup_and_create_new_generated_config_dir(config_version)
# Initialise the new configuration branch and directory with all new files
self.__create_new_configuration_branch_and_files()
self.__overwrite_directory(override_backup_dir, baseline_template_overrides_dir)
if self.__directory_is_not_empty(baseline_template_overrides_dir):
logger.warning(
f"Overrides directory [{baseline_template_overrides_dir}] is non-empty, please check for compatibility of overridden files"
)
self.__overwrite_directory(
configurable_templates_backup_dir, configurable_templates_dir
)
if self.__directory_is_not_empty(configurable_templates_dir):
logger.warning(
f"Configurable templates directory [{configurable_templates_dir}] is non-empty, please check for compatibility"
)
# Write out 'migrated' variables file
self.__get_variables_manager().set_all_variables(migrated_variables)
# If stack settings existed pre-migration, then replace the default with the existing settings
if stack_settings_exists_pre_migration:
logger.warning(
"Stack settings file was copied directly from previous version, please check for compatibility"
)
stack_config_file.write_text(current_stack_settings_variables)
# Commit the new variables file
self.config_repo.commit_changes(
f"Migrated variables file from version [{config_version}] to version [{app_version}]"
)
def execute_validation_functions(
cli_context: CliContext,
must_succeed_checks: Iterable[Callable[[click.Context], None]] = [],
should_succeed_checks: Iterable[Callable[[click.Context], None]] = [],
force: bool = False,
):
"""Run validation check functions. There are two types of checks: 'must' checks and 'should' checks.
'Must' check functions must complete and exit without raising Exceptions, or else the whole validation check will fail.
'Should' check functions should completed and exit without raising Exceptions. If any Exceptions are raised, and 'force' is False,
then this validation check will fail. If Exceptions are raised and 'force' is True, then these are raised as warning to the user
but the validation is successful.
All the 'Must' and 'Should' checks (taking into account 'force') need to succeed in order for this validation to be successful.
Args:
cli_context (CliContext): The current CLI context.
must_succeed_checks (Iterable[Callable[[click.Context], None]], optional): The check functions to run which must not raise any exceptions
in order for the validation to succeed.
should_succeed_checks (Iterable[Callable[[click.Context], None]], optional): The check functions to run, which may raise exceptions. If
'force' is True, any Exceptions are displayed as 'warnings' and won't fail the validation check. If 'force' is False, any Exceptions
will failed the validation.
force (bool, optional): Whether to ignore any should_succeed_checks which fail. Defaults to False.
"""
logger.info("Performing validation ...")
# Get the blocking errors
must_succeed_errors = _run_checks(cli_context, must_succeed_checks)
must_succeed_error_messages = "\n- ".join(set(must_succeed_errors))
# Get the non-blocking errors - 'warnings'
should_succeed_errors = _run_checks(cli_context, should_succeed_checks)
should_succeed_error_messages = "\n- ".join(set(should_succeed_errors))
all_errors = must_succeed_errors + should_succeed_errors
# If there's no errors, validation ends here and is successful
if not all_errors:
logger.debug("No errors found in validation.")
return
# If there's errors in the 'should' checks and force is True, then only warn for those errors
if should_succeed_errors and force:
logger.warning(
"Force flag `--force` applied. Ignoring the following issues:\n- %s",
should_succeed_error_messages,
)
# If there's no 'must' errors now, validation is successful
if not must_succeed_errors:
return
output_error_message = "Errors found during validation.\n\n"
# If there's forced 'should' errors, then we've already warned, and don't need to include this in the error message
if should_succeed_errors and not force:
output_error_message += f"Force-able issues:\n- {should_succeed_error_messages}\nUse the `--force` flag to ignore these issues.\n\n"
if must_succeed_errors:
output_error_message += f"Blocking issues (these cannot be bypassed and must be fixed):\n- {must_succeed_error_messages}\n\n"
output_error_message += "Validation failed. See error messages above."
error_and_exit(output_error_message)
def cli(
ctx,
debug,
configuration_dir,
data_dir,
environment,
docker_credentials_file,
additional_data_dir,
additional_env_var,
backup_dir,
):
if debug:
logger.info("Enabling debug logging")
enable_debug_logging()
ctx.obj = CliContext(
configuration_dir=configuration_dir,
data_dir=data_dir,
additional_data_dirs=additional_data_dir,
additional_env_variables=additional_env_var,
environment=environment,
docker_credentials_file=docker_credentials_file,
subcommand_args=ctx.obj,
debug=debug,
app_name=APP_NAME,
app_version=APP_VERSION,
commands=default_commands,
backup_dir=backup_dir,
)
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
sys.exit(1)
# attempt to set desired environment
initialised_environment = {}
for k, v in desired_environment.items():
if v is None:
logger.warning("Environment variable [%s] has not been set", k)
else:
logger.debug("Exporting environment variable [%s]", k)
os.environ[k] = v
initialised_environment[k] = v
if len(initialised_environment) != len(desired_environment):
error_and_exit(
"Could not set desired environment. Please ensure specified environment variables are set."
)
# For the `installer`/`launcher` commands, no further output/checks required.
if ctx.invoked_subcommand in ("launcher", "install"):
# Don't execute this function any further, continue to run subcommand with the current CLI context
return
check_docker_socket()
check_environment()
# Table of configuration variables to print
table = [
["Configuration directory", f"{ctx.obj.configuration_dir}"],
[
"Generated Configuration directory",
f"{ctx.obj.get_generated_configuration_dir()}",
],
["Data directory", f"{ctx.obj.data_dir}"],
["Backup directory", f"{ctx.obj.backup_dir}"],
["Environment", f"{ctx.obj.environment}"],
]
# Print out the configuration values as an aligned table
logger.info(
"%s (version: %s) CLI running with:\n\n%s\n",
APP_NAME,
APP_VERSION,
tabulate(table, colalign=("right", )),
)
if additional_data_dir:
logger.info(
"Additional data directories:\n\n%s\n",
tabulate(
additional_data_dir,
headers=["Environment Variable", "Path"],
colalign=("right", ),
),
)
if additional_env_var:
logger.info(
"Additional environment variables:\n\n%s\n",
tabulate(
additional_env_var,
headers=["Environment Variable", "Value"],
colalign=("right", ),
),
)
def backup(self,
ctx: Context,
backup_name: str = None,
allow_rolling_deletion: bool = True) -> List[Path]:
"""
Perform all backups present in the configuration file.
Args:
ctx: (Context). The current Click Context.
allow_rolling_deletion: (bool). Enable rolling backups (default True). Set to False to disable rolling
backups and keep all backup files.
Returns:
List[Path]: The list of backup files generated by running all backups.
"""
cli_context: CliContext = ctx.obj
logger.info("Initiating system backup")
# Get the key file for decrypting encrypted values used in a remote backup.
key_file = cli_context.get_key_file()
completed_backups = []
for backup_config in self.backups:
backup = BackupConfig.from_dict(backup_config)
if backup_name is not None and backup.name != backup_name:
logger.debug(
f"Skipping backup [{backup.name}] - only running backup [{backup_name}]"
)
continue
# Check if the set frequency matches today, if it does not then do not continue with the current backup.
if not backup.should_run():
continue
# create the backup
logger.debug(f"Backup [{backup.name}] running...")
backup_filename = backup.backup(ctx, allow_rolling_deletion)
completed_backups.append((backup.name, backup_filename))
logger.debug(
f"Backup [{backup.name}] complete. Output file: [{backup_filename}]"
)
# Get any remote backup strategies.
remote_backups = backup.get_remote_backups()
# Execute each of the remote backup strategies with the local backup file.
for remote_backup in remote_backups:
try:
logger.debug(
f"Backup [{backup.name}] remote backup [{remote_backup.name}] running..."
)
remote_backup.backup(backup_filename, key_file)
logger.debug(
f"Backup [{backup.name}] remote backup [{remote_backup.name}] complete."
)
except Exception as e:
logger.error(
f"Error while executing remote strategy [{remote_backup.name}] - {e}"
)
traceback.print_exc()
logger.info("Backups complete.")
if len(completed_backups) > 0:
logger.debug(f"Completed backups [{completed_backups}].")
else:
logger.warning(
"No backups successfully ran or completed. Use --debug flag for more detailed logs."
)
return completed_backups
