def migrate_configuration(self):
"""Migrates the configuration version to the current application version"""
if self.config_repo.is_repo_on_master_branch():
error_and_exit("Cannot migrate, repo is on master branch.")
config_version: str = self.config_repo.get_repository_version()
app_version: str = self.cli_context.app_version
# If the configuration version matches the application version, no migration is required.
if config_version == app_version:
logger.info(
f"Migration not required. Config version [{config_version}] matches application version [{app_version}]"
)
return
logger.info(
f"Migrating configuration version [{config_version}] to match application version [{app_version}]"
)
app_version_branch: str = self.config_repo.generate_branch_name(app_version)
if self.config_repo.does_branch_exist(app_version_branch):
# If the branch already exists, then this version has previously been installed.
logger.warning(
f"Version [{app_version}] of this application was previously installed. Rolling back to previous configuration. Manual remediation may be required."
)
# Switch to that branch, no further migration steps will be taken. This is effectively a roll-back.
self.config_repo.checkout_existing_branch(app_version_branch)
return
# Get the stack-settings file contents if it exists
stack_config_file = self.cli_context.get_stack_configuration_file()
stack_settings_exists_pre_migration = stack_config_file.is_file()
current_stack_settings_variables = (
stack_config_file.read_text()
if stack_settings_exists_pre_migration
else None
)
# Migrate the current configuration variables
current_variables = self.__get_variables_manager().get_all_variables()
# Compare migrated config to the 'clean config' of the new version, and make sure all variables have been set and are the same type.
clean_new_version_variables = VariablesManager(
self.cli_configuration.seed_app_configuration_file
).get_all_variables()
migrated_variables = self.cli_configuration.hooks.migrate_variables(
self.cli_context,
current_variables,
config_version,
clean_new_version_variables,
)
if not self.cli_configuration.hooks.is_valid_variables(
self.cli_context, migrated_variables, clean_new_version_variables
):
error_and_exit(
"Migrated variables did not pass application-specific variables validation function."
)
baseline_template_overrides_dir = (
self.cli_context.get_baseline_template_overrides_dir()
)
override_backup_dir: Path = self.__backup_directory(
baseline_template_overrides_dir
)
configurable_templates_dir = self.cli_context.get_configurable_templates_dir()
configurable_templates_backup_dir = self.__backup_directory(
configurable_templates_dir
)
# Backup and remove the existing generated config dir since it's now out of date
self.__backup_and_create_new_generated_config_dir(config_version)
# Initialise the new configuration branch and directory with all new files
self.__create_new_configuration_branch_and_files()
self.__overwrite_directory(override_backup_dir, baseline_template_overrides_dir)
if self.__directory_is_not_empty(baseline_template_overrides_dir):
logger.warning(
f"Overrides directory [{baseline_template_overrides_dir}] is non-empty, please check for compatibility of overridden files"
)
self.__overwrite_directory(
configurable_templates_backup_dir, configurable_templates_dir
)
if self.__directory_is_not_empty(configurable_templates_dir):
logger.warning(
f"Configurable templates directory [{configurable_templates_dir}] is non-empty, please check for compatibility"
)
# Write out 'migrated' variables file
self.__get_variables_manager().set_all_variables(migrated_variables)
# If stack settings existed pre-migration, then replace the default with the existing settings
if stack_settings_exists_pre_migration:
logger.warning(
"Stack settings file was copied directly from previous version, please check for compatibility"
)
stack_config_file.write_text(current_stack_settings_variables)
# Commit the new variables file
self.config_repo.commit_changes(
f"Migrated variables file from version [{config_version}] to version [{app_version}]"
)
def execute_validation_functions(
cli_context: CliContext,
must_succeed_checks: Iterable[Callable[[click.Context], None]] = [],
should_succeed_checks: Iterable[Callable[[click.Context], None]] = [],
force: bool = False,
):
"""Run validation check functions. There are two types of checks: 'must' checks and 'should' checks.
'Must' check functions must complete and exit without raising Exceptions, or else the whole validation check will fail.
'Should' check functions should completed and exit without raising Exceptions. If any Exceptions are raised, and 'force' is False,
then this validation check will fail. If Exceptions are raised and 'force' is True, then these are raised as warning to the user
but the validation is successful.
All the 'Must' and 'Should' checks (taking into account 'force') need to succeed in order for this validation to be successful.
Args:
cli_context (CliContext): The current CLI context.
must_succeed_checks (Iterable[Callable[[click.Context], None]], optional): The check functions to run which must not raise any exceptions
in order for the validation to succeed.
should_succeed_checks (Iterable[Callable[[click.Context], None]], optional): The check functions to run, which may raise exceptions. If
'force' is True, any Exceptions are displayed as 'warnings' and won't fail the validation check. If 'force' is False, any Exceptions
will failed the validation.
force (bool, optional): Whether to ignore any should_succeed_checks which fail. Defaults to False.
"""
logger.info("Performing validation ...")
# Get the blocking errors
must_succeed_errors = _run_checks(cli_context, must_succeed_checks)
must_succeed_error_messages = "\n- ".join(set(must_succeed_errors))
# Get the non-blocking errors - 'warnings'
should_succeed_errors = _run_checks(cli_context, should_succeed_checks)
should_succeed_error_messages = "\n- ".join(set(should_succeed_errors))
all_errors = must_succeed_errors + should_succeed_errors
# If there's no errors, validation ends here and is successful
if not all_errors:
logger.debug("No errors found in validation.")
return
# If there's errors in the 'should' checks and force is True, then only warn for those errors
if should_succeed_errors and force:
logger.warning(
"Force flag `--force` applied. Ignoring the following issues:\n- %s",
should_succeed_error_messages,
)
# If there's no 'must' errors now, validation is successful
if not must_succeed_errors:
return
output_error_message = "Errors found during validation.\n\n"
# If there's forced 'should' errors, then we've already warned, and don't need to include this in the error message
if should_succeed_errors and not force:
output_error_message += f"Force-able issues:\n- {should_succeed_error_messages}\nUse the `--force` flag to ignore these issues.\n\n"
if must_succeed_errors:
output_error_message += f"Blocking issues (these cannot be bypassed and must be fixed):\n- {must_succeed_error_messages}\n\n"
output_error_message += "Validation failed. See error messages above."
error_and_exit(output_error_message)
def backup(self,
ctx: Context,
backup_name: str = None,
allow_rolling_deletion: bool = True) -> List[Path]:
"""
Perform all backups present in the configuration file.
Args:
ctx: (Context). The current Click Context.
allow_rolling_deletion: (bool). Enable rolling backups (default True). Set to False to disable rolling
backups and keep all backup files.
Returns:
List[Path]: The list of backup files generated by running all backups.
"""
cli_context: CliContext = ctx.obj
logger.info("Initiating system backup")
# Get the key file for decrypting encrypted values used in a remote backup.
key_file = cli_context.get_key_file()
completed_backups = []
for backup_config in self.backups:
backup = BackupConfig.from_dict(backup_config)
if backup_name is not None and backup.name != backup_name:
logger.debug(
f"Skipping backup [{backup.name}] - only running backup [{backup_name}]"
)
continue
# Check if the set frequency matches today, if it does not then do not continue with the current backup.
if not backup.should_run():
continue
# create the backup
logger.debug(f"Backup [{backup.name}] running...")
backup_filename = backup.backup(ctx, allow_rolling_deletion)
completed_backups.append((backup.name, backup_filename))
logger.debug(
f"Backup [{backup.name}] complete. Output file: [{backup_filename}]"
)
# Get any remote backup strategies.
remote_backups = backup.get_remote_backups()
# Execute each of the remote backup strategies with the local backup file.
for remote_backup in remote_backups:
try:
logger.debug(
f"Backup [{backup.name}] remote backup [{remote_backup.name}] running..."
)
remote_backup.backup(backup_filename, key_file)
logger.debug(
f"Backup [{backup.name}] remote backup [{remote_backup.name}] complete."
)
except Exception as e:
logger.error(
f"Error while executing remote strategy [{remote_backup.name}] - {e}"
)
traceback.print_exc()
logger.info("Backups complete.")
if len(completed_backups) > 0:
logger.debug(f"Completed backups [{completed_backups}].")
else:
logger.warning(
"No backups successfully ran or completed. Use --debug flag for more detailed logs."
)
return completed_backups
def cli(
ctx,
debug,
configuration_dir,
data_dir,
environment,
docker_credentials_file,
additional_data_dir,
additional_env_var,
backup_dir,
):
if debug:
logger.info("Enabling debug logging")
enable_debug_logging()
ctx.obj = CliContext(
configuration_dir=configuration_dir,
data_dir=data_dir,
additional_data_dirs=additional_data_dir,
additional_env_variables=additional_env_var,
environment=environment,
docker_credentials_file=docker_credentials_file,
subcommand_args=ctx.obj,
debug=debug,
app_name=APP_NAME,
app_version=APP_VERSION,
commands=default_commands,
backup_dir=backup_dir,
)
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
sys.exit(1)
# attempt to set desired environment
initialised_environment = {}
for k, v in desired_environment.items():
if v is None:
logger.warning("Environment variable [%s] has not been set", k)
else:
logger.debug("Exporting environment variable [%s]", k)
os.environ[k] = v
initialised_environment[k] = v
if len(initialised_environment) != len(desired_environment):
error_and_exit(
"Could not set desired environment. Please ensure specified environment variables are set."
)
# For the `installer`/`launcher` commands, no further output/checks required.
if ctx.invoked_subcommand in ("launcher", "install"):
# Don't execute this function any further, continue to run subcommand with the current CLI context
return
check_docker_socket()
check_environment()
# Table of configuration variables to print
table = [
["Configuration directory", f"{ctx.obj.configuration_dir}"],
[
"Generated Configuration directory",
f"{ctx.obj.get_generated_configuration_dir()}",
],
["Data directory", f"{ctx.obj.data_dir}"],
["Backup directory", f"{ctx.obj.backup_dir}"],
["Environment", f"{ctx.obj.environment}"],
]
# Print out the configuration values as an aligned table
logger.info(
"%s (version: %s) CLI running with:\n\n%s\n",
APP_NAME,
APP_VERSION,
tabulate(table, colalign=("right", )),
)
if additional_data_dir:
logger.info(
"Additional data directories:\n\n%s\n",
tabulate(
additional_data_dir,
headers=["Environment Variable", "Path"],
colalign=("right", ),
),
)
if additional_env_var:
logger.info(
"Additional environment variables:\n\n%s\n",
tabulate(
additional_env_var,
headers=["Environment Variable", "Value"],
colalign=("right", ),
),
)
def diff(ctx):
cli_context: CliContext = ctx.obj
baseline_templates_dir = self.cli_configuration.baseline_templates_dir
override_templates_dir = cli_context.get_baseline_template_overrides_dir()
template_files_rel_paths = get_relative_paths_of_all_files_in_directory(
baseline_templates_dir
)
override_files_rel_paths = get_relative_paths_of_all_files_in_directory(
override_templates_dir
)
not_overriding_overrides = [
f for f in override_files_rel_paths if f not in template_files_rel_paths
]
if not_overriding_overrides:
error_message = (
"Overrides present with no matching baseline template:\n - "
)
error_message += "\n - ".join(not_overriding_overrides)
logger.warning(error_message)
overridden_templates = [
f for f in template_files_rel_paths if f in override_files_rel_paths
]
no_effect_overrides = [
f
for f in overridden_templates
if is_files_matching(f, baseline_templates_dir, override_templates_dir)
]
if no_effect_overrides:
error_message = "Overrides present which match baseline template:\n - "
error_message += "\n - ".join(no_effect_overrides)
logger.warning(error_message)
effective_overrides = [
f for f in overridden_templates if f not in no_effect_overrides
]
if effective_overrides:
logger.info("The following files differ from baseline templates:")
for template_rel_path in effective_overrides:
seed_template = baseline_templates_dir.joinpath(template_rel_path)
override_template = override_templates_dir.joinpath(
template_rel_path
)
template_text = open(seed_template).readlines()
override_text = open(override_template).readlines()
for line in difflib.unified_diff(
template_text,
override_text,
fromfile=f"template - {template_rel_path}",
tofile=f"override - {template_rel_path}",
lineterm="",
):
# remove superfluous \n characters added by unified_diff
print(line.rstrip())