def __generate_from_template( self, template_file: Path, target_file: Path, variables: dict ): """ Generate configuration file from the specified template file using the supplied variables. Args: template_file (Path): Template used to generate the file. target_file (Path): Location to write the generated file to. variables (dict): Variables used to populate the template. """ template = Template( template_file.read_text(), undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True, ) try: output_text = template.render(variables) target_file.write_text(output_text) except Exception as e: error_and_exit( f"Could not generate file from template. The configuration file is likely missing a setting: {e}" )
def override(ctx, template, force): cli_context: CliContext = ctx.obj cli_context.get_configuration_dir_state().verify_command_allowed( AppcliCommand.CONFIGURE_TEMPLATE_OVERRIDE, force) baseline_templates_dir = self.cli_configuration.baseline_templates_dir template_file_path = Path( os.path.join(baseline_templates_dir, template)) if not template_file_path.exists(): error_and_exit(f"Could not find template [{template}]") override_file_path = ( cli_context.get_baseline_template_overrides_dir().joinpath( template)) if override_file_path.exists(): if not force: error_and_exit( f"Override template already exists at [{override_file_path}]. Use --force to overwrite." ) logger.info( "Force flag provided. Overwriting existing override file.") # Makes the override and sub folders if they do not exist os.makedirs(override_file_path.parent, exist_ok=True) shutil.copy2(template_file_path, override_file_path) logger.info( f"Copied template [{template}] to [{override_file_path}]")
def get(ctx, template): baseline_templates_dir = self.cli_configuration.baseline_templates_dir template_file_path = Path(os.path.join(baseline_templates_dir, template)) if not template_file_path.exists(): error_and_exit(f"Could not find template [{template}]") print(template_file_path.read_text())
def __seed_configuration_dir(self): """Seed the raw configuration into the configuration directory""" print_header("Seeding configuration directory ...") logger.debug("Copying app configuration file ...") seed_app_configuration_file = self.cli_configuration.seed_app_configuration_file if not seed_app_configuration_file.is_file(): error_and_exit( f"Seed file [{seed_app_configuration_file}] is not valid. Release is corrupt." ) # Create the configuration directory and copy in the app config file target_app_configuration_file = self.cli_context.get_app_configuration_file() logger.debug( "Copying app configuration file to [%s] ...", target_app_configuration_file ) os.makedirs(target_app_configuration_file.parent, exist_ok=True) shutil.copy2(seed_app_configuration_file, target_app_configuration_file) stack_configuration_file = self.cli_configuration.stack_configuration_file target_stack_configuration_file = ( self.cli_context.get_stack_configuration_file() ) # Copy in the stack configuration file if stack_configuration_file.is_file(): shutil.copy2(stack_configuration_file, target_stack_configuration_file) # Create the configurable templates directory logger.debug("Copying configurable templates ...") configurable_templates_dir = self.cli_context.get_configurable_templates_dir() configurable_templates_dir.mkdir(parents=True, exist_ok=True) seed_configurable_templates_dir = ( self.cli_configuration.configurable_templates_dir ) if seed_configurable_templates_dir is None: logger.debug("No configurable templates directory defined") return if not seed_configurable_templates_dir.is_dir(): error_and_exit( f"Seed templates directory [{seed_configurable_templates_dir}] is not valid. Release is corrupt." ) # Copy each seed file to the configurable templates directory for source_file in seed_configurable_templates_dir.glob("**/*"): logger.debug(source_file) relative_file = source_file.relative_to(seed_configurable_templates_dir) target_file = configurable_templates_dir.joinpath(relative_file) if source_file.is_dir(): logger.debug("Creating directory [%s] ...", target_file) target_file.mkdir(parents=True, exist_ok=True) else: logger.debug("Copying seed file to [%s] ...", target_file) shutil.copy2(source_file, target_file)
def initialise_configuration(self): """Initialises the configuration repository""" if not self.config_repo.is_repo_on_master_branch(): error_and_exit( "Cannot initialise configuration, repo is not on master branch." ) # Populate with new configuration self.__create_new_configuration_branch_and_files()
def checkout_new_branch_from_master(self, branch_name: str): """Checkout a new branch from the HEAD of master Args: branch_name (str): name of the new branch """ if self.does_branch_exist(branch_name): error_and_exit(f"Cannot create new branch {branch_name}. Already exists.") self.checkout_existing_branch("master") self.repo.git.checkout("HEAD", b=branch_name)
def get(ctx, template): cli_context: CliContext = ctx.obj cli_context.get_configuration_dir_state().verify_command_allowed( AppcliCommand.CONFIGURE_TEMPLATE_GET) baseline_templates_dir = self.cli_configuration.baseline_templates_dir template_file_path = Path( os.path.join(baseline_templates_dir, template)) if not template_file_path.exists(): error_and_exit(f"Could not find template [{template}]") print(template_file_path.read_text())
def _validate_service_names( self, ctx: click.Context, param: click.Option, values: click.Tuple ): """Validates service names. Exits with error if any invalid service names are passed in. Args: ctx (click.Context): Current CLI context. param (click.Option): The option parameter to validate. values (click.Tuple): The values passed to the option, could be multiple. """ if not self.orchestrator.verify_service_names(ctx.obj, values): error_and_exit("One or more service names were not found.") return values
def __backup_and_create_new_directory( self, source_dir: Path, additional_filename_descriptor: str = "backup" ) -> Path: """Backs up a directory to a tar gzipped file with the current datetimestamp, deletes the existing directory, and creates a new empty directory in its place Args: source_dir (Path): Path to the directory to backup and delete additional_filename_descriptor (str, optional): an additional identifier to put into the new tgz filename. If not supplied, defaults to 'backup'. """ if os.path.exists(source_dir) and os.listdir(source_dir): # The datetime is accurate to seconds (microseconds was overkill), and we remove # colon (:) because `tar tvf` doesn't like filenames with colons current_datetime = ( datetime.now().replace(microsecond=0).isoformat().replace(":", "") ) # We have to do a replacement in case it has a slash in it, which causes the # creation of the tar file to fail clean_additional_filename_descriptor = ( additional_filename_descriptor.replace("/", "-") ) basename = os.path.basename(source_dir) output_filename = os.path.join( os.path.dirname(source_dir), f"{basename}_{clean_additional_filename_descriptor}_{current_datetime}.tgz", ) # Create the backup logger.debug(f"Backing up directory [{source_dir}] to [{output_filename}]") with tarfile.open(output_filename, "w:gz") as tar: tar.add(source_dir, arcname=os.path.basename(source_dir)) # Ensure the backup has been successfully created before deleting the existing generated configuration directory if not os.path.exists(output_filename): error_and_exit( f"Current generated configuration directory backup failed. Could not write out file [{output_filename}]." ) # Remove the existing directory shutil.rmtree(source_dir, ignore_errors=True) logger.debug( f"Deleted previous generated configuration directory [{source_dir}]" ) source_dir.mkdir(parents=True, exist_ok=True) logger.debug(f"Created clean directory [{source_dir}]") return source_dir
def _action_orchestrator( self, ctx: Context, action: ServiceAction, service_names: tuple[str, ...] = None, ): """Applies an action to service(s). Args: ctx (Context): Click Context for current CLI. action (ServiceAction): action to apply to service(s), ie start, stop ... service_names (tuple[str, ...], optional): The name(s) of the service(s) to effect. If not provided the action applies to all services. """ hooks = self.cli_configuration.hooks if action == ServiceAction.START: action_run_function = self.orchestrator.start pre_hook = hooks.pre_start post_hook = hooks.post_start elif action == ServiceAction.SHUTDOWN: action_run_function = self.orchestrator.shutdown pre_hook = hooks.pre_shutdown post_hook = hooks.post_shutdown else: error_and_exit(f"Unhandled action called: [{action.name}]") pre_run_log_message = ( f"{action.name} " + ( ", ".join(service_names) if service_names is not None and len(service_names) > 0 else self.cli_configuration.app_name ) + " ..." ) post_run_log_message = f"{action.name} command finished with code [%i]" logger.debug(f"Running pre-{action.name} hook") pre_hook(ctx) logger.info(pre_run_log_message) result = action_run_function(ctx.obj, service_names) logger.debug(f"Running post-{action.name} hook") post_hook(ctx, result) logger.info(post_run_log_message, result.returncode) sys.exit(result.returncode)
def apply_configuration_changes(self, message: str): """Applies the current configuration settings to templates to generate application files. Args: message (str): The message associated with the changes this applies. """ if self.config_repo.is_repo_on_master_branch(): error_and_exit("Cannot apply configuration, repo is on master branch.") # Commit changes to the configuration repository self.config_repo.commit_changes(message) # Re-generated generated configuration self.__regenerate_generated_configuration()
def set(ctx: Context, type: str, encrypted: bool, setting: str, value: str = None): """Set a configuration value, with specified type, and optional encryption. If the 'value' isn't passed in, then the user will be prompted. This is useful in the case where the value is sensitive and shouldn't be captured in terminal history. Note - appcli does not currently support encrypting non-string-typed values. Args: ctx (Context): Click Context for current CLI. type (str): Transform the input value as type encrypted (Bool, flag): flag to indicate if value should be encrypted setting (str): setting to set value (str, optional): value to assign to setting """ cli_context: CliContext = ctx.obj cli_context.get_configuration_dir_state().verify_command_allowed( AppcliCommand.CONFIGURE_SET) # Check if value was not provided if value is None: value = click.prompt("Please enter a value", type=str) # Transform input value as type transformed_value = StringTransformer.transform(value, type) # We don't support encrypting non-string-typed values yet, so error and exit. if encrypted and not isinstance(transformed_value, str): error_and_exit( "Cannot encrypt a non-string-typed value. Exiting without setting value." ) # Set settings value final_value = (encrypt_text(cli_context, transformed_value) if encrypted else transformed_value) configuration = ConfigurationManager(cli_context, self.cli_configuration) configuration.set_variable(setting, final_value) logger.debug( f"Successfully set variable [{setting}] to [{'### Encrypted Value ###' if encrypted else value}]." )
def install(ctx, install_dir: Path): cli_context: CliContext = ctx.obj cli_context.get_configuration_dir_state().verify_command_allowed( AppcliCommand.INSTALL) logger.info("Generating installer script ...") # Get the template from the appcli package launcher_template = pkg_resources.read_text( templates, INSTALLER_TEMPLATE_FILENAME) logger.debug(f"Read template file [{INSTALLER_TEMPLATE_FILENAME}]") environment: str = cli_context.environment target_install_dir: Path = install_dir / environment if cli_context.configuration_dir is None: cli_context = cli_context._replace( configuration_dir=target_install_dir / "conf") if cli_context.data_dir is None: cli_context = cli_context._replace( data_dir=target_install_dir / "data") if cli_context.backup_dir is None: cli_context = cli_context._replace( backup_dir=target_install_dir / "backup") render_variables = { "cli_context": cli_context, "configuration": self.configuration, "install_dir": f"{target_install_dir}", } logger.debug( f"Rendering template with render variables: [{render_variables}]" ) template = Template( launcher_template, undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True, ) try: output_text = template.render(render_variables) print(output_text) except Exception as e: error_and_exit( f"Could not generate file from template. The configuration file is likely missing a setting: {e}" )
def restore(self, ctx, backup_filename: Path): """Restore application data and configuration from the provided local backup `.tgz` file. This will create a backup of the existing data and config, remove the contents `conf`, `data` and `conf/.generated` and then extract the backup to the appropriate locations. `conf`, `data` and `conf/.generated` are mapped into appcli which means we keep the folder but replace their contents on restore. Args: backup_filename (string): The name of the file to use in restoring data. The path of the file will be pulled from `CliContext.obj.backup_dir`. """ cli_context: CliContext = ctx.obj logger.info( f"Initiating system restore with backup [{backup_filename}]") # Check that the backup file exists. backup_dir: Path = cli_context.backup_dir backup_name: Path = Path(os.path.join(backup_dir, backup_filename)) if not backup_name.is_file(): error_and_exit(f"Backup file [{backup_name}] not found.") # Perform a backup of the existing application config and data. logger.debug("Backup existing application data and configuration") restore_backup_name = self.backup( ctx, allow_rolling_deletion=False ) # False ensures we don't accidentally delete our backup logger.debug( f"Backup(s) complete. Generated backups: [{restore_backup_name}]") # Extract conf and data directories from the tar. # This will overwrite the contents of each directory, anything not in the backup (such as files matching the exclude glob patterns) will be left alone. try: with tarfile.open(backup_name) as tar: conf_dir: Path = cli_context.configuration_dir tar.extractall(conf_dir, members=self.__members( tar, os.path.basename(conf_dir))) data_dir: Path = cli_context.data_dir tar.extractall(data_dir, members=self.__members( tar, os.path.basename(data_dir))) except Exception as e: logger.error(f"Failed to extract backup. Reason: {e}") logger.info("Restore complete.")
def launcher(ctx): cli_context: CliContext = ctx.obj cli_context.get_configuration_dir_state().verify_command_allowed( AppcliCommand.LAUNCHER) logger.info("Generating launcher script ...") # Get the template from the appcli package launcher_template = pkg_resources.read_text( templates, LAUNCHER_TEMPLATE_FILENAME) logger.debug(f"Read template file [{LAUNCHER_TEMPLATE_FILENAME}]") render_variables = { "app_version": os.environ.get("APP_VERSION", "latest"), "app_name": configuration.app_name.upper(), "cli_context": ctx.obj, "configuration": self.configuration, "current_datetime": f"{datetime.datetime.utcnow().isoformat()}+00:00", # Since we're using utcnow(), we specify the offset manually } logger.debug( f"Rendering template with render variables: [{render_variables}]" ) template = Template( launcher_template, undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True, ) try: output_text = template.render(render_variables) print(output_text) except Exception as e: error_and_exit( f"Could not generate file from template. The configuration file is likely missing a setting: {e}" )
def check_environment(): """Confirm that mandatory environment variables and additional data directories are defined.""" ENV_VAR_CONFIG_DIR = f"{APP_NAME}_CONFIG_DIR" ENV_VAR_GENERATED_CONFIG_DIR = f"{APP_NAME}_GENERATED_CONFIG_DIR" ENV_VAR_DATA_DIR = f"{APP_NAME}_DATA_DIR" ENV_VAR_ENVIRONMENT = f"{APP_NAME}_ENVIRONMENT" launcher_set_mandatory_env_vars = [ ENV_VAR_CONFIG_DIR, ENV_VAR_GENERATED_CONFIG_DIR, ENV_VAR_DATA_DIR, ENV_VAR_ENVIRONMENT, ] launcher_env_vars_set = check_environment_variable_defined( launcher_set_mandatory_env_vars, "Mandatory environment variable [%s] not defined. This should be set within the script generated with the 'launcher' command.", "Cannot run without all mandatory environment variables defined", ) additional_env_vars_set = check_environment_variable_defined( configuration.mandatory_additional_env_variables, 'Mandatory additional environment variable [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-env-var "%s"="<value>"', "Cannot run without all mandatory additional environment variables defined", ) additional_data_dir_env_vars_set = check_environment_variable_defined( configuration.mandatory_additional_data_dirs, 'Mandatory additional data directory [%s] not defined. When running the \'launcher\' command, define with:\n\t--additional-data-dir "%s"="</path/to/dir>"', "Cannot run without all mandatory additional data directories defined", ) if not ( launcher_env_vars_set and additional_env_vars_set and additional_data_dir_env_vars_set ): error_and_exit( "Some mandatory environment variables weren't set. See error messages above." ) logger.info("All required environment variables are set.")
def migrate_configuration(self): """Migrates the configuration version to the current application version""" if self.config_repo.is_repo_on_master_branch(): error_and_exit("Cannot migrate, repo is on master branch.") config_version: str = self.config_repo.get_repository_version() app_version: str = self.cli_context.app_version # If the configuration version matches the application version, no migration is required. if config_version == app_version: logger.info( f"Migration not required. Config version [{config_version}] matches application version [{app_version}]" ) return logger.info( f"Migrating configuration version [{config_version}] to match application version [{app_version}]" ) app_version_branch: str = self.config_repo.generate_branch_name(app_version) if self.config_repo.does_branch_exist(app_version_branch): # If the branch already exists, then this version has previously been installed. logger.warning( f"Version [{app_version}] of this application was previously installed. Rolling back to previous configuration. Manual remediation may be required." ) # Switch to that branch, no further migration steps will be taken. This is effectively a roll-back. self.config_repo.checkout_existing_branch(app_version_branch) return # Get the stack-settings file contents if it exists stack_config_file = self.cli_context.get_stack_configuration_file() stack_settings_exists_pre_migration = stack_config_file.is_file() current_stack_settings_variables = ( stack_config_file.read_text() if stack_settings_exists_pre_migration else None ) # Migrate the current configuration variables current_variables = self.__get_variables_manager().get_all_variables() # Compare migrated config to the 'clean config' of the new version, and make sure all variables have been set and are the same type. clean_new_version_variables = VariablesManager( self.cli_configuration.seed_app_configuration_file ).get_all_variables() migrated_variables = self.cli_configuration.hooks.migrate_variables( self.cli_context, current_variables, config_version, clean_new_version_variables, ) if not self.cli_configuration.hooks.is_valid_variables( self.cli_context, migrated_variables, clean_new_version_variables ): error_and_exit( "Migrated variables did not pass application-specific variables validation function." ) baseline_template_overrides_dir = ( self.cli_context.get_baseline_template_overrides_dir() ) override_backup_dir: Path = self.__backup_directory( baseline_template_overrides_dir ) configurable_templates_dir = self.cli_context.get_configurable_templates_dir() configurable_templates_backup_dir = self.__backup_directory( configurable_templates_dir ) # Backup and remove the existing generated config dir since it's now out of date self.__backup_and_create_new_generated_config_dir(config_version) # Initialise the new configuration branch and directory with all new files self.__create_new_configuration_branch_and_files() self.__overwrite_directory(override_backup_dir, baseline_template_overrides_dir) if self.__directory_is_not_empty(baseline_template_overrides_dir): logger.warning( f"Overrides directory [{baseline_template_overrides_dir}] is non-empty, please check for compatibility of overridden files" ) self.__overwrite_directory( configurable_templates_backup_dir, configurable_templates_dir ) if self.__directory_is_not_empty(configurable_templates_dir): logger.warning( f"Configurable templates directory [{configurable_templates_dir}] is non-empty, please check for compatibility" ) # Write out 'migrated' variables file self.__get_variables_manager().set_all_variables(migrated_variables) # If stack settings existed pre-migration, then replace the default with the existing settings if stack_settings_exists_pre_migration: logger.warning( "Stack settings file was copied directly from previous version, please check for compatibility" ) stack_config_file.write_text(current_stack_settings_variables) # Commit the new variables file self.config_repo.commit_changes( f"Migrated variables file from version [{config_version}] to version [{app_version}]" )
def check_docker_socket(): """Check that the docker socket exists, and exit if it does not""" if not os.path.exists("/var/run/docker.sock"): error_msg = """Docker socket not present. Please launch with a mounted /var/run/docker.sock""" error_and_exit(error_msg)
def cli( ctx, debug, configuration_dir, data_dir, environment, docker_credentials_file, additional_data_dir, additional_env_var, backup_dir, ): if debug: logger.info("Enabling debug logging") enable_debug_logging() ctx.obj = CliContext( configuration_dir=configuration_dir, data_dir=data_dir, additional_data_dirs=additional_data_dir, additional_env_variables=additional_env_var, environment=environment, docker_credentials_file=docker_credentials_file, subcommand_args=ctx.obj, debug=debug, app_name=APP_NAME, app_version=APP_VERSION, commands=default_commands, backup_dir=backup_dir, ) if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) sys.exit(1) # attempt to set desired environment initialised_environment = {} for k, v in desired_environment.items(): if v is None: logger.warning("Environment variable [%s] has not been set", k) else: logger.debug("Exporting environment variable [%s]", k) os.environ[k] = v initialised_environment[k] = v if len(initialised_environment) != len(desired_environment): error_and_exit( "Could not set desired environment. Please ensure specified environment variables are set." ) # For the `installer`/`launcher` commands, no further output/checks required. if ctx.invoked_subcommand in ("launcher", "install"): # Don't execute this function any further, continue to run subcommand with the current CLI context return check_docker_socket() check_environment() # Table of configuration variables to print table = [ ["Configuration directory", f"{ctx.obj.configuration_dir}"], [ "Generated Configuration directory", f"{ctx.obj.get_generated_configuration_dir()}", ], ["Data directory", f"{ctx.obj.data_dir}"], ["Backup directory", f"{ctx.obj.backup_dir}"], ["Environment", f"{ctx.obj.environment}"], ] # Print out the configuration values as an aligned table logger.info( "%s (version: %s) CLI running with:\n\n%s\n", APP_NAME, APP_VERSION, tabulate(table, colalign=("right", )), ) if additional_data_dir: logger.info( "Additional data directories:\n\n%s\n", tabulate( additional_data_dir, headers=["Environment Variable", "Path"], colalign=("right", ), ), ) if additional_env_var: logger.info( "Additional environment variables:\n\n%s\n", tabulate( additional_env_var, headers=["Environment Variable", "Value"], colalign=("right", ), ), )