def create():
""" Create the directory structure required by the project configuration and set up each account accordingly. """
config = load_project_config()
if not config:
logger.error(
"No configuration file found for the project."
" Make sure the project has already been initialized ([bold]leverage project init[/bold])."
)
return
if (PROJECT_ROOT / "config").exists():
logger.error("Project has already been created.")
return
# Make project structure
_copy_project_template(config=config)
# Render project
_render_project_template(config=config)
# Format the code correctly
logger.info("Reformatting terraform configuration to the standard style.")
# NOTE: This is done just for the sake of making sure the docker image is already available,
# otherwise two animations try to stack on each other and rich does not support that.
# TODO: Modularize docker handling as to avoid this.
tfrun(entrypoint="/bin/sh -c",
command="'echo \"pull image\"'",
enable_mfa=False,
interactive=False)
with console.status("Formatting..."):
tfrun(command="fmt -recursive", enable_mfa=False, interactive=False)
logger.info("Finished setting up project.")
def configure_profile(profile, values):
""" Set profile in `config` file for AWS cli.
Args:
profile (str): Profile name.
values (dict): Mapping of values to be set in the profile.
"""
logger.info(f"\tConfiguring profile [bold]{profile}[/bold]")
for key, value in values.items():
awscli(f"configure set {key} {value} --profile {profile}")
def configure_accounts_profiles(profile, region, organization_accounts,
project_accounts):
""" Set up the required profiles for all accounts to be used with AWS cli. Backup previous profiles.
Args:
profile(str): Name of the profile to configure.
region (str): Region.
organization_accounts (dict): Name and id of all accounts in the organization.
project_accounts (dict): Name and email of all accounts in project configuration file.
"""
short_name, type = profile.split("-")
mfa_serial = ""
if PROFILES[type]["mfa"]:
logger.info("Fetching MFA device serial.")
mfa_serial = _get_mfa_serial(profile)
if not mfa_serial:
logger.error(
"No MFA device found for user. Please set up a device before configuring the accounts profiles."
)
raise Exit(1)
account_profiles = {}
for account in project_accounts:
account_name = account["name"]
# DevOps roles do not have permission over management account
if "security" in profile and account_name == "management":
continue
# TODO: Add remaining profiles for remaining accounts declared in code if enough information is available
account_id = organization_accounts.get(account_name, account.get("id"))
if account_id is None:
continue
# A profile identifier looks like `le-security-oaar`
account_profiles[
f"{short_name}-{account_name}-{PROFILES[type]['profile_role']}"] = {
"output": "json",
"region": region,
"role_arn":
f"arn:aws:iam::{account_id}:role/{PROFILES[type]['role']}",
"source_profile": profile,
"mfa_serial": mfa_serial
}
logger.info("Backing up account profiles file.")
_backup_file("config")
for profile_identifier, profile_values in account_profiles.items():
configure_profile(profile_identifier, profile_values)
def _render_project_template(config, source=TEMPLATE_DIR):
# Render base and non account related templates
template_files = list(source.glob(TEMPLATE_PATTERN))
config_templates = list(
(source / CONFIG_DIRECTORY).rglob(TEMPLATE_PATTERN))
template_files.extend(config_templates)
logger.info("Setting up common base files.")
_render_templates(template_files=template_files,
config=config,
source=source)
# Render each account's templates
for account in config["organization"]["accounts"]:
_render_account_templates(account=account,
config=config,
source=source)
logger.info("Project configuration finished.")
def _render_account_templates(account, config, source=TEMPLATE_DIR):
account_name = account["name"]
logger.info(f"Account: Setting up [bold]{account_name}[/bold].")
account_dir = source / account_name
layers = [CONFIG_DIRECTORY]
for account_name, account_layers in PROJECT_STRUCTURE[account_name].items(
):
layers = layers + [
f"{account_name}/{layer}" for layer in account_layers
]
for layer in layers:
logger.info(
f"\tLayer: Setting up [bold]{layer.split('/')[-1]}[/bold].")
layer_dir = account_dir / layer
layer_templates = layer_dir.glob(TEMPLATE_PATTERN)
_render_templates(template_files=layer_templates,
config=config,
source=source)
def ensure_image(docker_client, image, tag):
""" Check if the required image exists, if not, pull it from the registry.
Args:
docker_client (docker.DockerClient): Point of communication with Docker.
image (str): Name of the required image.
tag (str): Tag of the required image.
"""
found_image = docker_client.api.images(f"{image}:{tag}")
if found_image:
return
logger.info("Required docker image not found.")
stream = docker_client.api.pull(repository=image,
tag=tag,
stream=True,
decode=True)
logger.info(next(stream)["status"])
imageinfo = []
with console.status("Pulling image..."):
for status in stream:
status = status["status"]
if status.startswith("Digest") or status.startswith("Status"):
imageinfo.append(status)
for info in imageinfo:
logger.info(info)
def _load_configs_for_credentials():
""" Load all required values to configure credentials.
Raises:
Exit: If no project has been already initialized in the system.
Returns:
dict: Values needed to configure a credential and update the files accordingly.
"""
logger.info("Loading configuration file.")
project_config = load_project_config()
logger.info("Loading project environment configuration file.")
env_config = load_env_config()
terraform_config = {}
if PROJECT_COMMON_TFVARS.exists():
logger.info("Loading Terraform common configuration.")
terraform_config = hcl2.loads(PROJECT_COMMON_TFVARS.read_text())
config_values = {}
config_values["short_name"] = (project_config.get("short_name")
or env_config.get("PROJECT")
or terraform_config.get("project")
or _ask_for_short_name())
config_values["project_name"] = (project_config.get("project_name")
or terraform_config.get("project_long"))
config_values["primary_region"] = (project_config.get("primary_region") or
terraform_config.get("region_primary")
or _ask_for_region())
config_values["secondary_region"] = terraform_config.get(
"region_secondary")
config_values["organization"] = {"accounts": []}
# Accounts defined in Terraform code take priority
terraform_accounts = terraform_config.get("accounts", {})
if terraform_accounts:
config_values["organization"]["accounts"].extend([{
"name":
account_name,
"email":
account_info.get("email"),
"id":
account_info.get("id")
} for account_name, account_info in terraform_accounts.items()])
# Add accounts not found in terraform code
project_accounts = [
account for account in project_config.get("organization", {}).get(
"accounts", []) if account.get("name") not in terraform_accounts
]
if project_accounts:
config_values["organization"]["accounts"].extend(project_accounts)
config_values["mfa_enabled"] = env_config.get("MFA_ENABLED", "false")
config_values["terraform_image_tag"] = env_config.get(
"TERRAFORM_IMAGE_TAG", "1.0.9")
return config_values
def _copy_project_template(config):
""" Copy all files and directories from the Leverage project template to the project directory.
It excludes al jinja templates as those will be rendered directly to their final location.
Args:
config (dict): Project configuration.
"""
logger.info("Creating project directory structure.")
# Copy .gitignore file
copy2(src=TEMPLATE_DIR / ".gitignore", dst=PROJECT_ROOT / ".gitignore")
# Root config directory
copytree(src=TEMPLATE_DIR / CONFIG_DIRECTORY,
dst=PROJECT_ROOT / CONFIG_DIRECTORY,
ignore=IGNORE_PATTERNS)
# Accounts
for account in PROJECT_STRUCTURE:
_copy_account(account=account, primary_region=config["primary_region"])
logger.info("Finished creating directory structure.")
def configure_credentials(profile, file=None, make_backup=False):
""" Set credentials in `credentials` file for AWS cli. Make backup if required.
Args:
profile (str): Name of the profile to configure.
file (Path, optional): Credentials file. Defaults to None.
make_backup (bool, optional): Whether to make a backup of the credentials file. Defaults to False.
"""
file = file or _ask_for_credentials_location()
if file is not None and file != "manual":
key_id, secret_key = _extract_credentials(file)
else:
key_id, secret_key = _ask_for_credentials()
if make_backup:
logger.info("Backing up credentials file.")
_backup_file("credentials")
values = {"aws_access_key_id": key_id, "aws_secret_access_key": secret_key}
for key, value in values.items():
awscli(f"configure set {key} {value} --profile {profile}")
def configure(type, credentials_file, overwrite_existing_credentials,
skip_access_keys_setup, skip_assumable_roles_setup):
""" Configure credentials for the project.
It can handle the credentials required for the initial deployment of the project (BOOTSTRAP),
a management user (MANAGEMENT) or a devops/secops user (SECURITY).
"""
if skip_access_keys_setup and skip_assumable_roles_setup:
logger.info("Nothing to do. Exiting.")
return
config_values = _load_configs_for_credentials()
# Environment configuration variables are needed for the Leverage docker container
if not PROJECT_ENV_CONFIG.exists():
PROJECT_ENV_CONFIG.write_text(f"PROJECT={config_values['short_name']}")
type = type.lower()
short_name = config_values.get("short_name")
profile = f"{short_name}-{type}"
already_configured = _profile_is_configured(profile=profile)
if already_configured and not (skip_access_keys_setup
or overwrite_existing_credentials):
title_extra = "" if skip_assumable_roles_setup else " Continue on with assumable roles setup."
overwrite_existing_credentials = _ask_for_credentials_overwrite(
profile=profile,
skip_option_title=f"Skip credentials configuration.{title_extra}",
overwrite_option_title=
"Overwrite current credentials. Backups will be made.")
do_configure_credentials = False if skip_access_keys_setup else overwrite_existing_credentials or not already_configured
if do_configure_credentials:
logger.info(f"Configuring [bold]{type}[/bold] credentials.")
configure_credentials(profile,
credentials_file,
make_backup=already_configured)
logger.info(
f"[bold]{type.capitalize()} credentials configured in:[/bold]"
f" {(AWSCLI_CONFIG_DIR / short_name / 'credentials').as_posix()}")
if not _credentials_are_valid(profile):
logger.error(
f"Invalid {profile} credentials. Please check the given keys.")
return
accounts = config_values.get("organization", {}).get("accounts", False)
# First time configuring bootstrap credentials
if type == "bootstrap" and not already_configured:
management_account = next(
(account
for account in accounts if account["name"] == "management"), None)
if management_account:
logger.info("Fetching management account id.")
management_account_id = _get_management_account_id(profile=profile)
management_account["id"] = management_account_id
project_config_file = load_project_config()
if project_config_file and "accounts" in project_config_file.get(
"organization", {}):
project_config_file["organization"]["accounts"] = accounts
logger.info("Updating project configuration file.")
YAML().dump(data=project_config_file, stream=PROJECT_CONFIG)
skip_assumable_roles_setup = True
profile_for_organization = profile
# Security credentials don't have permission to access organization information
if type == "security":
for type_with_permission in ("management", "bootstrap"):
profile_to_check = f"{short_name}-{type_with_permission}"
if _profile_is_configured(profile_to_check):
profile_for_organization = profile_to_check
break
if skip_assumable_roles_setup:
logger.info("Skipping assumable roles configuration.")
else:
logger.info("Attempting to fetch organization accounts.")
organization_accounts = _get_organization_accounts(
profile_for_organization, config_values.get("project_name"))
logger.debug(f"Organization Accounts fetched: {organization_accounts}")
if organization_accounts or accounts:
logger.info("Configuring assumable roles.")
configure_accounts_profiles(profile,
config_values["primary_region"],
organization_accounts, accounts)
logger.info(
f"[bold]Account profiles configured in:[/bold]"
f" {(AWSCLI_CONFIG_DIR / short_name / 'config').as_posix()}")
for account in accounts:
try: # Some account may not already be created
account["id"] = organization_accounts[account["name"]]
except KeyError:
continue
else:
logger.info(
"No organization has been created yet or no accounts were configured.\n"
"Skipping assumable roles configuration.")
_update_account_ids(config=config_values)
def init():
""" Initializes and gets all the required resources to be able to create a new Leverage project. """
# Application's directory
if not LEVERAGE_DIR.exists():
logger.info(
"No [bold]Leverage[/bold] config directory found in user's home. Creating."
)
LEVERAGE_DIR.mkdir()
# Leverage project templates
if not TEMPLATES_REPO_DIR.exists():
TEMPLATES_REPO_DIR.mkdir(parents=True)
if not (TEMPLATES_REPO_DIR / ".git").exists():
logger.info("No project template found. Cloning template.")
git(f"clone {LEVERAGE_TEMPLATE_REPO} {TEMPLATES_REPO_DIR.as_posix()}")
logger.info("Finished cloning template.")
else:
logger.info("Project template found. Updating.")
git(f"-C {TEMPLATES_REPO_DIR.as_posix()} checkout master")
git(f"-C {TEMPLATES_REPO_DIR.as_posix()} pull")
logger.info("Finished updating template.")
# Leverage projects are git repositories too
logger.info("Initializing git repository in project directory.")
git("init")
# Project configuration file
if not PROJECT_CONFIG.exists():
logger.info(
f"No project configuration file found. Dropping configuration template [bold]{PROJECT_CONFIG_FILE}[/bold]."
)
copy2(src=CONFIG_FILE_TEMPLATE, dst=PROJECT_CONFIG_FILE)
else:
logger.warning(
f"Project configuration file [bold]{PROJECT_CONFIG_FILE}[/bold] already exists in directory."
)
logger.info("Project initialization finished.")