def map_iam_identities(
context: Context, cluster_name: str, eks_system_masters_roles_changes: Optional[ListChangeset]
) -> None:
if eks_system_masters_roles_changes and eks_system_masters_roles_changes.added_values:
for role in eks_system_masters_roles_changes.added_values:
if iam.get_role(role) is None:
_logger.debug(f"Skipping nonexisting IAM Role: {role}")
continue
arn = f"arn:aws:iam::{context.account_id}:role/{role}"
for line in sh.run_iterating(f"eksctl get iamidentitymapping --cluster {cluster_name} --arn {arn}"):
if line.startswith("Error: no iamidentitymapping with arn"):
_logger.debug(f"Adding IAM Identity Mapping - Role: {arn}, Username: {role}, Group: system:masters")
sh.run(
f"eksctl create iamidentitymapping --cluster {cluster_name} --arn {arn} "
f"--username {role} --group system:masters"
)
cast(List[str], context.eks_system_masters_roles).append(role)
ContextSerDe.dump_context_to_ssm(context=context)
break
else:
_logger.debug(f"Skip adding existing IAM Identity Mapping - Role: {arn}")
if eks_system_masters_roles_changes and eks_system_masters_roles_changes.removed_values:
for role in eks_system_masters_roles_changes.removed_values:
arn = f"arn:aws:iam::{context.account_id}:role/{role}"
for line in sh.run_iterating(f"eksctl get iamidentitymapping --cluster {cluster_name} --arn {arn}"):
if line.startswith("Error: no iamidentitymapping with arn"):
_logger.debug(f"Skip removing nonexisting IAM Identity Mapping - Role: {arn}")
break
else:
_logger.debug(f"Removing IAM Identity Mapping - Role: {arn}")
sh.run(f"eksctl delete iamidentitymapping --cluster {cluster_name} --arn {arn} --all")
cast(List[str], context.eks_system_masters_roles).remove(role)
ContextSerDe.dump_context_to_ssm(context=context)
def fetch_kubectl_data(context: "Context", k8s_context: str,
include_teams: bool) -> None:
_logger.debug("Fetching Kubectl data...")
if include_teams:
for team in context.teams:
_logger.debug("Fetching team %s URL parameter", team.name)
url = k8s.get_service_hostname(name="jupyterhub-public",
k8s_context=k8s_context,
namespace=team.name)
team.jupyter_url = url
landing_page_url: str = k8s.get_service_hostname(name="landing-page",
k8s_context=k8s_context,
namespace="env")
k8_dashboard_url: str = k8s.get_service_hostname(
name="kubernetes-dashboard",
k8s_context=k8s_context,
namespace="kubernetes-dashboard")
context.landing_page_url = f"https://{landing_page_url}"
if context.cognito_external_provider:
context.cognito_external_provider_redirect = context.landing_page_url
context.k8_dashboard_url = f"https://{k8_dashboard_url}"
_update_elbs(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
_logger.debug("Kubectl data fetched successfully.")
def deploy_env(env_name: str, manifest_dir: str) -> None:
docker.login(context=context)
_logger.debug("DockerHub and ECR Logged in")
cdk_toolkit.deploy(context=context)
_logger.debug("CDK Toolkit Stack deployed")
env.deploy(
context=context,
eks_system_masters_roles_changes=changeset.eks_system_masters_roles_changeset if changeset else None,
)
_logger.debug("Env Stack deployed")
eksctl.deploy_env(
context=context,
changeset=changeset,
)
_logger.debug("EKS Environment Stack deployed")
kubectl.deploy_env(context=context)
_logger.debug("Kubernetes Environment components deployed")
helm.deploy_env(context=context)
_logger.debug("Helm Charts installed")
k8s_context = utils.get_k8s_context(context=context)
kubectl.fetch_kubectl_data(context=context, k8s_context=k8s_context)
ContextSerDe.dump_context_to_ssm(context=context)
_logger.debug("Updating userpool redirect")
_update_userpool_client(context=context)
_update_userpool(context=context)
def deploy_toolkit(
context: "Context",
username: Optional[str],
password: Optional[str],
msg_ctx: MessagesContext,
top_level: str = "orbit",
) -> None:
credential_received: bool = username is not None and password is not None
stack_exist: bool = cfn.does_stack_exist(
stack_name=context.toolkit.stack_name)
credential_exist: bool = dockerhub.does_credential_exist(
context=context) if stack_exist else False
image_manifests = [
cast(ImageManifest, getattr(context.images, i))
for i in context.images.names
]
credential_required: bool = any([
im.get_source(account_id=context.account_id,
region=context.region) == "dockerhub"
for im in image_manifests
])
if stack_exist:
if credential_required and not credential_exist and not credential_received:
username, password = _request_dockerhub_credential(msg_ctx=msg_ctx)
dockerhub.store_credential(context=context,
username=username,
password=password)
credential_exist = True
elif credential_received:
dockerhub.store_credential(
context=context,
username=cast(str, username),
password=cast(str, password),
)
credential_exist = True
else:
context.toolkit.deploy_id = "".join(
random.choice(string.ascii_lowercase) for i in range(6))
if credential_required and not credential_received:
username, password = _request_dockerhub_credential(msg_ctx=msg_ctx)
credential_exist = False
msg_ctx.progress(6)
_logger.debug("context.toolkit.deploy_id: %s", context.toolkit.deploy_id)
template_filename: str = toolkit.synth(context=context,
top_level=top_level)
cfn.deploy_template(stack_name=context.toolkit.stack_name,
filename=template_filename,
env_tag=context.env_tag,
s3_bucket=None)
ContextSerDe.fetch_toolkit_data(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
if credential_exist is False:
dockerhub.store_credential(
context=context,
username=cast(str, username),
password=cast(str, password),
)
def deploy_credentials(filename: str, username: str, password: str, registry: str, debug: bool) -> None:
with MessagesContext("Deploying", debug=debug) as msg_ctx:
msg_ctx.progress(2)
manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
msg_ctx.info(f"Manifest loaded: {filename}")
msg_ctx.progress(3)
context_parameter_name: str = f"/orbit/{manifest.name}/context"
if not ssm.does_parameter_exist(name=context_parameter_name):
msg_ctx.error(f"Orbit Environment {manifest.name} cannot be found in the current account and region.")
return
context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)
msg_ctx.info("Current Context loaded")
msg_ctx.progress(4)
msg_ctx.info("Encrypting credentials with Toolkit KMS Key")
ciphertext = kms.encrypt(
context=context, plaintext=json.dumps({registry: {"username": username, "password": password}})
)
msg_ctx.progress(20)
msg_ctx.info("Starting Remote CodeBuild to deploy credentials")
deploy.deploy_credentials(env_name=context.name, ciphertext=ciphertext)
msg_ctx.info("Registry Credentials deployed")
msg_ctx.progress(98)
if cfn.does_stack_exist(stack_name=context.env_stack_name):
context = ContextSerDe.load_context_from_ssm(env_name=manifest.name, type=Context)
msg_ctx.info(f"Context updated: {filename}")
msg_ctx.progress(100)
def fetch_cluster_data(context: "Context", cluster_name: str) -> None:
_logger.debug("Fetching Cluster data...")
cluster_data = cast(Dict[str, Any], eks.describe_cluster(cluster_name=cluster_name))
context.eks_oidc_provider = cluster_data["cluster"]["identity"]["oidc"]["issuer"].replace("https://", "")
context.cluster_sg_id = cluster_data["cluster"]["resourcesVpcConfig"]["clusterSecurityGroupId"]
ContextSerDe.dump_context_to_ssm(context=context)
_logger.debug("Cluster data fetched successfully.")
def _team(context: "Context", team_context: "TeamContext",
output_path: str) -> None:
input = os.path.join(MODELS_PATH, "teams", "00-team.yaml")
output = os.path.join(output_path, f"{team_context.name}-00-team.yaml")
with open(input, "r") as file:
content: str = file.read()
content = utils.resolve_parameters(
content,
dict(
team=team_context.name,
efsid=context.shared_efs_fs_id,
efsapid=team_context.efs_ap_id,
efsprivateapid=team_context.efs_private_ap_id
if team_context.efs_private_ap_id else "",
account_id=context.account_id,
env_name=context.name,
role_prefix=f"/{context.role_prefix}/"
if context.role_prefix else "/",
team_kms_key_arn=team_context.team_kms_key_arn,
team_security_group_id=team_context.team_security_group_id,
cluster_pod_security_group_id=context.cluster_pod_sg_id,
team_context=ContextSerDe.dump_context_to_str(team_context),
env_context=ContextSerDe.dump_context_to_str(context),
region=context.region,
),
)
_logger.debug("Kubectl Team %s manifest:\n%s", team_context.name, content)
with open(output, "w") as file:
file.write(content)
# team rbac role
input = os.path.join(MODELS_PATH, "teams", "01-team-rbac-role.yaml")
output = os.path.join(output_path,
f"{team_context.name}-01-team-rbac-role.yaml")
with open(input, "r") as file:
content = file.read()
content = utils.resolve_parameters(
content, dict(env_name=context.name, team=team_context.name))
with open(output, "w") as file:
file.write(content)
# bind to admin role
if team_context.k8_admin:
# user service account
input = os.path.join(MODELS_PATH, "teams", "02-admin-binding.yaml")
output = os.path.join(output_path,
f"{team_context.name}-02-admin-binding.yaml")
with open(input, "r") as file:
content = file.read()
content = utils.resolve_parameters(content,
dict(team=team_context.name))
with open(output, "w") as file:
file.write(content)
def deploy_env(
filename: str,
debug: bool,
) -> None:
with MessagesContext("Deploying", debug=debug) as msg_ctx:
msg_ctx.progress(2)
manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
msg_ctx.info(f"Manifest loaded: {filename}")
msg_ctx.progress(5)
manifest_dir: str = os.path.dirname(os.path.abspath(filename))
_logger.debug("manifest directory is set to %s", manifest_dir)
manifest_validations(manifest)
context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)
msg_ctx.info("Current Context loaded")
msg_ctx.progress(10)
_logger.debug("Inspecting possible manifest changes...")
changeset: "Changeset" = extract_changeset(manifest=manifest, context=context, msg_ctx=msg_ctx)
_logger.debug(f"Changeset:\n{dump_changeset_to_str(changeset=changeset)}")
msg_ctx.progress(15)
_deploy_toolkit(
context=context,
)
msg_ctx.info("Toolkit deployed")
msg_ctx.progress(30)
deploy.deploy_env(
env_name=context.name,
manifest_dir=manifest_dir,
)
msg_ctx.info("Orbit Workbench deployed")
msg_ctx.progress(98)
if cfn.does_stack_exist(stack_name=context.env_stack_name):
context = ContextSerDe.load_context_from_manifest(manifest=manifest)
msg_ctx.info(f"Context updated: {filename}")
msg_ctx.progress(99)
if context.cognito_users_url:
msg_ctx.tip(f"Add users: {stylize(context.cognito_users_url, underline=True)}")
else:
RuntimeError("Cognito Users URL not found.")
if context.landing_page_url:
msg_ctx.tip(f"Access Orbit Workbench: {stylize(f'{context.landing_page_url}/orbit/login', underline=True)}")
else:
RuntimeError("Landing Page URL not found.")
msg_ctx.progress(100)
def deploy_teams(
filename: str,
debug: bool,
) -> None:
with MessagesContext("Deploying", debug=debug) as msg_ctx:
msg_ctx.progress(2)
manifest: "Manifest" = ManifestSerDe.load_manifest_from_file(filename=filename, type=Manifest)
msg_ctx.info(f"Manifest loaded: {filename}")
msg_ctx.info(f"Teams: {','.join([t.name for t in manifest.teams])}")
msg_ctx.progress(5)
manifest_dir: str = os.path.dirname(os.path.abspath(filename))
_logger.debug("manifest directory is set to %s", manifest_dir)
context_parameter_name: str = f"/orbit/{manifest.name}/context"
if not ssm.does_parameter_exist(name=context_parameter_name):
msg_ctx.error(f"Orbit Environment {manifest.name} cannot be found in the current account and region.")
return
context: "Context" = ContextSerDe.load_context_from_manifest(manifest=manifest)
msg_ctx.info("Current Context loaded")
msg_ctx.info(f"Teams: {','.join([t.name for t in context.teams])}")
msg_ctx.progress(10)
_logger.debug("Inspecting possible manifest changes...")
changeset: "Changeset" = extract_changeset(manifest=manifest, context=context, msg_ctx=msg_ctx)
_logger.debug(f"Changeset:\n{dump_changeset_to_str(changeset=changeset)}")
msg_ctx.progress(30)
deploy.deploy_teams(
env_name=context.name,
manifest_dir=manifest_dir,
)
msg_ctx.info("Orbit Workbench deployed")
msg_ctx.progress(98)
if cfn.does_stack_exist(stack_name=context.env_stack_name):
context = ContextSerDe.load_context_from_ssm(env_name=manifest.name, type=Context)
msg_ctx.info(f"Context updated: {filename}")
msg_ctx.progress(99)
if context.user_pool_id:
cognito_users_url = orbit_cognito.get_users_url(user_pool_id=context.user_pool_id, region=context.region)
msg_ctx.tip(f"Add users: {stylize(cognito_users_url, underline=True)}")
if context.landing_page_url:
msg_ctx.tip(f"Access Orbit Workbench: {stylize(f'{context.landing_page_url}/orbit/login', underline=True)}")
else:
raise RuntimeError("Landing Page URL not found.")
msg_ctx.progress(100)
def _deploy_toolkit(
context: "Context",
top_level: str = "orbit",
) -> None:
stack_exist: bool = cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
if not stack_exist:
context.toolkit.deploy_id = "".join(random.choice(string.ascii_lowercase) for i in range(6))
_logger.debug("context.toolkit.deploy_id: %s", context.toolkit.deploy_id)
template_filename: str = toolkit.synth(context=context, top_level=top_level)
cfn.deploy_template(
stack_name=context.toolkit.stack_name, filename=template_filename, env_tag=context.env_tag, s3_bucket=None
)
ContextSerDe.fetch_toolkit_data(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
def _deploy_images_batch(context: "Context", images: List[Tuple[str, Optional[str], Optional[str], List[str]]]) -> None:
_logger.debug("images:\n%s", images)
new_images_manifest = {name: getattr(context.images, name) for name in context.images.names}
max_workers = 5
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
futures: List[Future[Any]] = []
name: str = ""
dir: Optional[str] = None
script: Optional[str] = None
build_args: List[str] = []
for name, dir, script, build_args in images:
_logger.debug("name: %s | script: %s", name, script)
path = os.path.join(os.getcwd(), "bundle", name)
_logger.debug("path: %s", path)
image_attr_name = name.replace("-", "_")
image_def: ImageManifest = getattr(context.images, image_attr_name)
tag = image_def.version
if image_def.get_source(account_id=context.account_id, region=context.region) == "code":
dirs: List[Tuple[str, str]] = [(path, cast(str, dir))]
else:
dirs = []
bundle_path = bundle.generate_bundle(command_name=f"deploy_image-{name}", context=context, dirs=dirs)
_logger.debug("bundle_path: %s", bundle_path)
script_str = "NO_SCRIPT" if script is None else script
build_args = [] if build_args is None else build_args
buildspec = codebuild.generate_spec(
context=context,
plugins=False,
cmds_build=[
"orbit remote --command _deploy_image "
f"{context.name} {name} {dir} {script_str} {' '.join(build_args)}"
],
)
new_images_manifest[image_attr_name] = ImageManifest(
repository=f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/{name}",
version=tag,
path=None,
)
futures.append(executor.submit(_deploy_image_remotely, context, name, bundle_path, buildspec))
for f in futures:
f.result()
context.images = ImagesManifest(**new_images_manifest)
ContextSerDe.dump_context_to_ssm(context=context)
def deploy_team(context: "Context", manifest: Manifest,
team_manifest: TeamManifest) -> None:
# Pull team spacific custom cfn plugin, trigger pre_hook
team_context: Optional["TeamContext"] = create_team_context_from_manifest(
manifest=manifest, team_manifest=team_manifest)
_logger.debug(f"team_context={team_context}")
if team_context:
_logger.debug(f"team_context.plugins={team_context.plugins}")
_logger.debug("Calling team pre_hook")
for plugin in team_context.plugins:
hook: plugins.HOOK_TYPE = plugins.PLUGINS_REGISTRIES.get_hook(
context=context,
team_name=team_context.name,
plugin_name=plugin.plugin_id,
hook_name="pre_hook",
)
if hook is not None:
_logger.debug(f"Found pre_hook for plugin_id {plugin}")
hook(plugin.plugin_id, context, team_context,
plugin.parameters)
_logger.debug("End of pre_hook plugin execution")
else:
_logger.debug(
f"Skipping pre_hook for unknown Team: {team_manifest.name}")
args = [context.name, team_manifest.name]
cdk.deploy(
context=context,
stack_name=f"orbit-{manifest.name}-{team_manifest.name}",
app_filename=os.path.join(ORBIT_CLI_ROOT, "remote_files", "cdk",
"team.py"),
args=args,
)
team_context = context.get_team_by_name(name=team_manifest.name)
if team_context:
team_context.fetch_team_data()
else:
team_context = create_team_context_from_manifest(
manifest=manifest, team_manifest=team_manifest)
team_context.fetch_team_data()
context.teams.append(team_context)
_logger.debug(
f"team_context.helm_repository: s3://{context.toolkit.s3_bucket}/helm/repositories/teams/{team_context.name}"
)
team_context.team_helm_repository = f"s3://{context.toolkit.s3_bucket}/helm/repositories/teams/{team_context.name}"
team_context.user_helm_repository = f"s3://{context.toolkit.s3_bucket}/helm/repositories/user/{team_context.name}"
ContextSerDe.dump_context_to_ssm(context=context)
def cdk_prep_team_handler(stack_class: Type["Stack"]) -> None:
_logger.debug("sys.argv: %s", sys.argv)
if len(sys.argv) != 5:
raise ValueError(f"Unexpected number of values in sys.argv ({len(sys.argv)}) - {sys.argv}.")
stack_name: str = sys.argv[1]
# team_name: str = sys.argv[3]
parameters: Dict[str, Any] = _deserialize_parameters(parameters=sys.argv[4])
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=sys.argv[2], type=Context)
# Can not find /orbit/env_name/teams ssm param.
# team_context = context.get_team_by_name(name=team_name)
# if team_context is None:
# raise ValueError(f"Team {team_name} not found in the context.")
outdir = os.path.join(
".orbit.out",
context.name,
"cdk",
stack_name,
)
shutil.rmtree(outdir, ignore_errors=True)
os.makedirs(outdir, exist_ok=True)
# Can't be imported globally because we only have CDK installed on CodeBuild
from aws_cdk.core import App
app = App(outdir=outdir)
stack_class(app, stack_name, context, parameters) # type: ignore
app.synth(force=True)
def deploy_credentials(env_name: str, ciphertext: str) -> None:
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)
_logger.debug("Context loaded.")
@codeseeder.remote_function("orbit", codebuild_role=context.toolkit.admin_role)
def deploy_credentials(env_name: str, ciphertext: str) -> None:
new_credentials = json.loads(kms.decrypt(context=context, ciphertext=ciphertext))
secret_id = f"orbit-{env_name}-docker-credentials"
existing_credentials = secretsmanager.get_secret_value(secret_id=secret_id)
for registry, creds in new_credentials.items():
username = creds.get("username", "")
password = creds.get("password", "")
try:
subprocess.check_call(
f"docker login --username '{username}' --password '{password}' {registry}", shell=True
)
except Exception as e:
_logger.error("Invalid Registry Credentials")
_logger.exception(e)
return
else:
existing_credentials = {**existing_credentials, **new_credentials}
secretsmanager.put_secret_value(secret_id=secret_id, secret=existing_credentials)
_logger.debug("Registry Credentials deployed")
deploy_credentials(env_name=env_name, ciphertext=ciphertext)
def destroy_teams(args: Tuple[str, ...]) -> None:
_logger.debug("args %s", args)
env_name: str = args[0]
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name,
type=Context)
_logger.debug("context.name %s", context.name)
plugins.PLUGINS_REGISTRIES.load_plugins(context=context,
plugin_changesets=[],
teams_changeset=None)
kubectl.write_kubeconfig(context=context)
_logger.debug("Plugins loaded")
for team_context in context.teams:
plugins.PLUGINS_REGISTRIES.destroy_team_plugins(
context=context, team_context=team_context)
_logger.debug("Plugins destroyed")
for team_context in context.teams:
helm.destroy_team(context=context, team_context=team_context)
_logger.debug("Helm Charts uninstalled")
kubectl.destroy_teams(context=context)
_logger.debug("Kubernetes Team components destroyed")
eksctl.destroy_teams(context=context)
_logger.debug("EKS Team Stacks destroyed")
teams.destroy_all(context=context)
_logger.debug("Teams Stacks destroyed")
ssm.cleanup_teams(env_name=context.name)
def destroy_toolkit(
env: str,
debug: bool,
) -> None:
with MessagesContext("Destroying Environment Toolkit",
debug=debug) as msg_ctx:
msg_ctx.progress(10)
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
type=Context)
msg_ctx.info("Context loaded")
msg_ctx.progress(25)
try:
_destroy_toolkit(env_name=context.name)
except botocore.exceptions.ClientError as ex:
error = ex.response["Error"]
if "does not exist" not in error["Message"]:
raise
_logger.debug(f"Skipping toolkit destroy: {error['Message']}")
msg_ctx.info("Toolkit destroyed")
ssm.cleanup_env(env_name=context.name)
msg_ctx.info("Toolkit destroyed")
msg_ctx.progress(100)
def build_image(
env: str,
dir: Optional[str],
name: str,
script: Optional[str],
build_args: Optional[List[str]],
timeout: int = 30,
debug: bool = False,
source_registry: Optional[str] = None,
source_repository: Optional[str] = None,
source_version: Optional[str] = None,
) -> None:
with MessagesContext("Deploying Docker Image", debug=debug) as msg_ctx:
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env, type=Context)
msg_ctx.info("Manifest loaded")
if cfn.does_stack_exist(stack_name=f"orbit-{context.name}") is False:
msg_ctx.error("Please, deploy your environment before deploying any additional docker image")
return
msg_ctx.progress(3)
if dir:
dirs = [(dir, name)]
else:
dirs = []
bundle_path = bundle.generate_bundle(command_name=f"deploy_image-{name}", context=context, dirs=dirs)
msg_ctx.progress(5)
script_str = "NO_SCRIPT" if script is None else script
source_str = "NO_REPO" if source_registry is None else f"{source_registry} {source_repository} {source_version}"
build_args = [] if build_args is None else build_args
buildspec = codebuild.generate_spec(
context=context,
plugins=False,
cmds_build=[
f"orbit remote --command build_image " f"{env} {name} {script_str} {source_str} {' '.join(build_args)}"
],
changeset=None,
)
msg_ctx.progress(6)
remote.run(
command_name=f"deploy_image-{name}",
context=context,
bundle_path=bundle_path,
buildspec=buildspec,
codebuild_log_callback=msg_ctx.progress_bar_callback,
timeout=timeout,
)
msg_ctx.info("Docker Image deploy into ECR")
address = (
f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/{name}"
if name in [n.replace("_", "-") for n in context.images.names]
else f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}/users/{name}"
)
msg_ctx.info(f"ECR Image Address={address}")
msg_ctx.tip(f"ECR Image Address: {stylize(address, underline=True)}")
msg_ctx.progress(100)
def destroy_env(env: str, debug: bool) -> None:
with MessagesContext("Destroying", debug=debug) as msg_ctx:
ssm.cleanup_changeset(env_name=env)
ssm.cleanup_manifest(env_name=env)
if ssm.does_parameter_exist(name=f"/orbit/{env}/context") is False:
msg_ctx.info(f"Environment {env} not found. Destroying only possible remaining resources.")
elb.delete_load_balancers(env_name=env)
destroy_remaining_resources(env_name=env, top_level="orbit")
msg_ctx.progress(100)
return
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env, type=Context)
msg_ctx.info("Context loaded")
msg_ctx.info(f"Teams: {','.join([t.name for t in context.teams])}")
msg_ctx.progress(2)
if any(cfn.does_stack_exist(stack_name=t.stack_name) for t in context.teams):
msg_ctx.error("Found Teams dependent on the Envrionment.")
return
if (
cfn.does_stack_exist(stack_name=context.env_stack_name)
or cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
or cfn.does_stack_exist(stack_name=context.cdk_toolkit.stack_name)
):
bundle_path = bundle.generate_bundle(command_name="destroy", context=context)
msg_ctx.progress(5)
buildspec = codebuild.generate_spec(
context=context,
plugins=True,
cmds_build=[f"orbit remote --command destroy_env {env}"],
changeset=None,
)
remote.run(
command_name="destroy",
context=context,
bundle_path=bundle_path,
buildspec=buildspec,
codebuild_log_callback=msg_ctx.progress_bar_callback,
timeout=45,
)
msg_ctx.info("Env destroyed")
msg_ctx.progress(95)
try:
destroy_toolkit(env_name=context.name)
except botocore.exceptions.ClientError as ex:
error = ex.response["Error"]
if "does not exist" not in error["Message"]:
raise
_logger.debug(f"Skipping toolkit destroy: {error['Message']}")
msg_ctx.info("Toolkit destroyed")
ssm.cleanup_env(env_name=context.name)
msg_ctx.progress(100)
def list_env(env: str, variable: str) -> None:
ssm = utils.boto3_client("ssm")
res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True)
env_info: Dict[str, str] = {}
while True:
params = res["Parameters"]
for p in params:
if not p["Name"].endswith("context") or "teams" in p["Name"]:
continue
if len(env) > 0 and p["Name"].startswith(f"//orbit/{env}"):
continue
env_name = p["Name"].split("/")[2]
context: "Context" = ContextSerDe.load_context_from_ssm(
env_name=env_name, type=Context)
_logger.debug(f"found env: {env_name}")
if context.k8_dashboard_url:
k8_dashboard_url = context.k8_dashboard_url
else:
k8_dashboard_url = ""
if len(context.teams) > 0:
teams_list: str = ",".join([x.name for x in context.teams])
else:
teams_list = ""
if variable == "landing-page":
print(context.landing_page_url)
elif variable == "toolkitbucket":
print(context.toolkit.s3_bucket)
elif variable == "teams":
print(f"[{teams_list}]")
elif variable == "all":
env_info[env_name] = (
f"LandingPage={context.landing_page_url}, "
f"Teams=[{teams_list}], "
f"ToolkitBucket={context.toolkit.s3_bucket}"
f"K8Dashboard={k8_dashboard_url}")
else:
raise Exception(f"Unknown --variable option {variable}")
if "NextToken" in res:
res = ssm.get_parameters_by_path(Path="/orbit",
Recursive=True,
NextToken=res["NextToken"])
else:
break
if variable == "all":
if len(env_info) == 0:
click.echo("There are no Orbit environments available")
return
else:
print_list(
tittle="Available Orbit environments:",
items=[
f"Name={k}{stylize(',')}{v}" for k, v in env_info.items()
],
)
def fetch_kubectl_data(context: "Context", k8s_context: str) -> None:
_logger.debug("Fetching Kubectl data...")
ingress_url: str = k8s.get_ingress_dns(name="istio-ingress",
k8s_context=k8s_context,
namespace="istio-system")
if context.networking.frontend.custom_domain_name:
context.landing_page_url = f"https://{context.networking.frontend.custom_domain_name}"
else:
context.landing_page_url = f"https://{ingress_url}"
if context.cognito_external_provider:
context.cognito_external_provider_redirect = context.landing_page_url
_update_elbs(context=context)
ContextSerDe.dump_context_to_ssm(context=context)
_logger.debug("Kubectl data fetched successfully.")
def destroy_foundation(args: Tuple[str, ...]) -> None:
_logger.debug("args %s", args)
env_name: str = args[0]
context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=FoundationContext)
_logger.debug("context.name %s", context.name)
foundation.destroy(context=context)
_logger.debug("Demo Stack destroyed")
cdk_toolkit.destroy(context=context)
_logger.debug("CDK Toolkit Stack destroyed")
def list_env(env: str, variable: str) -> None:
ssm = utils.boto3_client("ssm")
res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True)
env_info: Dict[str, Any] = {}
if env and len(env) > 0:
_logger.debug(f"looking for {env}")
while True:
params = res["Parameters"]
for p in params:
if not p["Name"].endswith("context") or "teams" in p["Name"]:
continue
env_name = p["Name"].split("/")[2]
if len(env) > 0 and not env_name == env:
continue
env_name = p["Name"].split("/")[2]
context: "Context" = ContextSerDe.load_context_from_ssm(
env_name=env_name, type=Context)
_logger.debug(f"found env: {env_name}")
if context.k8_dashboard_url:
k8_dashboard_url = context.k8_dashboard_url
else:
k8_dashboard_url = ""
if len(context.teams) > 0:
teams_list: List[str] = [x.name for x in context.teams]
else:
teams_list = []
if variable == "landing-page":
print(context.landing_page_url)
elif variable == "toolkitbucket":
print(context.toolkit.s3_bucket)
elif variable == "all":
env_info[env_name] = {
"LandingPage": context.landing_page_url,
"Teams": teams_list,
"ToolkitBucket": context.toolkit.s3_bucket,
"K8Dashboard": k8_dashboard_url,
}
else:
raise Exception(f"Unknown --variable option {variable}")
if "NextToken" in res:
res = ssm.get_parameters_by_path(Path="/orbit",
Recursive=True,
NextToken=res["NextToken"])
else:
break
if variable == "all":
if len(env_info) == 0:
click.echo("There are no Orbit environments available")
return
else:
print("Available Orbit environments:")
print(json.dumps(env_info, indent=4, sort_keys=True))
def destroy_images(env: str) -> None:
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
type=Context)
_logger.debug("env %s", env)
@codeseeder.remote_function("orbit",
codebuild_role=context.toolkit.admin_role)
def destroy_images(env: str) -> None:
ecr.cleanup_remaining_repos(env_name=env)
destroy_images(env=env)
def _deploy_image(
env: str,
dir: str,
name: str,
script: Optional[str],
build_args: Optional[List[str]],
region: Optional[str],
debug: bool,
) -> None:
with MessagesContext("Deploying Docker Image", debug=debug) as msg_ctx:
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
type=Context)
if cfn.does_stack_exist(stack_name=f"orbit-{context.name}") is False:
msg_ctx.error(
"Please, deploy your environment before deploy any addicional docker image"
)
return
plugins.PLUGINS_REGISTRIES.load_plugins(
context=context,
msg_ctx=msg_ctx,
plugin_changesets=[],
teams_changeset=None,
)
msg_ctx.progress(3)
bundle_path = bundle.generate_bundle(
command_name=f"deploy_image-{name}",
context=context,
dirs=[(dir, name)])
msg_ctx.progress(4)
script_str = "NO_SCRIPT" if script is None else script
build_args = [] if build_args is None else build_args
buildspec = codebuild.generate_spec(
context=context,
plugins=True,
cmds_build=[
f"orbit remote --command _deploy_image {env} {name} {dir} {script_str} {' '.join(build_args)}"
],
changeset=None,
)
remote.run(
command_name=f"deploy_image-{name}",
context=context,
bundle_path=bundle_path,
buildspec=buildspec,
codebuild_log_callback=msg_ctx.progress_bar_callback,
timeout=30,
)
msg_ctx.info("Docker Image deploy into ECR")
address = f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/orbit-{context.name}-{name}"
msg_ctx.tip(f"ECR Image Address: {stylize(address, underline=True)}")
msg_ctx.progress(100)
def destroy_foundation(env: str, debug: bool) -> None:
with MessagesContext("Destroying", debug=debug) as msg_ctx:
ssm.cleanup_changeset(env_name=env, top_level="orbit-foundation")
ssm.cleanup_manifest(env_name=env, top_level="orbit-foundation")
if ssm.does_parameter_exist(name=f"/orbit-foundation/{env}/context") is False:
msg_ctx.info(f"Foundation {env} not found. Destroying only possible remaining resources.")
destroy_remaining_resources(env_name=env, top_level="orbit-foundation")
msg_ctx.progress(100)
return
context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env, type=FoundationContext)
msg_ctx.info("Context loaded")
msg_ctx.progress(2)
msg_ctx.progress(4)
if (
cfn.does_stack_exist(stack_name=cast(str, context.stack_name))
or cfn.does_stack_exist(stack_name=context.toolkit.stack_name)
or cfn.does_stack_exist(stack_name=context.cdk_toolkit.stack_name)
):
bundle_path = bundle.generate_bundle(command_name="destroy", context=cast(Context, context))
msg_ctx.progress(5)
buildspec = codebuild.generate_spec(
context=cast(Context, context),
plugins=False,
cmds_build=[f"orbit remote --command destroy_foundation {env}"],
changeset=None,
)
remote.run(
command_name="destroy",
context=cast(Context, context),
bundle_path=bundle_path,
buildspec=buildspec,
codebuild_log_callback=msg_ctx.progress_bar_callback,
timeout=45,
)
msg_ctx.info("Foundation destroyed")
msg_ctx.progress(95)
try:
destroy_toolkit(env_name=context.name, top_level="orbit-foundation")
except botocore.exceptions.ClientError as ex:
error = ex.response["Error"]
if "does not exist" not in error["Message"]:
raise
_logger.debug(f"Skipping toolkit destroy: {error['Message']}")
msg_ctx.info("Toolkit destroyed")
ssm.cleanup_env(env_name=context.name, top_level="orbit-foundation")
msg_ctx.progress(100)
def main() -> None:
_logger.debug("sys.argv: %s", sys.argv)
if len(sys.argv) == 3:
context: "Context" = ContextSerDe.load_context_from_ssm(
env_name=sys.argv[1], type=Context)
team_name: str = sys.argv[2]
else:
raise ValueError("Unexpected number of values in sys.argv.")
changeset: Optional["Changeset"] = load_changeset_from_ssm(
env_name=context.name)
_logger.debug("Changeset loaded.")
team_policies: Optional[List[str]] = None
image: Optional[str] = None
if changeset and changeset.teams_changeset and team_name in changeset.teams_changeset.added_teams_names:
manifest: Optional["Manifest"] = ManifestSerDe.load_manifest_from_ssm(
env_name=sys.argv[1], type=Manifest)
if manifest is None:
raise ValueError("manifest is None!")
team_manifest: Optional["TeamManifest"] = manifest.get_team_by_name(
name=team_name)
if team_manifest:
team_policies = team_manifest.policies
image = team_manifest.image
else:
raise ValueError(f"{team_name} not found in manifest!")
else:
team_context: Optional["TeamContext"] = context.get_team_by_name(
name=team_name)
if team_context:
team_policies = team_context.policies
image = team_context.image
else:
raise ValueError(f"Team {team_name} not found in the context.")
if team_policies is None:
raise ValueError("team_policies is None!")
stack_name: str = f"orbit-{context.name}-{team_name}"
outdir = os.path.join(".orbit.out", context.name, "cdk", stack_name)
os.makedirs(outdir, exist_ok=True)
shutil.rmtree(outdir)
app = App(outdir=outdir)
Team(scope=app,
id=stack_name,
context=context,
team_name=team_name,
team_policies=team_policies,
image=image)
app.synth(force=True)
def deploy_foundation(args: Tuple[str, ...]) -> None:
_logger.debug("args: %s", args)
if len(args) != 1:
raise ValueError("Unexpected number of values in args")
env_name: str = args[0]
context: "FoundationContext" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=FoundationContext)
_logger.debug("Context loaded.")
docker.login(context=context)
_logger.debug("DockerHub and ECR Logged in")
cdk_toolkit.deploy(context=context)
_logger.debug("CDK Toolkit Stack deployed")
foundation.deploy(context=context)
_logger.debug("Demo Stack deployed")
def list_images(env: str, region: Optional[str]) -> None:
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env,
type=Context)
names = utils.extract_images_names(env_name=env)
if names:
print_list(
tittle=
f"Available docker images into the {stylize(context.name)} env:",
items=[k for k in names],
)
else:
click.echo(
f"There is no docker image(s) into the {stylize(context.name)} env."
)
def destroy_env(args: Tuple[str, ...]) -> None:
_logger.debug("args %s", args)
env_name: str = args[0]
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)
_logger.debug("context.name %s", context.name)
kubectl.destroy_env(context=context)
_logger.debug("Kubernetes Environment components destroyed")
eksctl.destroy_env(context=context)
_logger.debug("EKS Environment Stacks destroyed")
env.destroy(context=context)
_logger.debug("Env Stack destroyed")
cdk_toolkit.destroy(context=context)
_logger.debug("CDK Toolkit Stack destroyed")
def delete_image(args: Tuple[str, ...]) -> None:
_logger.debug("args %s", args)
env_name: str = args[0]
context: "Context" = ContextSerDe.load_context_from_ssm(env_name=env_name, type=Context)
if len(args) == 2:
image_name: str = args[1]
else:
raise ValueError("Unexpected number of values in args.")
env.deploy(context=context, add_images=[], remove_images=[image_name], eks_system_masters_roles_changes=None)
_logger.debug("Env changes deployed")
ecr.delete_repo(repo=f"orbit-{context.name}-{image_name}")
_logger.debug("Docker Image Destroyed from ECR")