def _wait_for_execute(stack_name: str, changeset_type: str) -> None:
if changeset_type == "CREATE":
waiter = boto3_client("cloudformation").get_waiter(
"stack_create_complete")
elif changeset_type == "UPDATE":
waiter = boto3_client("cloudformation").get_waiter(
"stack_update_complete")
else:
raise RuntimeError(f"Invalid changeset type {changeset_type}")
waiter_config = {
"Delay": 5,
"MaxAttempts": 480,
}
waiter.wait(StackName=stack_name, WaiterConfig=waiter_config)
def _create_changeset(stack_name: str,
template_str: str,
env_tag: str,
template_path: str = "") -> Tuple[str, str]:
now = datetime.utcnow().isoformat()
description = f"Created by AWS Orbit Workbench CLI at {now} UTC"
changeset_name = CHANGESET_PREFIX + str(int(time.time()))
changeset_type = "UPDATE" if does_stack_exist(
stack_name=stack_name) else "CREATE"
kwargs = {
"ChangeSetName": changeset_name,
"StackName": stack_name,
"ChangeSetType": changeset_type,
"Capabilities": ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"],
"Description": description,
"Tags": ({
"Key": "Env",
"Value": env_tag
}, ),
}
if template_str:
kwargs.update({"TemplateBody": template_str})
elif template_path:
_logger.info(f"template_path={template_path}")
kwargs.update({"TemplateURL": template_path})
resp = boto3_client("cloudformation").create_change_set(**kwargs)
return str(resp["Id"]), changeset_type
def create_fargate_profile(
profile_name: str,
cluster_name: str,
role_arn: str,
subnets: List[str],
namespace: str,
selector_labels: Optional[Dict[str, Any]] = None,
) -> None:
_logger.debug(f"Creating EKS Fargate Profile: {profile_name}")
if describe_fargate_profile(profile_name=profile_name,
cluster_name=cluster_name) is not None:
_logger.debug(f"EKS Fargate Profile already exists: {profile_name}")
return
eks_client = boto3_client("eks")
eks_client.create_fargate_profile(
fargateProfileName=profile_name,
clusterName=cluster_name,
podExecutionRoleArn=role_arn,
subnets=subnets,
selectors=[{
"namespace": namespace,
"labels": selector_labels
}],
)
waiter_model = WaiterModel(WAITER_CONFIG)
waiter = create_waiter_with_client("FargateProfileCreated", waiter_model,
eks_client)
waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name)
_logger.debug(f"Created EKS Fargate Profile: {profile_name}")
def encrypt(context: Context, plaintext: str) -> str:
client = boto3_client("kms")
_logger.debug("Encrypting data")
response = client.encrypt(KeyId=context.toolkit.kms_arn,
Plaintext=plaintext.encode("utf-8"))
return base64.b64encode(response.get("CiphertextBlob")).decode("utf-8")
def update_nodegroup_autoscaling_group(
cluster_name: str,
nodegroup_manifest: ManagedNodeGroupManifest) -> None:
_logger.debug(
f"Updating AutoScaling Group for Cluster: {cluster_name}, NodeGroup: {nodegroup_manifest.name}"
)
_logger.debug(
f"DesiredCapacity: {nodegroup_manifest.nodes_num_desired}, Min: {nodegroup_manifest.nodes_num_min}, "
f"Max: {nodegroup_manifest.nodes_num_max}")
asg = get_nodegroup_autoscaling_group(
cluster_name=cluster_name, nodegroup_name=nodegroup_manifest.name)
if not asg:
_logger.debug(
f"No AutoScaling Group found for Cluster: {cluster_name}, NodeGroup: {nodegroup_manifest.name}"
)
return
client = boto3_client("autoscaling")
client.update_auto_scaling_group(
AutoScalingGroupName=asg["AutoScalingGroupName"],
MinSize=nodegroup_manifest.nodes_num_min,
MaxSize=nodegroup_manifest.nodes_num_max,
DesiredCapacity=nodegroup_manifest.nodes_num_desired,
)
def decrypt(context: Context, ciphertext: str) -> str:
client = boto3_client("kms")
_logger.debug("Decrypting data")
response = client.decrypt(KeyId=context.toolkit.kms_arn,
CiphertextBlob=base64.b64decode(ciphertext))
return cast(str, response["Plaintext"].decode("utf-8"))
def delete_images(repo: str) -> None:
client = boto3_client("ecr")
for chunk in _chunks(iterable=fetch_images(repo=repo), size=100):
client.batch_delete_image(repositoryName=repo,
imageIds=[{
"imageDigest": i
} for i in chunk])
def get_parameter_if_exists(name: str) -> Optional[Dict[str, Any]]:
client = boto3_client(service_name="ssm")
try:
json_str: str = client.get_parameter(Name=name)["Parameter"]["Value"]
except client.exceptions.ParameterNotFound:
return None
return cast(Dict[str, Any], json.loads(json_str))
def does_parameter_exist(name: str) -> bool:
client = boto3_client(service_name="ssm")
try:
client.get_parameter(Name=name)
return True
except client.exceptions.ParameterNotFound:
return False
def get_nodegroup_autoscaling_group(
cluster_name: str, nodegroup_name: str) -> Optional[Dict[str, Any]]:
_logger.debug(
f"Getting AutoScaling Group for Cluster: {cluster_name}, NodeGroup: {nodegroup_name}"
)
client = boto3_client("autoscaling")
paginator = client.get_paginator("describe_auto_scaling_groups")
for response in paginator.paginate():
for asg in response["AutoScalingGroups"]:
found_cluster_tag = False
found_nodegroup_tag = False
for tag in asg["Tags"]:
if tag["Key"] == "eks:cluster-name" and tag[
"Value"] == cluster_name:
found_cluster_tag = True
if tag["Key"] == "eks:nodegroup-name" and tag[
"Value"] == nodegroup_name:
found_nodegroup_tag = True
if found_cluster_tag and found_nodegroup_tag:
_logger.debug(
f"Found AutoScaling Group: {asg['AutoScalingGroupName']}")
return cast(Dict[str, Any], asg)
return None
def get_kms_key_scratch_bucket(context: "Context") -> Optional[str]:
if not context.scratch_bucket_arn:
return None
bucket_name = context.scratch_bucket_arn.split(":::")[1]
_logger.debug(f"Getting KMS Key for scratch bucket: {bucket_name}")
try:
s3_client = boto3_client("s3")
encryption = cast(
Dict[str, Any],
s3_client.get_bucket_encryption(Bucket=bucket_name),
)
if ("ServerSideEncryptionConfiguration" not in encryption
or "Rules"
not in encryption["ServerSideEncryptionConfiguration"]):
return None
for r in encryption["ServerSideEncryptionConfiguration"]["Rules"]:
if ("ApplyServerSideEncryptionByDefault" in r
and "SSEAlgorithm"
in r["ApplyServerSideEncryptionByDefault"] and
r["ApplyServerSideEncryptionByDefault"]["SSEAlgorithm"]
== "aws:kms"):
return cast(
str, r["ApplyServerSideEncryptionByDefault"]
["KMSMasterKeyID"])
return None
except botocore.exceptions.ClientError as e:
if "ServerSideEncryptionConfigurationNotFoundError" in str(e):
return None
raise e
def get_role(role_name: str) -> Optional[Dict[str, Any]]:
_logger.debug(f"Getting Role: {role_name}")
iam_client = boto3_client("iam")
try:
return cast(Dict[str, Any], iam_client.get_role(RoleName=role_name))
except iam_client.exceptions.NoSuchEntityException:
return None
def _filter_repos(env_name: str, page: Dict[str, Any]) -> Iterator[str]:
client = boto3_client("ecr")
for repo in page["repositories"]:
response: Dict[str, Any] = client.list_tags_for_resource(
resourceArn=repo["repositoryArn"])
for tag in response["tags"]:
if tag["Key"] == "Env" and tag["Value"] == f"orbit-{env_name}":
yield repo["repositoryName"]
def describe_repositories(repository_names: List[str]) -> List[Dict[str, Any]]:
client = boto3_client("ecr")
try:
return cast(
List[Dict[str, Any]], client.describe_repositories(repositoryNames=repository_names)["repositories"]
)
except client.exceptions.RepositoryNotFoundException:
return []
def _delete_objects(bucket: str, chunk: List[Dict[str, str]]) -> None:
client_s3 = boto3_client("s3")
try:
client_s3.delete_objects(Bucket=bucket, Delete={"Objects": chunk})
except client_s3.exceptions.ClientError as ex:
if "SlowDown" in str(ex):
time.sleep(random.randint(3, 10))
client_s3.delete_objects(Bucket=bucket, Delete={"Objects": chunk})
def get_env_vpc_id(env_name: str) -> str:
ec2_client = boto3_client("ec2")
paginator = ec2_client.get_paginator("describe_vpcs")
response_iterator = paginator.paginate(Filters=[{"Name": "tag:Env", "Values": [f"orbit-{env_name}"]}])
for response in response_iterator:
for vpc in response["Vpcs"]:
return cast(str, vpc["VpcId"])
raise ValueError(f"VPC not found for env {env_name}.")
def update_assume_role_roles(
account_id: str,
role_name: str,
roles_to_add: Optional[List[str]] = None,
roles_to_remove: Optional[List[str]] = None,
) -> None:
if not roles_to_add and not roles_to_remove:
raise Exception("One of roles_to_add or roles_to_remove is required")
_logger.debug(
f"Updating AssumeRolePolicy for {role_name}, Adding: {roles_to_add}, Removing: {roles_to_remove}"
)
iam_client = boto3_client("iam")
assume_role_policy = iam_client.get_role(
RoleName=role_name)["Role"]["AssumeRolePolicyDocument"]
statements = []
roles_to_add_set = (set() if roles_to_add is None else {
f"arn:aws:iam::{account_id}:role/{role}"
for role in roles_to_add if get_role(role)
})
roles_to_remove_set = (set() if roles_to_remove is None else {
f"arn:aws:iam::{account_id}:role/{role}"
for role in roles_to_remove
})
_logger.debug("current_policies: %s", assume_role_policy["Statement"])
for statement in assume_role_policy["Statement"]:
arn = statement.get("Principal", {}).get("AWS", None)
if arn in roles_to_remove_set:
_logger.debug("Removing %s from AssumeRolePolicy", arn)
continue
elif arn in roles_to_add_set:
_logger.debug(
"AssumeRolePolicy Statement (%s) found containing %s",
statement, arn)
roles_to_add_set.remove(arn)
statements.append(statement)
else:
_logger.debug("Keeping %s in AssumeRolePolicy", statement)
statements.append(statement)
for arn in roles_to_add_set:
_logger.debug("Adding %s to AssumeRolePolicy", arn)
statements.append({
"Effect": "Allow",
"Action": "sts:AssumeRole",
"Principal": {
"AWS": arn
}
})
assume_role_policy["Statement"] = statements
policy_body = json.dumps(assume_role_policy)
_logger.debug("policy_body: %s", policy_body)
iam_client.update_assume_role_policy(RoleName=role_name,
PolicyDocument=policy_body)
def fetch_toolkit_data(context: V) -> None:
_logger.debug("Fetching Toolkit data...")
if not (isinstance(context, Context)
or isinstance(context, FoundationContext)):
raise ValueError("Unknown 'context' Type")
top_level = "orbit" if isinstance(context,
Context) else "orbit-foundation"
resp_type = Dict[str, List[Dict[str, List[Dict[str, str]]]]]
try:
response: resp_type = boto3_client(
"cloudformation").describe_stacks(
StackName=context.toolkit.stack_name)
_logger.debug("%s stack found.", context.toolkit.stack_name)
except botocore.exceptions.ClientError as ex:
error: Dict[str, Any] = ex.response["Error"]
if error[
"Code"] == "ValidationError" and f"{context.toolkit.stack_name} not found" in error[
"Message"]:
_logger.debug("Toolkit stack not found.")
return
if (error["Code"] == "ValidationError"
and f"{context.toolkit.stack_name} does not exist"
in error["Message"]):
_logger.debug("Toolkit stack does not exist.")
return
raise
if len(response["Stacks"]) < 1:
_logger.debug("Toolkit stack not found.")
return
if "Outputs" not in response["Stacks"][0]:
_logger.debug("Toolkit stack with empty outputs")
return
for output in response["Stacks"][0]["Outputs"]:
if output["ExportName"] == f"{top_level}-{context.name}-deploy-id":
_logger.debug("Export value: %s", output["OutputValue"])
context.toolkit.deploy_id = output["OutputValue"]
if output["ExportName"] == f"{top_level}-{context.name}-kms-arn":
_logger.debug("Export value: %s", output["OutputValue"])
context.toolkit.kms_arn = output["OutputValue"]
if context.toolkit.deploy_id is None:
raise RuntimeError(
f"Stack {context.toolkit.stack_name} does not have the expected {top_level}-{context.name}-deploy-id output."
)
if context.toolkit.kms_arn is None:
raise RuntimeError(
f"Stack {context.toolkit.stack_name} does not have the expected {top_level}-{context.name}-kms-arn output."
)
context.toolkit.kms_alias = f"{top_level}-{context.name}-{context.toolkit.deploy_id}"
context.toolkit.s3_bucket = (
f"{top_level}-{context.name}-toolkit-{context.account_id}-{context.toolkit.deploy_id}"
)
context.cdk_toolkit.s3_bucket = (
f"{top_level}-{context.name}-cdk-toolkit-{context.account_id}-{context.toolkit.deploy_id}"
)
_logger.debug("Toolkit data fetched successfully.")
def delete_docker_credentials(secret_id: str) -> None:
client = boto3_client("secretsmanager")
try:
_logger.debug("Deleting Secret: %s", secret_id)
client.delete_secret(SecretId=secret_id,
ForceDeleteWithoutRecovery=True)
except ClientError as e:
_logger.exception(e)
def start(
context: "Context",
project_name: str,
stream_name: str,
bundle_location: str,
buildspec: Dict[str, Any],
timeout: int,
overrides: Optional[Dict[str, Any]] = None,
) -> str:
client = boto3_client("codebuild")
repo: Optional[str] = None
credentials: Optional[str] = None
if context.images.code_build.get_source(
account_id=context.account_id,
region=context.region).startswith("ecr"):
version = context.images.code_build.version
repo = f"{context.images.code_build.repository}:{version}"
if not any(match in repo
for match in [".amazonaws.com/", "public.ecr.aws"]):
repo = f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/{repo}:{version}"
credentials = "SERVICE_ROLE"
_logger.debug("Repository: %s", repo)
_logger.debug("Credentials: %s", credentials)
build_params = {
"projectName":
project_name,
"sourceTypeOverride":
"S3",
"sourceLocationOverride":
bundle_location,
"buildspecOverride":
yaml.safe_dump(data=buildspec, sort_keys=False, indent=4),
"timeoutInMinutesOverride":
timeout,
"privilegedModeOverride":
True,
"logsConfigOverride": {
"cloudWatchLogs": {
"status": "ENABLED",
"groupName": f"/aws/codebuild/{project_name}",
"streamName": stream_name,
},
"s3Logs": {
"status": "DISABLED"
},
},
}
if repo:
build_params["imageOverride"] = repo
if credentials:
build_params["imagePullCredentialsTypeOverride"] = credentials
if overrides:
build_params = {**build_params, **overrides}
response: Dict[str, Any] = client.start_build(**build_params)
return str(response["build"]["id"])
def describe_cluster(cluster_name: str, ) -> Optional[Dict[str, Any]]:
_logger.debug(f"Describing Cluster: {cluster_name}")
eks_client = boto3_client("eks")
try:
return cast(Dict[str, Any],
eks_client.describe_cluster(name=cluster_name))
except eks_client.exceptions.ResourceNotFoundException:
return None
def put_parameter(name: str, obj: Dict[str, Any]) -> None:
client = boto3_client(service_name="ssm")
client.put_parameter(
Name=name,
Value=str(json.dumps(obj=obj, sort_keys=True)),
Overwrite=True,
Tier="Intelligent-Tiering",
Type="String",
)
def create_repository(repository_name: str) -> None:
client = boto3_client("ecr")
params: Dict[str, Any] = {"repositoryName": repository_name}
response = client.create_repository(**params)
if "repository" in response and "repositoryName" in response["repository"]:
_logger.debug("ECR repository not exist, creating for %s", repository_name)
else:
_logger.error("ECR repository creation failed, response %s", response)
raise RuntimeError(response)
def list_env(env: str, variable: str) -> None:
ssm = utils.boto3_client("ssm")
res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True)
env_info: Dict[str, str] = {}
while True:
params = res["Parameters"]
for p in params:
if not p["Name"].endswith("context") or "teams" in p["Name"]:
continue
if len(env) > 0 and p["Name"].startswith(f"//orbit/{env}"):
continue
env_name = p["Name"].split("/")[2]
context: "Context" = ContextSerDe.load_context_from_ssm(
env_name=env_name, type=Context)
_logger.debug(f"found env: {env_name}")
if context.k8_dashboard_url:
k8_dashboard_url = context.k8_dashboard_url
else:
k8_dashboard_url = ""
if len(context.teams) > 0:
teams_list: str = ",".join([x.name for x in context.teams])
else:
teams_list = ""
if variable == "landing-page":
print(context.landing_page_url)
elif variable == "toolkitbucket":
print(context.toolkit.s3_bucket)
elif variable == "teams":
print(f"[{teams_list}]")
elif variable == "all":
env_info[env_name] = (
f"LandingPage={context.landing_page_url}, "
f"Teams=[{teams_list}], "
f"ToolkitBucket={context.toolkit.s3_bucket}"
f"K8Dashboard={k8_dashboard_url}")
else:
raise Exception(f"Unknown --variable option {variable}")
if "NextToken" in res:
res = ssm.get_parameters_by_path(Path="/orbit",
Recursive=True,
NextToken=res["NextToken"])
else:
break
if variable == "all":
if len(env_info) == 0:
click.echo("There are no Orbit environments available")
return
else:
print_list(
tittle="Available Orbit environments:",
items=[
f"Name={k}{stylize(',')}{v}" for k, v in env_info.items()
],
)
def get_stack_status(stack_name: str) -> str:
client = boto3_client("cloudformation")
try:
resp = client.describe_stacks(StackName=stack_name)
if len(resp["Stacks"]) < 1:
raise ValueError(f"CloudFormation stack {stack_name} not found.")
except botocore.exceptions.ClientError:
raise
return cast(str, resp["Stacks"][0]["StackStatus"])
def delete_cert_from_iam(context: "FoundationContext") -> None:
iam_client = boto3_client("iam")
cert_name = f"{context.name}-{context.region}"
try:
iam_client.delete_server_certificate(ServerCertificateName=cert_name)
except botocore.exceptions.ClientError as ex:
if ex.response["Error"]["Code"] == "NoSuchEntity":
pass
else:
raise ex
def get_credential(region: Optional[str] = None) -> Tuple[str, str]:
if region is None:
ecr_client = boto3_client("ecr")
else:
_logger.debug("Creating custom boto3 session for region %s", region)
ecr_client = boto3.Session(region_name=region).client("ecr")
result = ecr_client.get_authorization_token()
auth = result["authorizationData"][0]
auth_token = b64decode(auth["authorizationToken"]).decode()
return cast(Tuple[str, str], tuple(auth_token.split(sep=":", maxsplit=1)))
def delete_cert_from_iam() -> None:
iam_client = boto3_client("iam")
cert_name = "AWSORBIT"
try:
iam_client.delete_server_certificate(ServerCertificateName=cert_name)
except botocore.exceptions.ClientError as ex:
if ex.response["Error"]["Code"] == "NoSuchEntity":
pass
else:
raise ex
def _fetch_repo_uri(names: List[str], context: "Context") -> Dict[str, str]:
names = [f"orbit-{context.name}-{x}" for x in names]
ret: Dict[str, str] = {x: "" for x in names}
client = boto3_client("ecr")
paginator = client.get_paginator("describe_repositories")
for page in paginator.paginate(repositoryNames=names):
for repo in page["repositories"]:
ret[repo["repositoryName"]] = repo["repositoryUri"]
ret = {k.replace(f"orbit-{context.name}-", ""): v for k, v in ret.items()}
return ret
def _delete_targets(context: "Context", fs_id: str) -> None:
client = boto3_client("efs")
for target in _fetch_targets(context=context, fs_id=fs_id):
try:
_logger.debug(f"Deleting MountTargetId: {target}")
client.delete_mount_target(MountTargetId=target)
except client.exceptions.MountTargetNotFound:
_logger.warning(
f"Ignoring MountTargetId {target} deletion cause it does not exist anymore."
)