def _wait_for_execute(stack_name: str, changeset_type: str) -> None: if changeset_type == "CREATE": waiter = boto3_client("cloudformation").get_waiter( "stack_create_complete") elif changeset_type == "UPDATE": waiter = boto3_client("cloudformation").get_waiter( "stack_update_complete") else: raise RuntimeError(f"Invalid changeset type {changeset_type}") waiter_config = { "Delay": 5, "MaxAttempts": 480, } waiter.wait(StackName=stack_name, WaiterConfig=waiter_config)
def _create_changeset(stack_name: str, template_str: str, env_tag: str, template_path: str = "") -> Tuple[str, str]: now = datetime.utcnow().isoformat() description = f"Created by AWS Orbit Workbench CLI at {now} UTC" changeset_name = CHANGESET_PREFIX + str(int(time.time())) changeset_type = "UPDATE" if does_stack_exist( stack_name=stack_name) else "CREATE" kwargs = { "ChangeSetName": changeset_name, "StackName": stack_name, "ChangeSetType": changeset_type, "Capabilities": ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"], "Description": description, "Tags": ({ "Key": "Env", "Value": env_tag }, ), } if template_str: kwargs.update({"TemplateBody": template_str}) elif template_path: _logger.info(f"template_path={template_path}") kwargs.update({"TemplateURL": template_path}) resp = boto3_client("cloudformation").create_change_set(**kwargs) return str(resp["Id"]), changeset_type
def create_fargate_profile( profile_name: str, cluster_name: str, role_arn: str, subnets: List[str], namespace: str, selector_labels: Optional[Dict[str, Any]] = None, ) -> None: _logger.debug(f"Creating EKS Fargate Profile: {profile_name}") if describe_fargate_profile(profile_name=profile_name, cluster_name=cluster_name) is not None: _logger.debug(f"EKS Fargate Profile already exists: {profile_name}") return eks_client = boto3_client("eks") eks_client.create_fargate_profile( fargateProfileName=profile_name, clusterName=cluster_name, podExecutionRoleArn=role_arn, subnets=subnets, selectors=[{ "namespace": namespace, "labels": selector_labels }], ) waiter_model = WaiterModel(WAITER_CONFIG) waiter = create_waiter_with_client("FargateProfileCreated", waiter_model, eks_client) waiter.wait(fargateProfileName=profile_name, clusterName=cluster_name) _logger.debug(f"Created EKS Fargate Profile: {profile_name}")
def encrypt(context: Context, plaintext: str) -> str: client = boto3_client("kms") _logger.debug("Encrypting data") response = client.encrypt(KeyId=context.toolkit.kms_arn, Plaintext=plaintext.encode("utf-8")) return base64.b64encode(response.get("CiphertextBlob")).decode("utf-8")
def update_nodegroup_autoscaling_group( cluster_name: str, nodegroup_manifest: ManagedNodeGroupManifest) -> None: _logger.debug( f"Updating AutoScaling Group for Cluster: {cluster_name}, NodeGroup: {nodegroup_manifest.name}" ) _logger.debug( f"DesiredCapacity: {nodegroup_manifest.nodes_num_desired}, Min: {nodegroup_manifest.nodes_num_min}, " f"Max: {nodegroup_manifest.nodes_num_max}") asg = get_nodegroup_autoscaling_group( cluster_name=cluster_name, nodegroup_name=nodegroup_manifest.name) if not asg: _logger.debug( f"No AutoScaling Group found for Cluster: {cluster_name}, NodeGroup: {nodegroup_manifest.name}" ) return client = boto3_client("autoscaling") client.update_auto_scaling_group( AutoScalingGroupName=asg["AutoScalingGroupName"], MinSize=nodegroup_manifest.nodes_num_min, MaxSize=nodegroup_manifest.nodes_num_max, DesiredCapacity=nodegroup_manifest.nodes_num_desired, )
def decrypt(context: Context, ciphertext: str) -> str: client = boto3_client("kms") _logger.debug("Decrypting data") response = client.decrypt(KeyId=context.toolkit.kms_arn, CiphertextBlob=base64.b64decode(ciphertext)) return cast(str, response["Plaintext"].decode("utf-8"))
def delete_images(repo: str) -> None: client = boto3_client("ecr") for chunk in _chunks(iterable=fetch_images(repo=repo), size=100): client.batch_delete_image(repositoryName=repo, imageIds=[{ "imageDigest": i } for i in chunk])
def get_parameter_if_exists(name: str) -> Optional[Dict[str, Any]]: client = boto3_client(service_name="ssm") try: json_str: str = client.get_parameter(Name=name)["Parameter"]["Value"] except client.exceptions.ParameterNotFound: return None return cast(Dict[str, Any], json.loads(json_str))
def does_parameter_exist(name: str) -> bool: client = boto3_client(service_name="ssm") try: client.get_parameter(Name=name) return True except client.exceptions.ParameterNotFound: return False
def get_nodegroup_autoscaling_group( cluster_name: str, nodegroup_name: str) -> Optional[Dict[str, Any]]: _logger.debug( f"Getting AutoScaling Group for Cluster: {cluster_name}, NodeGroup: {nodegroup_name}" ) client = boto3_client("autoscaling") paginator = client.get_paginator("describe_auto_scaling_groups") for response in paginator.paginate(): for asg in response["AutoScalingGroups"]: found_cluster_tag = False found_nodegroup_tag = False for tag in asg["Tags"]: if tag["Key"] == "eks:cluster-name" and tag[ "Value"] == cluster_name: found_cluster_tag = True if tag["Key"] == "eks:nodegroup-name" and tag[ "Value"] == nodegroup_name: found_nodegroup_tag = True if found_cluster_tag and found_nodegroup_tag: _logger.debug( f"Found AutoScaling Group: {asg['AutoScalingGroupName']}") return cast(Dict[str, Any], asg) return None
def get_kms_key_scratch_bucket(context: "Context") -> Optional[str]: if not context.scratch_bucket_arn: return None bucket_name = context.scratch_bucket_arn.split(":::")[1] _logger.debug(f"Getting KMS Key for scratch bucket: {bucket_name}") try: s3_client = boto3_client("s3") encryption = cast( Dict[str, Any], s3_client.get_bucket_encryption(Bucket=bucket_name), ) if ("ServerSideEncryptionConfiguration" not in encryption or "Rules" not in encryption["ServerSideEncryptionConfiguration"]): return None for r in encryption["ServerSideEncryptionConfiguration"]["Rules"]: if ("ApplyServerSideEncryptionByDefault" in r and "SSEAlgorithm" in r["ApplyServerSideEncryptionByDefault"] and r["ApplyServerSideEncryptionByDefault"]["SSEAlgorithm"] == "aws:kms"): return cast( str, r["ApplyServerSideEncryptionByDefault"] ["KMSMasterKeyID"]) return None except botocore.exceptions.ClientError as e: if "ServerSideEncryptionConfigurationNotFoundError" in str(e): return None raise e
def get_role(role_name: str) -> Optional[Dict[str, Any]]: _logger.debug(f"Getting Role: {role_name}") iam_client = boto3_client("iam") try: return cast(Dict[str, Any], iam_client.get_role(RoleName=role_name)) except iam_client.exceptions.NoSuchEntityException: return None
def _filter_repos(env_name: str, page: Dict[str, Any]) -> Iterator[str]: client = boto3_client("ecr") for repo in page["repositories"]: response: Dict[str, Any] = client.list_tags_for_resource( resourceArn=repo["repositoryArn"]) for tag in response["tags"]: if tag["Key"] == "Env" and tag["Value"] == f"orbit-{env_name}": yield repo["repositoryName"]
def describe_repositories(repository_names: List[str]) -> List[Dict[str, Any]]: client = boto3_client("ecr") try: return cast( List[Dict[str, Any]], client.describe_repositories(repositoryNames=repository_names)["repositories"] ) except client.exceptions.RepositoryNotFoundException: return []
def _delete_objects(bucket: str, chunk: List[Dict[str, str]]) -> None: client_s3 = boto3_client("s3") try: client_s3.delete_objects(Bucket=bucket, Delete={"Objects": chunk}) except client_s3.exceptions.ClientError as ex: if "SlowDown" in str(ex): time.sleep(random.randint(3, 10)) client_s3.delete_objects(Bucket=bucket, Delete={"Objects": chunk})
def get_env_vpc_id(env_name: str) -> str: ec2_client = boto3_client("ec2") paginator = ec2_client.get_paginator("describe_vpcs") response_iterator = paginator.paginate(Filters=[{"Name": "tag:Env", "Values": [f"orbit-{env_name}"]}]) for response in response_iterator: for vpc in response["Vpcs"]: return cast(str, vpc["VpcId"]) raise ValueError(f"VPC not found for env {env_name}.")
def update_assume_role_roles( account_id: str, role_name: str, roles_to_add: Optional[List[str]] = None, roles_to_remove: Optional[List[str]] = None, ) -> None: if not roles_to_add and not roles_to_remove: raise Exception("One of roles_to_add or roles_to_remove is required") _logger.debug( f"Updating AssumeRolePolicy for {role_name}, Adding: {roles_to_add}, Removing: {roles_to_remove}" ) iam_client = boto3_client("iam") assume_role_policy = iam_client.get_role( RoleName=role_name)["Role"]["AssumeRolePolicyDocument"] statements = [] roles_to_add_set = (set() if roles_to_add is None else { f"arn:aws:iam::{account_id}:role/{role}" for role in roles_to_add if get_role(role) }) roles_to_remove_set = (set() if roles_to_remove is None else { f"arn:aws:iam::{account_id}:role/{role}" for role in roles_to_remove }) _logger.debug("current_policies: %s", assume_role_policy["Statement"]) for statement in assume_role_policy["Statement"]: arn = statement.get("Principal", {}).get("AWS", None) if arn in roles_to_remove_set: _logger.debug("Removing %s from AssumeRolePolicy", arn) continue elif arn in roles_to_add_set: _logger.debug( "AssumeRolePolicy Statement (%s) found containing %s", statement, arn) roles_to_add_set.remove(arn) statements.append(statement) else: _logger.debug("Keeping %s in AssumeRolePolicy", statement) statements.append(statement) for arn in roles_to_add_set: _logger.debug("Adding %s to AssumeRolePolicy", arn) statements.append({ "Effect": "Allow", "Action": "sts:AssumeRole", "Principal": { "AWS": arn } }) assume_role_policy["Statement"] = statements policy_body = json.dumps(assume_role_policy) _logger.debug("policy_body: %s", policy_body) iam_client.update_assume_role_policy(RoleName=role_name, PolicyDocument=policy_body)
def fetch_toolkit_data(context: V) -> None: _logger.debug("Fetching Toolkit data...") if not (isinstance(context, Context) or isinstance(context, FoundationContext)): raise ValueError("Unknown 'context' Type") top_level = "orbit" if isinstance(context, Context) else "orbit-foundation" resp_type = Dict[str, List[Dict[str, List[Dict[str, str]]]]] try: response: resp_type = boto3_client( "cloudformation").describe_stacks( StackName=context.toolkit.stack_name) _logger.debug("%s stack found.", context.toolkit.stack_name) except botocore.exceptions.ClientError as ex: error: Dict[str, Any] = ex.response["Error"] if error[ "Code"] == "ValidationError" and f"{context.toolkit.stack_name} not found" in error[ "Message"]: _logger.debug("Toolkit stack not found.") return if (error["Code"] == "ValidationError" and f"{context.toolkit.stack_name} does not exist" in error["Message"]): _logger.debug("Toolkit stack does not exist.") return raise if len(response["Stacks"]) < 1: _logger.debug("Toolkit stack not found.") return if "Outputs" not in response["Stacks"][0]: _logger.debug("Toolkit stack with empty outputs") return for output in response["Stacks"][0]["Outputs"]: if output["ExportName"] == f"{top_level}-{context.name}-deploy-id": _logger.debug("Export value: %s", output["OutputValue"]) context.toolkit.deploy_id = output["OutputValue"] if output["ExportName"] == f"{top_level}-{context.name}-kms-arn": _logger.debug("Export value: %s", output["OutputValue"]) context.toolkit.kms_arn = output["OutputValue"] if context.toolkit.deploy_id is None: raise RuntimeError( f"Stack {context.toolkit.stack_name} does not have the expected {top_level}-{context.name}-deploy-id output." ) if context.toolkit.kms_arn is None: raise RuntimeError( f"Stack {context.toolkit.stack_name} does not have the expected {top_level}-{context.name}-kms-arn output." ) context.toolkit.kms_alias = f"{top_level}-{context.name}-{context.toolkit.deploy_id}" context.toolkit.s3_bucket = ( f"{top_level}-{context.name}-toolkit-{context.account_id}-{context.toolkit.deploy_id}" ) context.cdk_toolkit.s3_bucket = ( f"{top_level}-{context.name}-cdk-toolkit-{context.account_id}-{context.toolkit.deploy_id}" ) _logger.debug("Toolkit data fetched successfully.")
def delete_docker_credentials(secret_id: str) -> None: client = boto3_client("secretsmanager") try: _logger.debug("Deleting Secret: %s", secret_id) client.delete_secret(SecretId=secret_id, ForceDeleteWithoutRecovery=True) except ClientError as e: _logger.exception(e)
def start( context: "Context", project_name: str, stream_name: str, bundle_location: str, buildspec: Dict[str, Any], timeout: int, overrides: Optional[Dict[str, Any]] = None, ) -> str: client = boto3_client("codebuild") repo: Optional[str] = None credentials: Optional[str] = None if context.images.code_build.get_source( account_id=context.account_id, region=context.region).startswith("ecr"): version = context.images.code_build.version repo = f"{context.images.code_build.repository}:{version}" if not any(match in repo for match in [".amazonaws.com/", "public.ecr.aws"]): repo = f"{context.account_id}.dkr.ecr.{context.region}.amazonaws.com/{repo}:{version}" credentials = "SERVICE_ROLE" _logger.debug("Repository: %s", repo) _logger.debug("Credentials: %s", credentials) build_params = { "projectName": project_name, "sourceTypeOverride": "S3", "sourceLocationOverride": bundle_location, "buildspecOverride": yaml.safe_dump(data=buildspec, sort_keys=False, indent=4), "timeoutInMinutesOverride": timeout, "privilegedModeOverride": True, "logsConfigOverride": { "cloudWatchLogs": { "status": "ENABLED", "groupName": f"/aws/codebuild/{project_name}", "streamName": stream_name, }, "s3Logs": { "status": "DISABLED" }, }, } if repo: build_params["imageOverride"] = repo if credentials: build_params["imagePullCredentialsTypeOverride"] = credentials if overrides: build_params = {**build_params, **overrides} response: Dict[str, Any] = client.start_build(**build_params) return str(response["build"]["id"])
def describe_cluster(cluster_name: str, ) -> Optional[Dict[str, Any]]: _logger.debug(f"Describing Cluster: {cluster_name}") eks_client = boto3_client("eks") try: return cast(Dict[str, Any], eks_client.describe_cluster(name=cluster_name)) except eks_client.exceptions.ResourceNotFoundException: return None
def put_parameter(name: str, obj: Dict[str, Any]) -> None: client = boto3_client(service_name="ssm") client.put_parameter( Name=name, Value=str(json.dumps(obj=obj, sort_keys=True)), Overwrite=True, Tier="Intelligent-Tiering", Type="String", )
def create_repository(repository_name: str) -> None: client = boto3_client("ecr") params: Dict[str, Any] = {"repositoryName": repository_name} response = client.create_repository(**params) if "repository" in response and "repositoryName" in response["repository"]: _logger.debug("ECR repository not exist, creating for %s", repository_name) else: _logger.error("ECR repository creation failed, response %s", response) raise RuntimeError(response)
def list_env(env: str, variable: str) -> None: ssm = utils.boto3_client("ssm") res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True) env_info: Dict[str, str] = {} while True: params = res["Parameters"] for p in params: if not p["Name"].endswith("context") or "teams" in p["Name"]: continue if len(env) > 0 and p["Name"].startswith(f"//orbit/{env}"): continue env_name = p["Name"].split("/")[2] context: "Context" = ContextSerDe.load_context_from_ssm( env_name=env_name, type=Context) _logger.debug(f"found env: {env_name}") if context.k8_dashboard_url: k8_dashboard_url = context.k8_dashboard_url else: k8_dashboard_url = "" if len(context.teams) > 0: teams_list: str = ",".join([x.name for x in context.teams]) else: teams_list = "" if variable == "landing-page": print(context.landing_page_url) elif variable == "toolkitbucket": print(context.toolkit.s3_bucket) elif variable == "teams": print(f"[{teams_list}]") elif variable == "all": env_info[env_name] = ( f"LandingPage={context.landing_page_url}, " f"Teams=[{teams_list}], " f"ToolkitBucket={context.toolkit.s3_bucket}" f"K8Dashboard={k8_dashboard_url}") else: raise Exception(f"Unknown --variable option {variable}") if "NextToken" in res: res = ssm.get_parameters_by_path(Path="/orbit", Recursive=True, NextToken=res["NextToken"]) else: break if variable == "all": if len(env_info) == 0: click.echo("There are no Orbit environments available") return else: print_list( tittle="Available Orbit environments:", items=[ f"Name={k}{stylize(',')}{v}" for k, v in env_info.items() ], )
def get_stack_status(stack_name: str) -> str: client = boto3_client("cloudformation") try: resp = client.describe_stacks(StackName=stack_name) if len(resp["Stacks"]) < 1: raise ValueError(f"CloudFormation stack {stack_name} not found.") except botocore.exceptions.ClientError: raise return cast(str, resp["Stacks"][0]["StackStatus"])
def delete_cert_from_iam(context: "FoundationContext") -> None: iam_client = boto3_client("iam") cert_name = f"{context.name}-{context.region}" try: iam_client.delete_server_certificate(ServerCertificateName=cert_name) except botocore.exceptions.ClientError as ex: if ex.response["Error"]["Code"] == "NoSuchEntity": pass else: raise ex
def get_credential(region: Optional[str] = None) -> Tuple[str, str]: if region is None: ecr_client = boto3_client("ecr") else: _logger.debug("Creating custom boto3 session for region %s", region) ecr_client = boto3.Session(region_name=region).client("ecr") result = ecr_client.get_authorization_token() auth = result["authorizationData"][0] auth_token = b64decode(auth["authorizationToken"]).decode() return cast(Tuple[str, str], tuple(auth_token.split(sep=":", maxsplit=1)))
def delete_cert_from_iam() -> None: iam_client = boto3_client("iam") cert_name = "AWSORBIT" try: iam_client.delete_server_certificate(ServerCertificateName=cert_name) except botocore.exceptions.ClientError as ex: if ex.response["Error"]["Code"] == "NoSuchEntity": pass else: raise ex
def _fetch_repo_uri(names: List[str], context: "Context") -> Dict[str, str]: names = [f"orbit-{context.name}-{x}" for x in names] ret: Dict[str, str] = {x: "" for x in names} client = boto3_client("ecr") paginator = client.get_paginator("describe_repositories") for page in paginator.paginate(repositoryNames=names): for repo in page["repositories"]: ret[repo["repositoryName"]] = repo["repositoryUri"] ret = {k.replace(f"orbit-{context.name}-", ""): v for k, v in ret.items()} return ret
def _delete_targets(context: "Context", fs_id: str) -> None: client = boto3_client("efs") for target in _fetch_targets(context=context, fs_id=fs_id): try: _logger.debug(f"Deleting MountTargetId: {target}") client.delete_mount_target(MountTargetId=target) except client.exceptions.MountTargetNotFound: _logger.warning( f"Ignoring MountTargetId {target} deletion cause it does not exist anymore." )