def update_stack(
stack_name: str,
template_body: str,
boto3_session: boto3.session.Session,
parameters: Optional[List] = None,
) -> None:
cfn = boto3_session.client("cloudformation")
ec2 = boto3_session.client("ec2")
update_waiter = cfn.get_waiter("stack_update_complete")
print(f"Updating stack {stack_name}. This may take a few minutes... "
f"Check the CloudFormation Console for updates")
stack_output = get_output(stack_name, boto3_session)
ec2.stop_instances(
InstanceIds=[stack_output[constants.cloudformation.MASTER_ID]])
stop_waiter = ec2.get_waiter("instance_stopped")
stop_waiter.wait(
InstanceIds=[stack_output[constants.cloudformation.MASTER_ID]],
WaiterConfig={"Delay": 10},
)
terminate_running_agents(
stack_output[constants.cloudformation.AGENT_TAG_NAME], boto3_session)
try:
if parameters:
cfn.update_stack(
StackName=stack_name,
TemplateBody=template_body,
Parameters=parameters,
Capabilities=["CAPABILITY_IAM"],
)
else:
cfn.update_stack(StackName=stack_name,
TemplateBody=template_body,
Capabilities=["CAPABILITY_IAM"])
except ClientError as e:
if e.response["Error"]["Message"] != "No updates are to be performed.":
raise e
print(e.response["Error"]["Message"])
ec2.start_instances(
InstanceIds=[stack_output[constants.cloudformation.MASTER_ID]])
start_waiter = ec2.get_waiter("instance_running")
start_waiter.wait(
InstanceIds=[stack_output[constants.cloudformation.MASTER_ID]],
WaiterConfig={"Delay": 10},
)
return
update_waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10})
def get_client(service: str, session: boto3.session.Session, region: str = None):
"""
Instantiates an AWS API client
:param service: Service targeted, e.g. ec2
:param session: The aws session
:param region: Region desired, e.g. us-east-2
:return:
"""
return AWSFacadeUtils._clients.setdefault(
(service, region),
session.client(service, region_name=region) if region else session.client(service))
def get_role_list_data(boto3_session: boto3.session.Session) -> Dict:
client = boto3_session.client('iam')
paginator = client.get_paginator('list_roles')
roles: List[Dict] = []
for page in paginator.paginate():
roles.extend(page['Roles'])
return {'Roles': roles}
def get_group_list_data(boto3_session: boto3.session.Session) -> Dict:
client = boto3_session.client('iam')
paginator = client.get_paginator('list_groups')
groups: List[Dict] = []
for page in paginator.paginate():
groups.extend(page['Groups'])
return {'Groups': groups}
def get_s3_bucket_details(
boto3_session: boto3.session.Session,
bucket_data: Dict,
) -> Generator[Tuple[str, Dict, Dict, Dict, Dict, Dict], None, None]:
"""
Iterates over all S3 buckets. Yields bucket name (string), S3 bucket policies (JSON), ACLs (JSON),
default encryption policy (JSON), Versioning (JSON), and Public Access Block (JSON)
"""
# a local store for s3 clients so that we may re-use clients for an AWS region
s3_regional_clients: Dict[Any, Any] = {}
for bucket in bucket_data['Buckets']:
# Note: bucket['Region'] is sometimes None because
# client.get_bucket_location() does not return a location constraint for buckets
# in us-east-1 region
client = s3_regional_clients.get(bucket['Region'])
if not client:
client = boto3_session.client('s3', bucket['Region'])
s3_regional_clients[bucket['Region']] = client
acl = get_acl(bucket, client)
policy = get_policy(bucket, client)
encryption = get_encryption(bucket, client)
versioning = get_versioning(bucket, client)
public_access_block = get_public_access_block(bucket, client)
yield bucket[
'Name'], acl, policy, encryption, versioning, public_access_block
def get_volumes(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('ec2', region_name=region)
paginator = client.get_paginator('describe_volumes')
volumes: List[Dict] = []
for page in paginator.paginate():
volumes.extend(page['Volumes'])
return volumes
def list_spot_requests_for_stack(
stack_name: str, boto3_session: boto3.session.Session) -> List[Dict]:
tag_key, tag_val = get_management_tag_key_value(stack_name)
ec2 = boto3_session.client("ec2")
response = ec2.describe_spot_instance_requests(Filters=[
{
"Name": f"tag:{tag_key}",
"Values": [tag_val]
},
{
"Name": "state",
"Values": ["open", "active"]
},
])
spot_requests = response["SpotInstanceRequests"]
reqs = []
for s in spot_requests:
req = {
"id": s["SpotInstanceRequestId"],
"state": s["State"],
"statusCode": s["Status"]["Code"],
"statusMessage": s["Status"]["Message"],
"instanceId": s.get("InstanceId", None),
}
reqs.append(req)
return reqs
def stop_master(master_id: str,
boto3_session: boto3.session.Session,
delete: bool = False) -> None:
ec2 = boto3_session.client("ec2")
waiter = ec2.get_waiter("instance_stopped")
try:
ec2.stop_instances(InstanceIds=[master_id])
except ClientError as ex:
if delete:
error_code = ex.response.get("Error", {}).get("Code")
if error_code in DELETE_MASTER_IGNORE_ERRORS:
print(
f"Failed to stop Master Instance: {error_code}. "
"This error is ignored as the instance is going to be deleted."
)
return
raise
ec2.modify_instance_attribute(Attribute="disableApiTermination",
Value="false",
InstanceId=master_id)
for n in range(NUM_WAITS):
print("Waiting For Master Instance To Stop")
try:
waiter.wait(InstanceIds=[master_id], WaiterConfig={"Delay": 10})
break
except WaiterError as e:
if n == NUM_WAITS - 1:
raise e
print("Master Instance Stopped")
def get_secret_list(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('secretsmanager', region_name=region)
paginator = client.get_paginator('list_secrets')
secrets: List[Dict] = []
for page in paginator.paginate():
secrets.extend(page['SecretList'])
return secrets
def _test_objects_exist(
session: boto3.session.Session,
bucket_name: str,
bucket_root: str,
file_list: List[str],
) -> None:
s3 = session.resource("s3")
bucket = s3.Bucket(bucket_name)
# dirnames = _file_list_dirnames(file_list, bucket_root)
bucket_objects = []
for obj in bucket.objects.filter(Prefix=bucket_root):
# skip directory objects
if not os.path.splitext(obj.key)[-1]:
continue
bucket_objects.append(obj.key)
assert len(bucket_objects) == len(file_list)
for path in file_list:
found = False
for bucket_path in bucket_objects:
if bucket_path.endswith(path):
found = True
continue
if not found:
log.error("{0} not found in bucket".format(bucket_path))
assert False
def terminate_running_agents(agent_tag_name: str,
boto3_session: boto3.session.Session) -> None:
ec2 = boto3_session.client("ec2")
waiter = ec2.get_waiter("instance_terminated")
response = ec2.describe_instances(Filters=[
{
"Name": "tag:Name",
"Values": [agent_tag_name]
},
{
"Name": "instance-state-name",
"Values": ["running"]
},
])
reservations = response["Reservations"]
instance_ids = []
for reservation in reservations:
for instance in reservation["Instances"]:
instance_ids.append(instance["InstanceId"])
if instance_ids:
ec2.terminate_instances(InstanceIds=instance_ids)
waiter.wait(InstanceIds=instance_ids, WaiterConfig={"Delay": 10})
def delete_spot_requests_and_agents(
stack_name: str, boto3_session: boto3.session.Session
) -> List[str]:
"""
List all spot requests. Any requests that have an associated instance,
terminate the instances (this will automatically cancel the spot
request). Any requests that do not have an associated instance, cancel
the spot requests.
Returns the list of instance_ids that were deleted so at the end of spot
cleanup, we can wait until all instances have been terminated.
"""
spot_reqs = list_spot_requests_for_stack(stack_name, boto3_session)
instances_to_del = []
requests_to_term = []
for req in spot_reqs:
if req["instanceId"] is not None:
instances_to_del.append(req["instanceId"])
else:
requests_to_term.append(req["id"])
ec2 = boto3_session.client("ec2")
if len(instances_to_del) > 0:
ec2.terminate_instances(InstanceIds=instances_to_del)
if len(requests_to_term) > 0:
ec2.cancel_spot_instance_requests(SpotInstanceRequestIds=requests_to_term)
return instances_to_del
def get_sqs_queue_attributes(
boto3_session: boto3.session.Session,
queue_urls: List[str],
) -> List[Tuple[str, Any]]:
"""
Iterates over all SQS queues. Returns a dict with url as key, and attributes as value.
"""
client = boto3_session.client('sqs')
queue_attributes = []
for queue_url in queue_urls:
try:
response = client.get_queue_attributes(QueueUrl=queue_url,
AttributeNames=['All'])
except ClientError as e:
if e.response['Error'][
'Code'] == 'AWS.SimpleQueueService.NonExistentQueue':
logger.warning(
f"Failed to retrieve SQS queue {queue_url} - Queue does not exist error"
)
continue
else:
raise
queue_attributes.append((queue_url, response['Attributes']))
return queue_attributes
def task_role_arn(session: boto3.session.Session) -> [str, bool]:
"""This is used for both the original deployed service and the copy that cloud-debug creates."""
iam = session.client("iam")
try:
resp = iam.get_role(DEBUG_TASK_ROLE_NAME)
role_arn = resp['Role']['Arn']
print("[INFO] Found existing cloud-debug role {}".format(role_arn))
return role_arn, False
except botocore.errorfactory.NoSuchEntityException:
pass
policy_doc = '''
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ecs-tasks.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
'''
#TODO: if this fails look for a role that has the right permissions that we can reuse
resp = iam.create_role(RoleName=DEBUG_TASK_ROLE_NAME,
AssumeRolePolicyDocument=policy_doc)
iam.attach_role_policy(
RoleName=DEBUG_TASK_ROLE_NAME,
PolicyArn='arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore')
return resp['Role']['Arn'], True
def get_s3_bucket_list(boto3_session: boto3.session.Session) -> List[Dict]:
client = boto3_session.client('s3')
# NOTE no paginator available for this operation
buckets = client.list_buckets()
for bucket in buckets['Buckets']:
try:
bucket['Region'] = client.get_bucket_location(
Bucket=bucket['Name'])['LocationConstraint']
except ClientError as e:
if "AccessDenied" in e.args[0]:
# If we don't have perms to call get_bucket_location(), set region to None and keep going
logger.warning(
"get_bucket_location(bucket='{}') AccessDenied, skipping.".
format(bucket['Name']))
bucket['Region'] = None
continue
elif "NoSuchBucket" in e.args[0]:
logger.warning(
"get_bucket_location({}) threw NoSuchBucket exception, skipping"
.format(bucket['Name']))
bucket['Region'] = None
continue
elif "AllAccessDisabled" in e.args[0]:
logger.warning(
"get_bucket_location({}) failed - bucket is disabled, skipping"
.format(bucket['Name']))
else:
raise
return buckets
def get_eks_clusters(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('eks', region_name=region)
clusters: List[Dict] = []
paginator = client.get_paginator('list_clusters')
for page in paginator.paginate():
clusters.extend(page['clusters'])
return clusters
def delete(stack_name: str, boto3_session: boto3.session.Session) -> None:
ec2 = boto3_session.client("ec2")
# First, shut down the master so no new agents are started.
stack_output = get_output(stack_name, boto3_session)
master_id = stack_output[constants.cloudformation.MASTER_ID]
describe_instance_response = ec2.describe_instances(
Filters=[{"Name": "instance-id", "Values": [master_id]}],
)
if describe_instance_response["Reservations"]:
print("Stopping Master Instance")
stop_master(master_id, boto3_session)
# Second, terminate the agents so nothing can write to the checkpoint bucket. We create agent
# instances outside of cloudformation, so we have to manually terminate them.
print("Terminating Running Agents")
terminate_running_agents(stack_output[constants.cloudformation.AGENT_TAG_NAME], boto3_session)
print("Agents Terminated")
# Third, empty the bucket that was created for this stack.
bucket_name = get_output(stack_name, boto3_session).get(
constants.cloudformation.CHECKPOINT_BUCKET
)
if bucket_name:
print("Emptying Checkpoint Bucket")
empty_bucket(bucket_name, boto3_session)
print("Checkpoint Bucket Empty")
delete_stack(stack_name, boto3_session)
def get_images(boto3_session: boto3.session.Session, region: str,
image_ids: List[str]) -> List[Dict]:
client = boto3_session.client('ec2',
region_name=region,
config=get_botocore_config())
images = []
try:
self_images = client.describe_images(Owners=['self'])['Images']
images.extend(self_images)
except ClientError as e:
logger.warning(
f"Failed retrieve images for region - {region}. Error - {e}")
try:
if image_ids:
images_in_use = client.describe_images(
ImageIds=image_ids)['Images']
# Ensure we're not adding duplicates
_ids = [image["ImageId"] for image in images]
for image in images_in_use:
if image["ImageId"] not in _ids:
images.append(image)
except ClientError as e:
logger.warning(
f"Failed retrieve images for region - {region}. Error - {e}")
return images
def get_ec2_info(instance_id: str, boto3_session: boto3.session.Session) -> Dict:
ec2 = boto3_session.client("ec2")
response = ec2.describe_instances(InstanceIds=[instance_id])
info = response["Reservations"][0]["Instances"][0]
assert isinstance(info, dict), f"expected a dict of instance info but got {info}"
return info
def terminate_running_agents(agent_tag_name: str, boto3_session: boto3.session.Session) -> None:
ec2 = boto3_session.client("ec2")
waiter = ec2.get_waiter("instance_terminated")
response = ec2.describe_instances(
Filters=[
{"Name": "tag:Name", "Values": [agent_tag_name]},
{"Name": "instance-state-name", "Values": ["running", "pending"]},
]
)
reservations = response["Reservations"]
instance_ids = []
for reservation in reservations:
for instance in reservation["Instances"]:
instance_ids.append(instance["InstanceId"])
if instance_ids:
ec2.terminate_instances(InstanceIds=instance_ids)
for n in range(NUM_WAITS):
print("Waiting For Agents To Terminate")
try:
waiter.wait(InstanceIds=instance_ids, WaiterConfig={"Delay": 10})
break
except WaiterError as e:
if n == NUM_WAITS - 1:
raise e
def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config())
paginator = client.get_paginator('describe_auto_scaling_groups')
asgs: List[Dict] = []
for page in paginator.paginate():
asgs.extend(page['AutoScalingGroups'])
return asgs
def create_stack(
stack_name: str,
template_body: str,
boto3_session: boto3.session.Session,
parameters: Optional[List] = None,
) -> None:
print(
f"Creating stack {stack_name}. This may take a few minutes... "
f"Check the CloudFormation Console for updates"
)
cfn = boto3_session.client("cloudformation")
create_waiter = cfn.get_waiter("stack_create_complete")
if parameters:
cfn.create_stack(
StackName=stack_name,
TemplateBody=template_body,
Parameters=parameters,
Capabilities=["CAPABILITY_IAM"],
)
else:
cfn.create_stack(
StackName=stack_name, TemplateBody=template_body, Capabilities=["CAPABILITY_IAM"]
)
create_waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10})
def _autodiscover_accounts(
neo4j_session: neo4j.Session,
boto3_session: boto3.session.Session,
account_id: str,
sync_tag: int,
common_job_parameters: Dict,
) -> None:
logger.info("Trying to autodiscover accounts.")
try:
# Fetch all accounts
client = boto3_session.client('organizations')
paginator = client.get_paginator('list_accounts')
accounts: List[Dict] = []
for page in paginator.paginate():
accounts.extend(page['Accounts'])
# Filter out every account which is not in the ACTIVE status
# and select only the Id and Name fields
filtered_accounts: Dict[str, str] = {
x['Name']: x['Id']
for x in accounts if x['Status'] == 'ACTIVE'
}
# Add them to the graph
logger.info("Loading autodiscovered accounts.")
organizations.load_aws_accounts(neo4j_session, filtered_accounts,
sync_tag, common_job_parameters)
except botocore.exceptions.ClientError:
logger.warning(
f"The current account ({account_id}) doesn't have enough permissions to perform autodiscovery."
)
def get_ecs_cluster_arns(boto3_session: boto3.session.Session, region: str) -> List[str]:
client = boto3_session.client('ecs', region_name=region)
paginator = client.get_paginator('list_clusters')
cluster_arns: List[str] = []
for page in paginator.paginate():
cluster_arns.extend(page.get('clusterArns', []))
return cluster_arns
def get_apigateway_rest_apis(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('apigateway', region_name=region)
paginator = client.get_paginator('get_rest_apis')
apis: List[Any] = []
for page in paginator.paginate():
apis.extend(page['items'])
return apis
def get_snapshots(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('ec2', region_name=region)
paginator = client.get_paginator('describe_snapshots')
snapshots: List[Dict] = []
for page in paginator.paginate():
snapshots.extend(page['Snapshots'])
return snapshots
def get_sqs_queue_list(boto3_session: boto3.session.Session, region: str) -> List[str]:
client = boto3_session.client('sqs', region_name=region)
paginator = client.get_paginator('list_queues')
queues: List[Any] = []
for page in paginator.paginate():
queues.extend(page.get('QueueUrls', []))
return queues
def get_ecr_repository_images(boto3_session: boto3.session.Session, region: str, repository_name: str) -> List[Dict]:
logger.debug("Getting ECR images in repository '%s' for region '%s'.", repository_name, region)
client = boto3_session.client('ecr', region_name=region)
paginator = client.get_paginator('list_images')
ecr_repository_images: List[Dict] = []
for page in paginator.paginate(repositoryName=repository_name):
ecr_repository_images.extend(page['imageIds'])
return ecr_repository_images
def get_dynamodb_tables(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
client = boto3_session.client('dynamodb', region_name=region)
paginator = client.get_paginator('list_tables')
dynamodb_tables = []
for page in paginator.paginate():
for table_name in page['TableNames']:
dynamodb_tables.append(client.describe_table(TableName=table_name))
return dynamodb_tables
def get_ecr_repositories(boto3_session: boto3.session.Session, region: str) -> List[Dict]:
logger.info("Getting ECR repositories for region '%s'.", region)
client = boto3_session.client('ecr', region_name=region)
paginator = client.get_paginator('describe_repositories')
ecr_repositories: List[Dict] = []
for page in paginator.paginate():
ecr_repositories.extend(page['repositories'])
return ecr_repositories
