def get_load_balancer_v2_target_groups(client: botocore.client.BaseClient, load_balancer_arn: str) -> List[Dict]:
paginator = client.get_paginator('describe_target_groups')
target_groups: List[Dict] = []
for page in paginator.paginate(LoadBalancerArn=load_balancer_arn):
target_groups.extend(page['TargetGroups'])
# Add instance data
for target_group in target_groups:
target_group['Targets'] = []
target_health = client.describe_target_health(TargetGroupArn=target_group['TargetGroupArn'])
for target_health_description in target_health['TargetHealthDescriptions']:
target_group['Targets'].append(target_health_description['Target']['Id'])
return target_groups
def get_policy(bucket: Dict, client: botocore.client.BaseClient) -> str:
"""
Gets the S3 bucket policy. Returns policy string or None if no policy
"""
try:
policy = client.get_bucket_policy(Bucket=bucket['Name'])
except ClientError as e:
# no policy is defined for this bucket
if "NoSuchBucketPolicy" in e.args[0]:
logger.warning(
"get_bucket_policy({}) failed because the bucket no longer exists"
.format(bucket['Name']))
policy = None
elif "AccessDenied" in e.args[0]:
logger.warning(
"Access denied trying to retrieve S3 bucket {} policy".format(
bucket['Name']))
policy = None
elif "NoSuchBucket" in e.args[0]:
logger.warning(
"get_bucket_policy({}) threw NoSuchBucket exception, skipping".
format(bucket['Name']))
policy = None
elif "AllAccessDisabled" in e.args[0]:
# Catches the following error : "An error occurred (AllAccessDisabled) when calling the
# GetBucketAcl operation: All access to this object has been disabled"
logger.warning(
"Failed to retrieve S3 bucket {} policies - Bucket is disabled"
.format(bucket['Name']))
policy = None
else:
raise
return policy
def get_load_balancer_v2_listeners(client: botocore.client.BaseClient, load_balancer_arn: str) -> List[Dict]:
paginator = client.get_paginator('describe_listeners')
listeners: List[Dict] = []
for page in paginator.paginate(LoadBalancerArn=load_balancer_arn):
listeners.extend(page['Listeners'])
return listeners
def get_acl(bucket: Dict, client: botocore.client.BaseClient) -> Optional[str]:
"""
Gets the S3 bucket ACL. Returns ACL string
"""
try:
acl = client.get_bucket_acl(Bucket=bucket['Name'])
except ClientError as e:
if "AccessDenied" in e.args[0]:
logger.warning(
"Failed to retrieve S3 bucket {} ACL - Access Denied".format(
bucket['Name']))
return None
elif "NoSuchBucket" in e.args[0]:
logger.warning(
"Failed to retrieve S3 bucket {} ACL - No Such Bucket".format(
bucket['Name']))
return None
elif "AllAccessDisabled" in e.args[0]:
logger.warning(
"Failed to retrieve S3 bucket {} ACL - Bucket is disabled".
format(bucket['Name']))
return None
elif "EndpointConnectionError" in e.args[0]:
logger.warning(
"Failed to retrieve S3 bucket {} ACL - EndpointConnectionError"
.format(bucket['Name']))
return None
else:
raise
return acl
def full_results(
client: botocore.client.BaseClient,
method: str,
args: List[str],
kwargs: Dict[str, Any],
) -> Dict[str, Any]:
"""Returns JSON results for an AWS botocore call. Flattens paginated results (if any)."""
if client.can_paginate(method):
paginator = client.get_paginator(method)
full_result: Dict[str, Any] = paginator.paginate(
*args, **kwargs
).build_full_result()
return full_result
else:
single_result: Dict[str, Any] = getattr(client, method)(*args, **kwargs)
return single_result
def get_zone_record_sets(client: botocore.client.BaseClient,
zone_id: str) -> List[Dict]:
resource_record_sets: List[Dict] = []
paginator = client.get_paginator('list_resource_record_sets')
pages = paginator.paginate(HostedZoneId=zone_id)
for page in pages:
resource_record_sets.extend(page['ResourceRecordSets'])
return resource_record_sets
def get_event_source_mappings(lambda_function: Dict,
client: botocore.client.BaseClient) -> List[Any]:
event_source_mappings: List[Any] = []
paginator = client.get_paginator('list_event_source_mappings')
for page in paginator.paginate(
FunctionName=lambda_function['FunctionName']):
event_source_mappings.extend(page['EventSourceMappings'])
return event_source_mappings
def get_function_aliases(lambda_function: Dict,
client: botocore.client.BaseClient) -> List[Any]:
aliases: List[Any] = []
paginator = client.get_paginator('list_aliases')
for page in paginator.paginate(
FunctionName=lambda_function['FunctionName']):
aliases.extend(page['Aliases'])
return aliases
def get_grants(key: str, client: botocore.client.BaseClient) -> List[Any]:
"""
Gets the KMS Key Grants.
"""
grants: List[Any] = []
paginator = client.get_paginator('list_grants')
for page in paginator.paginate(KeyId=key['KeyId']): # type: ignore
grants.extend(page['Grants'])
return grants
def get_aliases(key: Dict, client: botocore.client.BaseClient) -> List[Any]:
"""
Gets the KMS Key Aliases.
"""
aliases: List[Any] = []
paginator = client.get_paginator('list_aliases')
for page in paginator.paginate(KeyId=key['KeyId']):
aliases.extend(page['Aliases'])
return aliases
def get_rest_api_resources(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
"""
Gets the collection of Resource resources.
"""
resources: List[Any] = []
paginator = client.get_paginator('get_resources')
response_iterator = paginator.paginate(restApiId=api['id'])
for page in response_iterator:
resources.extend(page['items'])
return resources
def fetch_application_lbs(s3: botocore.client.BaseClient, bucket_name: str,
application: str) -> List[Tuple[str, str]]:
try:
response = s3.get_object(Bucket=bucket_name, Key=application)
except botocore.exceptions.ClientError:
print("Failed to fetch", application, "from S3")
raise
lines = [l.split(" ") for l in sorted(response["Body"].read().split("\n"))]
return [(lb[0], lb[1]) for lb in lines]
def fetch_current_lb_config(s3: botocore.client.BaseClient, bucket_name: str,
application: Application) -> str:
try:
response = s3.get_object(Bucket=bucket_name, Key=application)
except botocore.exceptions.ClientError:
print("Failed to fetch currently registered upstreams for",
application)
raise
lines = [l.split(" ") for l in sorted(response["Body"].read().split("\n"))]
return [(lb[0], lb[1]) for lb in lines]
def get_rest_api_stages(api: Dict, client: botocore.client.BaseClient) -> List[Any]:
"""
Gets the REST API Stage Resources.
"""
try:
stages = client.get_stages(restApiId=api['id'])
except ClientError as e:
logger.warning(f'Failed to retrieve Stages for Api Id - {api["id"]} - {e}')
raise
return stages['item']
def get_policy(key: Dict, client: botocore.client.BaseClient) -> Any:
"""
Gets the KMS Key policy. Returns policy string or None if we are unable to retrieve it.
"""
try:
policy = client.get_key_policy(KeyId=key["KeyId"], PolicyName='default')
except ClientError:
policy = None
logger.warning(f"Failed to retrieve Key Policy for key id - {key['KeyId']}, skipping.", exc_info=True)
return policy
def get_acl(bucket: Dict, client: botocore.client.BaseClient) -> Optional[str]:
"""
Gets the S3 bucket ACL. Returns ACL string
"""
try:
acl = client.get_bucket_acl(Bucket=bucket['Name'])
except ClientError as e:
if _is_common_exception(e, bucket):
return None
else:
raise
return acl
def get_zones(
client: botocore.client.BaseClient) -> List[Tuple[Dict, List[Dict]]]:
paginator = client.get_paginator('list_hosted_zones')
hosted_zones: List[Dict] = []
for page in paginator.paginate():
hosted_zones.extend(page['HostedZones'])
results: List[Tuple[Dict, List[Dict]]] = []
for hosted_zone in hosted_zones:
record_sets = get_zone_record_sets(client, hosted_zone['Id'])
results.append((hosted_zone, record_sets))
return results
def get_encryption(bucket: Dict, client: botocore.client.BaseClient) -> Optional[str]:
"""
Gets the S3 bucket default encryption configuration. Returns encryption configuration or None if not enabled
"""
try:
encryption = client.get_bucket_encryption(Bucket=bucket['Name'])
except ClientError as e:
if "ServerSideEncryptionConfigurationNotFoundError" in e.args[0]:
return None
elif _is_common_exception(e, bucket):
return None
else:
raise
return encryption
def get_policy(key: Dict, client: botocore.client.BaseClient) -> Any:
"""
Gets the KMS Key policy. Returns policy string or None if no policy
"""
try:
policy = client.get_key_policy(KeyId=key["KeyId"],
PolicyName='default')
except ClientError as e:
policy = None
logger.warning(
"Failed to retrieve Key Policy for key id - {}. Error - {}".format(
key["KeyId"], e))
raise
return policy
def get_policy(bucket: Dict, client: botocore.client.BaseClient) -> str:
"""
Gets the S3 bucket policy. Returns policy string or None if no policy
"""
try:
policy = client.get_bucket_policy(Bucket=bucket['Name'])
except ClientError as e:
# no policy is defined for this bucket
if "NoSuchBucketPolicy" in e.args[0]:
policy = None
elif _is_common_exception(e, bucket):
policy = None
else:
raise
return policy
def scan_s3_files(s3: botocore.client.BaseClient, bucket: str, prefix: str):
print(f"Start scanning! - S3 Path: {bucket}/{prefix}")
paginator: botocore.paginate.Paginator = s3.get_paginator("list_objects_v2")
pages: botocore.paginate.PageIterator = paginator.paginate(Bucket=bucket, Prefix=prefix)
path_key_list = list()
for idx, page in enumerate(pages):
print(f"Page: {idx}")
sum_val = 0
if page.get("Contents"):
for obj in page["Contents"]:
sum_val += 1
path_key_list.append(obj["Key"])
print(f"Full path {bucket}/{obj['Key']}")
print(f"Files are successfully scanned!")
return path_key_list
def get_policy(key: Dict, client: botocore.client.BaseClient) -> Any:
"""
Gets the KMS Key policy. Returns policy string or None if we are unable to retrieve it.
"""
try:
policy = client.get_key_policy(KeyId=key["KeyId"], PolicyName='default')
except ClientError as e:
policy = None
if e.response['Error']['Code'] == 'AccessDeniedException':
logger.warning(
f"kms:get_key_policy on key id {key['KeyId']} failed with AccessDeniedException; continuing sync.",
exc_info=True,
)
else:
raise
return policy
def get_rest_api_client_certificate(stages: Dict, client: botocore.client.BaseClient) -> Optional[Any]:
"""
Gets the current ClientCertificate resource if present, else returns None.
"""
response = None
for stage in stages:
if 'clientCertificateId' in stage:
try:
response = client.get_client_certificate(clientCertificateId=stage['clientCertificateId'])
response['stageName'] = stage['stageName']
except ClientError as e:
logger.warning(f"Failed to retrive Client Certificate for Stage {stage['stageName']} - {e}")
raise
else:
return []
return response
def get_grants(key: Dict, client: botocore.client.BaseClient) -> List[Any]:
"""
Gets the KMS Key Grants.
"""
grants: List[Any] = []
paginator = client.get_paginator('list_grants')
try:
for page in paginator.paginate(KeyId=key['KeyId']):
grants.extend(page['Grants'])
except ClientError as e:
if e.response['Error']['Code'] == 'AccessDeniedException':
logger.warning(
f'kms:list_grants on key_id {key["KeyId"]} failed with AccessDeniedException; continuing sync.',
exc_info=True,
)
else:
raise
return grants
def get_encryption(bucket: Dict,
client: botocore.client.BaseClient) -> Optional[Dict]:
"""
Gets the S3 bucket default encryption configuration.
"""
encryption = None
try:
encryption = client.get_bucket_encryption(Bucket=bucket['Name'])
except ClientError as e:
if _is_common_exception(e, bucket):
pass
else:
raise
except EndpointConnectionError:
logger.warning(
f"Failed to retrieve S3 bucket encryption for {bucket['Name']} - Could not connect to the endpoint URL",
)
return encryption
def get_acl(bucket: Dict,
client: botocore.client.BaseClient) -> Optional[Dict]:
"""
Gets the S3 bucket ACL.
"""
acl = None
try:
acl = client.get_bucket_acl(Bucket=bucket['Name'])
except ClientError as e:
if _is_common_exception(e, bucket):
pass
else:
raise
except EndpointConnectionError:
logger.warning(
f"Failed to retrieve S3 bucket ACL for {bucket['Name']} - Could not connect to the endpoint URL",
)
return acl
def get_public_access_block(
bucket: Dict, client: botocore.client.BaseClient) -> Optional[Dict]:
"""
Gets the S3 bucket public access block configuration.
"""
public_access_block = None
try:
public_access_block = client.get_public_access_block(
Bucket=bucket['Name'])
except ClientError as e:
if _is_common_exception(e, bucket):
pass
else:
raise
except EndpointConnectionError:
logger.warning(
f"Failed to retrieve S3 bucket public access block for {bucket['Name']}"
" - Could not connect to the endpoint URL", )
return public_access_block
def get_latest_partition(s3: botocore.client.BaseClient, bucket: str, prefix: str):
result: object = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/")
partition_dts: list = [o.get("Prefix") for o in result.get("CommonPrefixes")]
return max(partition_dts)
