def create_inventory_destination_buckets(input_parameters):
""" Create inventory destination buckets
Arguments:
input_parameters {config.S3InsightsInput} -- Input parameters for the current execution
"""
template_file_path = utility.get_file_path(__file__, "template/inventory-destination.json")
with open(template_file_path, "r") as template_file:
template_text = template_file.read()
stacks = []
regions = input_parameters.supported_regions
for region in regions:
bucket_name = get_destination_bucket_name(
input_parameters.run_id,
region)
topic_name = utility.get_resource_name(
input_parameters.run_id,
'sns',
'notification-topic')
acceleration_status = 'Enabled'
parameters = [
{
'ParameterKey': 'BucketName',
'ParameterValue': bucket_name
},
{
'ParameterKey': 'SQSArn',
'ParameterValue': config.DeploymentDetails.sqs_arn
},
{
'ParameterKey': 'TopicName',
'ParameterValue': topic_name
},
{
'ParameterKey': 'AccelerationStatus',
'ParameterValue': acceleration_status
}
]
stack_name = get_stack_name(input_parameters.run_id)
cloudformation_client = awshelper.get_client(
awshelper.ServiceName.cloudformation,
region)
response = cloudformation_client.create_stack(
StackName=stack_name,
TemplateBody=template_text,
Parameters=parameters)
logging.info(f'create stack response: {response}')
stacks.append(StackDetails(cloudformation_client, stack_name))
wait_for_stack_operations_to_finish(
stacks,
'create_in_progress',
'create_complete',
20)
def get_stack_name(run_id):
""" Get CloudFormation stack name
Arguments:
run_id {string} -- run_id for the current Step Function execution
Returns:
string -- CloudFormation Stack name
"""
return utility.get_resource_name(
run_id,
'stack',
'dest-resources')
def get_destination_bucket_name(run_id, region):
""" Get destination bucket name for a region
Arguments:
run_id {string} -- run_id for the current execution
region {string} -- AWS region name
Returns:
string -- Destination bucket name
"""
host_account_id = awshelper.SessionManager.get_host_account_id()
return utility.get_resource_name(
run_id,
f'dest-{host_account_id}',
region)
def remove_bucket_inventory_configuration_internal(s3_client, run_id, account_id, region, bucket_name):
""" Helper function for removing nventory configuration from the given S3 bucket
Arguments:
s3_client {boto3.S3.Client} -- Boto3 S3 client
run_id {string} -- run_id for the current Step Function execution
account_id {string} -- AWS account id
region {string} -- AWS region name
bucket_name {string} -- Bucket name
"""
try:
id = utility.get_resource_name(run_id, 's3-inventory', 'orc')
response = s3_client.delete_bucket_inventory_configuration(
Bucket=bucket_name,
Id=id)
logging.info(f'delete bucket inventory response for {account_id}:{region}:{bucket_name} = {response}')
except ClientError as e:
logging.error(f'error while deleting inventory configuration from {account_id}:{region}:{bucket_name}. error details:{e}')
def create_bucket_inventory_configurations(run_id, source_buckets):
""" Enable S3 inventory for the given list of source buckets
Arguments:
run_id {string} -- run_id for the current Step Function execution
source_buckets {dict<string, dict<string, list(string)>>} -- Source buckets
"""
host_account_id = awshelper.SessionManager.get_host_account_id()
for account_id in source_buckets:
for region in source_buckets[account_id]:
s3_resource = awshelper.get_resource(awshelper.ServiceName.s3, account_id, run_id)
s3_client = awshelper.get_client(
awshelper.ServiceName.s3,
region,
account_id,
run_id)
for bucket_name in source_buckets[account_id][region]:
logging.info(f'Processing {bucket_name} in {region} from {account_id}')
is_empty, client_error = is_bucket_empty(s3_resource, bucket_name)
if client_error is None:
if is_empty:
# Update DB status
logging.info(f'{bucket_name} in {region} from {account_id} is empty')
ddb.update_source_bucket_inventory_status(bucket_name, ddb.BucketInventoryStatus.bucket_is_empty)
else:
destination_prefix = account_id + "/" + region
destination_bucket = "arn:aws:s3:::" + get_destination_bucket_name(run_id, region)
inventory_id = utility.get_resource_name(run_id, 's3-inventory', 'orc')
inventory_configuration_orc = {
"Schedule": {
"Frequency": "Daily"
},
"IsEnabled": True,
"Destination": {
"S3BucketDestination": {
"Prefix": destination_prefix,
"Format": "ORC",
"Bucket": destination_bucket,
"AccountId": host_account_id
}
},
"OptionalFields": [
"Size",
"LastModifiedDate",
"StorageClass",
"ETag",
"ReplicationStatus",
"IsMultipartUploaded",
"EncryptionStatus",
"ObjectLockMode",
"ObjectLockRetainUntilDate",
"ObjectLockLegalHoldStatus"
],
"IncludedObjectVersions": "All",
"Id": inventory_id
}
try:
response = s3_client.put_bucket_inventory_configuration(
Bucket=bucket_name,
Id=inventory_id,
InventoryConfiguration=inventory_configuration_orc)
logging.info(f'put bucket inventory configuration response:{response}')
ddb.update_source_bucket_inventory_status(bucket_name, ddb.BucketInventoryStatus.in_progress)
except ClientError as e:
logging.error(f'error while creating inventory configuration on {account_id}:{region}:{bucket_name}. error details:{e}')