def test_current_table(current_s3_table):
from historical.s3.models import CurrentS3Model
CurrentS3Model(**S3_BUCKET).save()
items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1'))
assert len(items) == 1
assert isinstance(items[0].ttl, int)
assert items[0].ttl > 0
def test_current_table(current_s3_table): # pylint: disable=W0613
"""Tests for the Current PynamoDB model."""
from historical.s3.models import CurrentS3Model
CurrentS3Model(**S3_BUCKET).save()
items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1'))
assert len(items) == 1
assert isinstance(items[0].ttl, int)
assert items[0].ttl > 0
def test_serialization():
"""Tests that the dictionary serialization for PynamoDB objects works properly."""
from historical.s3.models import CurrentS3Model
bucket = S3_BUCKET.copy()
bucket['eventTime'] = datetime(
year=2017, month=5, day=12, hour=10, minute=30,
second=0).isoformat() + 'Z'
bucket = CurrentS3Model(**bucket)
dictionary = dict(bucket)
assert dictionary['version'] == VERSION
assert dictionary['configuration']['LifecycleRules'][0]['Prefix'] is None
def create_delete_model(record):
"""Create an S3 model from a record."""
arn = "arn:aws:s3:::{}".format(cloudwatch.filter_request_parameters('bucketName', record))
log.debug('[-] Deleting Dynamodb Records. Hash Key: {arn}'.format(arn=arn))
data = {
'arn': arn,
'principalId': cloudwatch.get_principal(record),
'userIdentity': cloudwatch.get_user_identity(record),
'accountId': record['account'],
'eventTime': record['detail']['eventTime'],
'BucketName': cloudwatch.filter_request_parameters('bucketName', record),
'Region': cloudwatch.get_region(record),
'Tags': {},
'configuration': {},
'eventSource': record["detail"]["eventSource"]
}
return CurrentS3Model(**data)
def historical_table(current_s3_table):
for x in range(0, 10):
bucket = json.loads(S3_BUCKET.replace("{number}", "{}".format(x)))
CurrentS3Model(**bucket).save()
def process_update_records(update_records):
"""Process the requests for S3 bucket update requests"""
events = sorted(update_records, key=lambda x: x['account'])
# Group records by account for more efficient processing
for account_id, events in groupby(events, lambda x: x['account']):
events = list(events)
# Grab the bucket names (de-dupe events):
buckets = {}
for e in events:
# If the creation date is present, then use it:
bucket_event = buckets.get(e["detail"]["requestParameters"]["bucketName"], {
"creationDate": e["detail"]["requestParameters"].get("creationDate")
})
bucket_event.update(e["detail"]["requestParameters"])
buckets[e["detail"]["requestParameters"]["bucketName"]] = bucket_event
buckets[e["detail"]["requestParameters"]["bucketName"]]["eventDetails"] = e
# Query AWS for current configuration
for b, item in buckets.items():
log.debug("[~] Processing Create/Update for: {}".format(b))
# If the bucket does not exist, then simply drop the request --
# If this happens, there is likely a Delete event that has occurred and will be processed soon.
try:
bucket_details = get_bucket(b,
account_number=account_id,
include_created=(item.get("creationDate") is None),
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION)
if bucket_details.get("Error"):
log.error("[X] Unable to fetch details about bucket: {}. "
"The error details are: {}".format(b, bucket_details["Error"]))
continue
except ClientError as ce:
if ce.response["Error"]["Code"] == "NoSuchBucket":
log.warning("[?] Received update request for bucket: {} that does not "
"currently exist. Skipping.".format(b))
continue
# Catch Access Denied exceptions as well:
if ce.response["Error"]["Code"] == "AccessDenied":
log.error("[X] Unable to fetch details for S3 Bucket: {} in {}. Access is Denied. Skipping...".format(
b, account_id
))
continue
raise Exception(ce)
# Pull out the fields we want:
data = {
"arn": "arn:aws:s3:::{}".format(b),
"principalId": cloudwatch.get_principal(item["eventDetails"]),
"userIdentity": cloudwatch.get_user_identity(item["eventDetails"]),
"accountId": account_id,
"eventTime": item["eventDetails"]["detail"]["eventTime"],
"BucketName": b,
"Region": bucket_details["Region"],
# Duplicated in top level and configuration for secondary index
"Tags": bucket_details["Tags"] or {},
"eventSource": item["eventDetails"]["detail"]["eventSource"]
}
# Remove the fields we don't care about:
del bucket_details["Arn"]
del bucket_details["GrantReferences"]
del bucket_details["Region"]
if not bucket_details.get("CreationDate"):
bucket_details["CreationDate"] = item["creationDate"]
data["configuration"] = bucket_details
current_revision = CurrentS3Model(**data)
current_revision.save()
def process_update_records(update_records):
"""Process the requests for S3 bucket update requests"""
events = sorted(update_records, key=lambda x: x['account'])
# Group records by account for more efficient processing
for account_id, events in groupby(events, lambda x: x['account']):
events = list(events)
# Grab the bucket names (de-dupe events):
buckets = {}
for event in events:
# If the creation date is present, then use it:
bucket_event = buckets.get(
event['detail']['requestParameters']['bucketName'], {
'creationDate':
event['detail']['requestParameters'].get('creationDate')
})
bucket_event.update(event['detail']['requestParameters'])
buckets[event['detail']['requestParameters']
['bucketName']] = bucket_event
buckets[event['detail']['requestParameters']
['bucketName']]['eventDetails'] = event
# Query AWS for current configuration
for b_name, item in buckets.items():
LOG.debug(f'[~] Processing Create/Update for: {b_name}')
# If the bucket does not exist, then simply drop the request --
# If this happens, there is likely a Delete event that has occurred and will be processed soon.
try:
bucket_details = get_bucket(
b_name,
account_number=account_id,
include_created=(item.get('creationDate') is None),
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION)
if bucket_details.get('Error'):
LOG.error(
f"[X] Unable to fetch details about bucket: {b_name}. "
f"The error details are: {bucket_details['Error']}")
continue
except ClientError as cerr:
if cerr.response['Error']['Code'] == 'NoSuchBucket':
LOG.warning(
f'[?] Received update request for bucket: {b_name} that does not '
'currently exist. Skipping.')
continue
# Catch Access Denied exceptions as well:
if cerr.response['Error']['Code'] == 'AccessDenied':
LOG.error(
f'[X] Unable to fetch details for S3 Bucket: {b_name} in {account_id}. Access is Denied. '
'Skipping...')
continue
raise Exception(cerr)
# Pull out the fields we want:
data = {
'arn':
f'arn:aws:s3:::{b_name}',
'principalId':
cloudwatch.get_principal(item['eventDetails']),
'userIdentity':
cloudwatch.get_user_identity(item['eventDetails']),
'userAgent':
item['eventDetails']['detail'].get('userAgent'),
'sourceIpAddress':
item['eventDetails']['detail'].get('sourceIPAddress'),
'requestParameters':
item['eventDetails']['detail'].get('requestParameters'),
'accountId':
account_id,
'eventTime':
item['eventDetails']['detail']['eventTime'],
'BucketName':
b_name,
'Region':
bucket_details.pop('Region'),
# Duplicated in top level and configuration for secondary index
'Tags':
bucket_details.pop('Tags', {}) or {},
'eventSource':
item['eventDetails']['detail']['eventSource'],
'eventName':
item['eventDetails']['detail']['eventName'],
'version':
VERSION
}
# Remove the fields we don't care about:
del bucket_details['Arn']
del bucket_details['GrantReferences']
del bucket_details['_version']
del bucket_details['Name']
if not bucket_details.get('CreationDate'):
bucket_details['CreationDate'] = item['creationDate']
data['configuration'] = bucket_details
current_revision = CurrentS3Model(**data)
current_revision.save()