def test_current_table(current_s3_table): from historical.s3.models import CurrentS3Model CurrentS3Model(**S3_BUCKET).save() items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1')) assert len(items) == 1 assert isinstance(items[0].ttl, int) assert items[0].ttl > 0
def test_current_table(current_s3_table): # pylint: disable=W0613 """Tests for the Current PynamoDB model.""" from historical.s3.models import CurrentS3Model CurrentS3Model(**S3_BUCKET).save() items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1')) assert len(items) == 1 assert isinstance(items[0].ttl, int) assert items[0].ttl > 0
def test_serialization(): """Tests that the dictionary serialization for PynamoDB objects works properly.""" from historical.s3.models import CurrentS3Model bucket = S3_BUCKET.copy() bucket['eventTime'] = datetime( year=2017, month=5, day=12, hour=10, minute=30, second=0).isoformat() + 'Z' bucket = CurrentS3Model(**bucket) dictionary = dict(bucket) assert dictionary['version'] == VERSION assert dictionary['configuration']['LifecycleRules'][0]['Prefix'] is None
def create_delete_model(record): """Create an S3 model from a record.""" arn = "arn:aws:s3:::{}".format(cloudwatch.filter_request_parameters('bucketName', record)) log.debug('[-] Deleting Dynamodb Records. Hash Key: {arn}'.format(arn=arn)) data = { 'arn': arn, 'principalId': cloudwatch.get_principal(record), 'userIdentity': cloudwatch.get_user_identity(record), 'accountId': record['account'], 'eventTime': record['detail']['eventTime'], 'BucketName': cloudwatch.filter_request_parameters('bucketName', record), 'Region': cloudwatch.get_region(record), 'Tags': {}, 'configuration': {}, 'eventSource': record["detail"]["eventSource"] } return CurrentS3Model(**data)
def historical_table(current_s3_table): for x in range(0, 10): bucket = json.loads(S3_BUCKET.replace("{number}", "{}".format(x))) CurrentS3Model(**bucket).save()
def process_update_records(update_records): """Process the requests for S3 bucket update requests""" events = sorted(update_records, key=lambda x: x['account']) # Group records by account for more efficient processing for account_id, events in groupby(events, lambda x: x['account']): events = list(events) # Grab the bucket names (de-dupe events): buckets = {} for e in events: # If the creation date is present, then use it: bucket_event = buckets.get(e["detail"]["requestParameters"]["bucketName"], { "creationDate": e["detail"]["requestParameters"].get("creationDate") }) bucket_event.update(e["detail"]["requestParameters"]) buckets[e["detail"]["requestParameters"]["bucketName"]] = bucket_event buckets[e["detail"]["requestParameters"]["bucketName"]]["eventDetails"] = e # Query AWS for current configuration for b, item in buckets.items(): log.debug("[~] Processing Create/Update for: {}".format(b)) # If the bucket does not exist, then simply drop the request -- # If this happens, there is likely a Delete event that has occurred and will be processed soon. try: bucket_details = get_bucket(b, account_number=account_id, include_created=(item.get("creationDate") is None), assume_role=HISTORICAL_ROLE, region=CURRENT_REGION) if bucket_details.get("Error"): log.error("[X] Unable to fetch details about bucket: {}. " "The error details are: {}".format(b, bucket_details["Error"])) continue except ClientError as ce: if ce.response["Error"]["Code"] == "NoSuchBucket": log.warning("[?] Received update request for bucket: {} that does not " "currently exist. Skipping.".format(b)) continue # Catch Access Denied exceptions as well: if ce.response["Error"]["Code"] == "AccessDenied": log.error("[X] Unable to fetch details for S3 Bucket: {} in {}. Access is Denied. Skipping...".format( b, account_id )) continue raise Exception(ce) # Pull out the fields we want: data = { "arn": "arn:aws:s3:::{}".format(b), "principalId": cloudwatch.get_principal(item["eventDetails"]), "userIdentity": cloudwatch.get_user_identity(item["eventDetails"]), "accountId": account_id, "eventTime": item["eventDetails"]["detail"]["eventTime"], "BucketName": b, "Region": bucket_details["Region"], # Duplicated in top level and configuration for secondary index "Tags": bucket_details["Tags"] or {}, "eventSource": item["eventDetails"]["detail"]["eventSource"] } # Remove the fields we don't care about: del bucket_details["Arn"] del bucket_details["GrantReferences"] del bucket_details["Region"] if not bucket_details.get("CreationDate"): bucket_details["CreationDate"] = item["creationDate"] data["configuration"] = bucket_details current_revision = CurrentS3Model(**data) current_revision.save()
def process_update_records(update_records): """Process the requests for S3 bucket update requests""" events = sorted(update_records, key=lambda x: x['account']) # Group records by account for more efficient processing for account_id, events in groupby(events, lambda x: x['account']): events = list(events) # Grab the bucket names (de-dupe events): buckets = {} for event in events: # If the creation date is present, then use it: bucket_event = buckets.get( event['detail']['requestParameters']['bucketName'], { 'creationDate': event['detail']['requestParameters'].get('creationDate') }) bucket_event.update(event['detail']['requestParameters']) buckets[event['detail']['requestParameters'] ['bucketName']] = bucket_event buckets[event['detail']['requestParameters'] ['bucketName']]['eventDetails'] = event # Query AWS for current configuration for b_name, item in buckets.items(): LOG.debug(f'[~] Processing Create/Update for: {b_name}') # If the bucket does not exist, then simply drop the request -- # If this happens, there is likely a Delete event that has occurred and will be processed soon. try: bucket_details = get_bucket( b_name, account_number=account_id, include_created=(item.get('creationDate') is None), assume_role=HISTORICAL_ROLE, region=CURRENT_REGION) if bucket_details.get('Error'): LOG.error( f"[X] Unable to fetch details about bucket: {b_name}. " f"The error details are: {bucket_details['Error']}") continue except ClientError as cerr: if cerr.response['Error']['Code'] == 'NoSuchBucket': LOG.warning( f'[?] Received update request for bucket: {b_name} that does not ' 'currently exist. Skipping.') continue # Catch Access Denied exceptions as well: if cerr.response['Error']['Code'] == 'AccessDenied': LOG.error( f'[X] Unable to fetch details for S3 Bucket: {b_name} in {account_id}. Access is Denied. ' 'Skipping...') continue raise Exception(cerr) # Pull out the fields we want: data = { 'arn': f'arn:aws:s3:::{b_name}', 'principalId': cloudwatch.get_principal(item['eventDetails']), 'userIdentity': cloudwatch.get_user_identity(item['eventDetails']), 'userAgent': item['eventDetails']['detail'].get('userAgent'), 'sourceIpAddress': item['eventDetails']['detail'].get('sourceIPAddress'), 'requestParameters': item['eventDetails']['detail'].get('requestParameters'), 'accountId': account_id, 'eventTime': item['eventDetails']['detail']['eventTime'], 'BucketName': b_name, 'Region': bucket_details.pop('Region'), # Duplicated in top level and configuration for secondary index 'Tags': bucket_details.pop('Tags', {}) or {}, 'eventSource': item['eventDetails']['detail']['eventSource'], 'eventName': item['eventDetails']['detail']['eventName'], 'version': VERSION } # Remove the fields we don't care about: del bucket_details['Arn'] del bucket_details['GrantReferences'] del bucket_details['_version'] del bucket_details['Name'] if not bucket_details.get('CreationDate'): bucket_details['CreationDate'] = item['creationDate'] data['configuration'] = bucket_details current_revision = CurrentS3Model(**data) current_revision.save()