Exemplo n.º 1
0
def test_current_table(current_s3_table):
    from historical.s3.models import CurrentS3Model

    CurrentS3Model(**S3_BUCKET).save()

    items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1'))

    assert len(items) == 1
    assert isinstance(items[0].ttl, int)
    assert items[0].ttl > 0
Exemplo n.º 2
0
def test_current_table(current_s3_table):  # pylint: disable=W0613
    """Tests for the Current PynamoDB model."""
    from historical.s3.models import CurrentS3Model

    CurrentS3Model(**S3_BUCKET).save()

    items = list(CurrentS3Model.query('arn:aws:s3:::testbucket1'))

    assert len(items) == 1
    assert isinstance(items[0].ttl, int)
    assert items[0].ttl > 0
Exemplo n.º 3
0
def test_serialization():
    """Tests that the dictionary serialization for PynamoDB objects works properly."""
    from historical.s3.models import CurrentS3Model

    bucket = S3_BUCKET.copy()
    bucket['eventTime'] = datetime(
        year=2017, month=5, day=12, hour=10, minute=30,
        second=0).isoformat() + 'Z'

    bucket = CurrentS3Model(**bucket)
    dictionary = dict(bucket)

    assert dictionary['version'] == VERSION
    assert dictionary['configuration']['LifecycleRules'][0]['Prefix'] is None
Exemplo n.º 4
0
def create_delete_model(record):
    """Create an S3 model from a record."""
    arn = "arn:aws:s3:::{}".format(cloudwatch.filter_request_parameters('bucketName', record))
    log.debug('[-] Deleting Dynamodb Records. Hash Key: {arn}'.format(arn=arn))

    data = {
        'arn': arn,
        'principalId': cloudwatch.get_principal(record),
        'userIdentity': cloudwatch.get_user_identity(record),
        'accountId': record['account'],
        'eventTime': record['detail']['eventTime'],
        'BucketName': cloudwatch.filter_request_parameters('bucketName', record),
        'Region': cloudwatch.get_region(record),
        'Tags': {},
        'configuration': {},
        'eventSource': record["detail"]["eventSource"]
    }

    return CurrentS3Model(**data)
Exemplo n.º 5
0
def historical_table(current_s3_table):
    for x in range(0, 10):
        bucket = json.loads(S3_BUCKET.replace("{number}", "{}".format(x)))
        CurrentS3Model(**bucket).save()
Exemplo n.º 6
0
def process_update_records(update_records):
    """Process the requests for S3 bucket update requests"""
    events = sorted(update_records, key=lambda x: x['account'])

    # Group records by account for more efficient processing
    for account_id, events in groupby(events, lambda x: x['account']):
        events = list(events)

        # Grab the bucket names (de-dupe events):
        buckets = {}
        for e in events:
            # If the creation date is present, then use it:
            bucket_event = buckets.get(e["detail"]["requestParameters"]["bucketName"], {
                "creationDate": e["detail"]["requestParameters"].get("creationDate")
            })
            bucket_event.update(e["detail"]["requestParameters"])

            buckets[e["detail"]["requestParameters"]["bucketName"]] = bucket_event
            buckets[e["detail"]["requestParameters"]["bucketName"]]["eventDetails"] = e

        # Query AWS for current configuration
        for b, item in buckets.items():
            log.debug("[~] Processing Create/Update for: {}".format(b))
            # If the bucket does not exist, then simply drop the request --
            # If this happens, there is likely a Delete event that has occurred and will be processed soon.
            try:
                bucket_details = get_bucket(b,
                                            account_number=account_id,
                                            include_created=(item.get("creationDate") is None),
                                            assume_role=HISTORICAL_ROLE,
                                            region=CURRENT_REGION)
                if bucket_details.get("Error"):
                    log.error("[X] Unable to fetch details about bucket: {}. "
                              "The error details are: {}".format(b, bucket_details["Error"]))
                    continue

            except ClientError as ce:
                if ce.response["Error"]["Code"] == "NoSuchBucket":
                    log.warning("[?] Received update request for bucket: {} that does not "
                                "currently exist. Skipping.".format(b))
                    continue

                # Catch Access Denied exceptions as well:
                if ce.response["Error"]["Code"] == "AccessDenied":
                    log.error("[X] Unable to fetch details for S3 Bucket: {} in {}. Access is Denied. Skipping...".format(
                        b, account_id
                    ))
                    continue
                raise Exception(ce)

            # Pull out the fields we want:
            data = {
                "arn": "arn:aws:s3:::{}".format(b),
                "principalId": cloudwatch.get_principal(item["eventDetails"]),
                "userIdentity": cloudwatch.get_user_identity(item["eventDetails"]),
                "accountId": account_id,
                "eventTime": item["eventDetails"]["detail"]["eventTime"],
                "BucketName": b,
                "Region": bucket_details["Region"],
                # Duplicated in top level and configuration for secondary index
                "Tags": bucket_details["Tags"] or {},
                "eventSource": item["eventDetails"]["detail"]["eventSource"]
            }

            # Remove the fields we don't care about:
            del bucket_details["Arn"]
            del bucket_details["GrantReferences"]
            del bucket_details["Region"]

            if not bucket_details.get("CreationDate"):
                bucket_details["CreationDate"] = item["creationDate"]

            data["configuration"] = bucket_details

            current_revision = CurrentS3Model(**data)
            current_revision.save()
Exemplo n.º 7
0
def process_update_records(update_records):
    """Process the requests for S3 bucket update requests"""
    events = sorted(update_records, key=lambda x: x['account'])

    # Group records by account for more efficient processing
    for account_id, events in groupby(events, lambda x: x['account']):
        events = list(events)

        # Grab the bucket names (de-dupe events):
        buckets = {}
        for event in events:
            # If the creation date is present, then use it:
            bucket_event = buckets.get(
                event['detail']['requestParameters']['bucketName'], {
                    'creationDate':
                    event['detail']['requestParameters'].get('creationDate')
                })
            bucket_event.update(event['detail']['requestParameters'])

            buckets[event['detail']['requestParameters']
                    ['bucketName']] = bucket_event
            buckets[event['detail']['requestParameters']
                    ['bucketName']]['eventDetails'] = event

        # Query AWS for current configuration
        for b_name, item in buckets.items():
            LOG.debug(f'[~] Processing Create/Update for: {b_name}')
            # If the bucket does not exist, then simply drop the request --
            # If this happens, there is likely a Delete event that has occurred and will be processed soon.
            try:
                bucket_details = get_bucket(
                    b_name,
                    account_number=account_id,
                    include_created=(item.get('creationDate') is None),
                    assume_role=HISTORICAL_ROLE,
                    region=CURRENT_REGION)
                if bucket_details.get('Error'):
                    LOG.error(
                        f"[X] Unable to fetch details about bucket: {b_name}. "
                        f"The error details are: {bucket_details['Error']}")
                    continue

            except ClientError as cerr:
                if cerr.response['Error']['Code'] == 'NoSuchBucket':
                    LOG.warning(
                        f'[?] Received update request for bucket: {b_name} that does not '
                        'currently exist. Skipping.')
                    continue

                # Catch Access Denied exceptions as well:
                if cerr.response['Error']['Code'] == 'AccessDenied':
                    LOG.error(
                        f'[X] Unable to fetch details for S3 Bucket: {b_name} in {account_id}. Access is Denied. '
                        'Skipping...')
                    continue
                raise Exception(cerr)

            # Pull out the fields we want:
            data = {
                'arn':
                f'arn:aws:s3:::{b_name}',
                'principalId':
                cloudwatch.get_principal(item['eventDetails']),
                'userIdentity':
                cloudwatch.get_user_identity(item['eventDetails']),
                'userAgent':
                item['eventDetails']['detail'].get('userAgent'),
                'sourceIpAddress':
                item['eventDetails']['detail'].get('sourceIPAddress'),
                'requestParameters':
                item['eventDetails']['detail'].get('requestParameters'),
                'accountId':
                account_id,
                'eventTime':
                item['eventDetails']['detail']['eventTime'],
                'BucketName':
                b_name,
                'Region':
                bucket_details.pop('Region'),
                # Duplicated in top level and configuration for secondary index
                'Tags':
                bucket_details.pop('Tags', {}) or {},
                'eventSource':
                item['eventDetails']['detail']['eventSource'],
                'eventName':
                item['eventDetails']['detail']['eventName'],
                'version':
                VERSION
            }

            # Remove the fields we don't care about:
            del bucket_details['Arn']
            del bucket_details['GrantReferences']
            del bucket_details['_version']
            del bucket_details['Name']

            if not bucket_details.get('CreationDate'):
                bucket_details['CreationDate'] = item['creationDate']

            data['configuration'] = bucket_details

            current_revision = CurrentS3Model(**data)
            current_revision.save()