def test_bucket_scan_empty_bucket(self):
manager = self.manager
visitor = EncryptExtantKeys({'report-only': True}, manager, self.log_dir)
result = visitor.process([{"Name": self.b}])
# Assert that we get the right remediated counts
self.assertEqual(
result, [{'Count': 0, 'Remediated': 0, 'Bucket': 'cloud-maid-ftest'}])
def process_key_event(event, context):
init()
processor = EncryptExtantKeys(config)
for record in event.get('Records', []):
bucket = record['s3']['bucket']['name']
key = {'Key': record['s3']['object']['key']}
version = record['s3']['object'].get('versionId')
if version is not None:
result = processor.process_version(s3, key, bucket)
else:
result = processor.process_key(s3, key, bucket)
if not result:
return
print("remediated %s:%s" % (bucket, key['Key']))
def test_encrypt_keys(self):
self.generate_contents()
manager = self.manager
visitor = EncryptExtantKeys({}, manager, self.log_dir)
result = visitor.process([{"Name": self.b}])
self.assertEqual(
result, [{'Count': 3, 'Remediated': 3, 'Bucket': self.b}])
# Assert that we get the right remediated counts in the log
self.assertTrue(
"keys:3 remediated:3" in self.output.getvalue())
self.assertTrue(
'ServerSideEncryption' in self.client.head_object(
Bucket=self.b, Key='home.txt'))
def process_key_event(event, context):
init()
processor = EncryptExtantKeys(config)
for record in event.get('Records', []):
bucket = record['s3']['bucket']['name']
key = {'Key': record['s3']['object']['key']}
version = record['s3']['object'].get('versionId')
if version is not None:
result = processor.process_version(s3, key, bucket)
else:
result = processor.process_key(s3, key, bucket)
if not result:
return
print("remediated %s:%s" % (bucket, key['Key']))
def process_key_event(event, context):
processor = EncryptExtantKeys(config)
for record in event.get('Records', []):
bucket = record['s3']['bucket']['name']
key = {
'Key': record['s3']['object']['key'],
'Size': record['s3']['object']['size']
}
version = record['s3']['object'].get('versionId')
try:
if version is not None:
key['VersionId'] = version
# lambda event is always latest version, but IsLatest
# is not in record
key['IsLatest'] = True
result = retry(processor.process_version, s3, key, bucket)
else:
result = retry(processor.process_key, s3, key, bucket)
except ClientError as e:
# Ensure we know which key caused an issue
print("error %s:%s code:%s" %
(bucket, key['Key'], e.response['Error']))
raise
if not result:
return
print("remediated %s:%s" % (bucket, key['Key']))
def process_key_event(event, context):
init()
processor = EncryptExtantKeys(config)
for record in event.get('Records', []):
bucket = record['s3']['bucket']['name']
key = {'Key': record['s3']['object']['key'], 'Size': record['s3']['object']['size']}
version = record['s3']['object'].get('versionId')
if version is not None:
key['VersionId'] = version
key['IsLatest'] = True # lambda event is always latest version, but IsLatest is not in record
result = processor.process_version(s3, key, bucket)
else:
result = processor.process_key(s3, key, bucket)
if not result:
return
print("remediated %s:%s" % (bucket, key['Key']))
def get_key_visitors(account_info):
if not account_info.get('visitors'):
return [EncryptExtantKeys(keyconfig)]
visitors = []
for v in account_info.get('visitors'):
if v['type'] == 'encrypt-keys':
vi = EncryptExtantKeys(v)
vi.visitor_name = 'encrypt-keys'
vi.inventory_filter = filter_encrypted
visitors.append(vi)
elif v['type'] == 'object-acl':
vi = ObjectAclCheck(v)
vi.visitor_name = 'object-acl'
vi.inventory_filter = None
visitors.append(vi)
return visitors
def process_keyset(account_info, bucket, key_set):
session = get_session(account_info)
s3 = session.client('s3', region_name=bucket['region'], config=s3config)
processor = EncryptExtantKeys(keyconfig)
remediation_count = 0
denied_count = 0
contents_key, _, _ = BUCKET_OBJ_DESC[bucket['versioned']]
processor = (bucket['versioned'] and processor.process_version
or processor.process_key)
connection.hincrby(
'keys-scanned', bucket_id(account_info, bucket['name']),
len(key_set.get(contents_key, [])))
log.info("processing page size: %d on %s",
len(key_set.get(contents_key, ())),
bucket_id(account_info, bucket['name']))
with bucket_ops(account_info, bucket, 'key'):
for k in key_set.get(contents_key, []):
try:
result = processor(s3, bucket_name=bucket['name'], key=k)
except ConnectionError:
continue
except ClientError as e:
# https://goo.gl/HZLv9b
code = e.response['Error']['Code']
if code == '403': # Permission Denied
denied_count += 1
continue
elif code == '404': # Not Found
continue
elif code in ('503', '400'): # Slow Down, or token err
# TODO, consider backoff alg usage, and re-queue of keys
time.sleep(3)
continue
raise
if result is False:
continue
remediation_count += 1
if remediation_count:
connection.hincrby(
'keys-matched',
bucket_id(account_info, bucket['name']),
remediation_count)
if denied_count:
connection.hincrby(
'keys-denied',
bucket_id(account_info, bucket['name']),
denied_count)
def get_key_visitors(account_info):
if not account_info.get('visitors'):
return [EncryptExtantKeys(keyconfig)]
visitors = []
for v in account_info.get('visitors'):
if v['type'] == 'encrypt-keys':
vi = EncryptExtantKeys(v)
vi.visitor_name = 'encrypt-keys'
vi.inventory_filter = filter_encrypted
visitors.append(vi)
elif v['type'] == 'object-acl':
vi = ObjectAclCheck(v)
vi.visitor_name = 'object-acl'
vi.inventory_filter = None
visitors.append(vi)
return visitors
