def test_bucket_scan_empty_bucket(self): manager = self.manager visitor = EncryptExtantKeys({'report-only': True}, manager, self.log_dir) result = visitor.process([{"Name": self.b}]) # Assert that we get the right remediated counts self.assertEqual( result, [{'Count': 0, 'Remediated': 0, 'Bucket': 'cloud-maid-ftest'}])
def process_key_event(event, context): init() processor = EncryptExtantKeys(config) for record in event.get('Records', []): bucket = record['s3']['bucket']['name'] key = {'Key': record['s3']['object']['key']} version = record['s3']['object'].get('versionId') if version is not None: result = processor.process_version(s3, key, bucket) else: result = processor.process_key(s3, key, bucket) if not result: return print("remediated %s:%s" % (bucket, key['Key']))
def test_encrypt_keys(self): self.generate_contents() manager = self.manager visitor = EncryptExtantKeys({}, manager, self.log_dir) result = visitor.process([{"Name": self.b}]) self.assertEqual( result, [{'Count': 3, 'Remediated': 3, 'Bucket': self.b}]) # Assert that we get the right remediated counts in the log self.assertTrue( "keys:3 remediated:3" in self.output.getvalue()) self.assertTrue( 'ServerSideEncryption' in self.client.head_object( Bucket=self.b, Key='home.txt'))
def process_key_event(event, context): processor = EncryptExtantKeys(config) for record in event.get('Records', []): bucket = record['s3']['bucket']['name'] key = { 'Key': record['s3']['object']['key'], 'Size': record['s3']['object']['size'] } version = record['s3']['object'].get('versionId') try: if version is not None: key['VersionId'] = version # lambda event is always latest version, but IsLatest # is not in record key['IsLatest'] = True result = retry(processor.process_version, s3, key, bucket) else: result = retry(processor.process_key, s3, key, bucket) except ClientError as e: # Ensure we know which key caused an issue print("error %s:%s code:%s" % (bucket, key['Key'], e.response['Error'])) raise if not result: return print("remediated %s:%s" % (bucket, key['Key']))
def process_key_event(event, context): init() processor = EncryptExtantKeys(config) for record in event.get('Records', []): bucket = record['s3']['bucket']['name'] key = {'Key': record['s3']['object']['key'], 'Size': record['s3']['object']['size']} version = record['s3']['object'].get('versionId') if version is not None: key['VersionId'] = version key['IsLatest'] = True # lambda event is always latest version, but IsLatest is not in record result = processor.process_version(s3, key, bucket) else: result = processor.process_key(s3, key, bucket) if not result: return print("remediated %s:%s" % (bucket, key['Key']))
def get_key_visitors(account_info): if not account_info.get('visitors'): return [EncryptExtantKeys(keyconfig)] visitors = [] for v in account_info.get('visitors'): if v['type'] == 'encrypt-keys': vi = EncryptExtantKeys(v) vi.visitor_name = 'encrypt-keys' vi.inventory_filter = filter_encrypted visitors.append(vi) elif v['type'] == 'object-acl': vi = ObjectAclCheck(v) vi.visitor_name = 'object-acl' vi.inventory_filter = None visitors.append(vi) return visitors
def process_keyset(account_info, bucket, key_set): session = get_session(account_info) s3 = session.client('s3', region_name=bucket['region'], config=s3config) processor = EncryptExtantKeys(keyconfig) remediation_count = 0 denied_count = 0 contents_key, _, _ = BUCKET_OBJ_DESC[bucket['versioned']] processor = (bucket['versioned'] and processor.process_version or processor.process_key) connection.hincrby( 'keys-scanned', bucket_id(account_info, bucket['name']), len(key_set.get(contents_key, []))) log.info("processing page size: %d on %s", len(key_set.get(contents_key, ())), bucket_id(account_info, bucket['name'])) with bucket_ops(account_info, bucket, 'key'): for k in key_set.get(contents_key, []): try: result = processor(s3, bucket_name=bucket['name'], key=k) except ConnectionError: continue except ClientError as e: # https://goo.gl/HZLv9b code = e.response['Error']['Code'] if code == '403': # Permission Denied denied_count += 1 continue elif code == '404': # Not Found continue elif code in ('503', '400'): # Slow Down, or token err # TODO, consider backoff alg usage, and re-queue of keys time.sleep(3) continue raise if result is False: continue remediation_count += 1 if remediation_count: connection.hincrby( 'keys-matched', bucket_id(account_info, bucket['name']), remediation_count) if denied_count: connection.hincrby( 'keys-denied', bucket_id(account_info, bucket['name']), denied_count)