def parse_buckets(self, bucket, params):
"""
Parse a single S3 bucket TODO
"""
bucket['name'] = bucket.pop('Name')
api_client = params['api_clients']['us-east-1']
bucket['CreationDate'] = str(bucket['CreationDate'])
bucket['region'] = get_s3_bucket_location(api_client, bucket['name'])
# h4ck :: fix issue #59, location constraint can be EU or eu-west-1 for Ireland...
if bucket['region'] == 'EU':
bucket['region'] = 'eu-west-1'
# h4ck :: S3 is global but region-aware...
if bucket['region'] not in params['api_clients']:
printInfo('Skipping bucket %s (region %s outside of scope)' % (bucket['name'], bucket['region']))
self.buckets_count -= 1
return
api_client = params['api_clients'][bucket['region']]
get_s3_bucket_logging(api_client, bucket['name'], bucket)
get_s3_bucket_versioning(api_client, bucket['name'], bucket)
get_s3_bucket_webhosting(api_client, bucket['name'], bucket)
bucket['grantees'] = get_s3_acls(api_client, bucket['name'], bucket)
# TODO:
# CORS
# Lifecycle
# Notification ?
# Get bucket's policy
get_s3_bucket_policy(api_client, bucket['name'], bucket)
# If requested, get key properties
#if params['check_encryption'] or params['check_acls']:
# get_s3_bucket_keys(api_client, bucket['name'], bucket, params['check_encryption'],
# params['check_acls'])
bucket['id'] = self.get_non_aws_id(bucket['name'])
self.buckets[bucket['id']] = bucket
def parse_buckets(self, bucket, params):
"""
Parse a single S3 bucket
TODO:
- CORS
- Lifecycle
- Notification ?
- Get bucket's policy
:param bucket:
:param params:
:return:
"""
bucket['name'] = bucket.pop('Name')
api_client = params['api_clients'][get_s3_list_region(list(params['api_clients'].keys())[0])]
bucket['CreationDate'] = str(bucket['CreationDate'])
bucket['region'] = get_s3_bucket_location(api_client, bucket['name'])
# h4ck :: fix issue #59, location constraint can be EU or eu-west-1 for Ireland...
if bucket['region'] == 'EU':
bucket['region'] = 'eu-west-1'
# h4ck :: S3 is global but region-aware...
if bucket['region'] not in params['api_clients']:
printInfo('Skipping bucket %s (region %s outside of scope)' % (bucket['name'], bucket['region']))
self.buckets_count -= 1
return
api_client = params['api_clients'][bucket['region']]
get_s3_bucket_logging(api_client, bucket['name'], bucket)
get_s3_bucket_versioning(api_client, bucket['name'], bucket)
get_s3_bucket_webhosting(api_client, bucket['name'], bucket)
get_s3_bucket_default_encryption(api_client, bucket['name'], bucket)
bucket['grantees'] = get_s3_acls(api_client, bucket['name'], bucket)
get_s3_bucket_policy(api_client, bucket['name'], bucket)
get_s3_bucket_secure_transport(api_client, bucket['name'], bucket)
# If requested, get key properties
#if params['check_encryption'] or params['check_acls']:
# get_s3_bucket_keys(api_client, bucket['name'], bucket, params['check_encryption'],
# params['check_acls'])
bucket['id'] = self.get_non_aws_id(bucket['name'])
self.buckets[bucket['id']] = bucket
def main():
# Parse arguments
parser = OpinelArgumentParser()
parser.add_argument('debug')
parser.add_argument('profile')
parser.add_argument('regions')
parser.add_argument('partition-name')
parser.add_argument('bucket-name')
parser.parser.add_argument('--aws-account-id',
dest='aws_account_id',
default=[ None ],
nargs='+',
help='Bleh.')
parser.parser.add_argument('--from',
dest='from_date',
default=[ None ],
nargs='+',
help='Bleh.')
parser.parser.add_argument('--to',
dest='to_date',
default=[ None ],
nargs='+',
help='Bleh.')
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Arguments
profile_name = args.profile[0]
try:
from_date = datetime.datetime.strptime(args.from_date[0], "%Y/%m/%d").date()
to_date = datetime.datetime.strptime(args.to_date[0], "%Y/%m/%d").date()
delta = to_date - from_date
except Exception as e:
printException(e)
printError('Error: dates must be formatted of the following format YYYY/MM/DD')
return 42
if delta.days < 0:
printError('Error: your \'to\' date is earlier than your \'from\' date')
return 42
# Search for AWS credentials
credentials = read_creds(profile_name)
if not credentials['AccessKeyId']:
return 42
# Fetch AWS account ID
if not args.aws_account_id[0]:
printInfo('Fetching the AWS account ID...')
aws_account_id = get_aws_account_id(credentials)
else:
aws_account_id = args.aws_account_id[0]
global cloudtrail_log_path
cloudtrail_log_path = cloudtrail_log_path.replace('AWS_ACCOUNT_ID', aws_account_id)
# Create download dir
if not os.path.exists(download_folder):
os.makedirs(download_folder)
# Iterate through regions
s3_clients = {}
for region in build_region_list('cloudtrail', args.regions, args.partition_name):
# Connect to CloudTrail
cloudtrail_client = connect_service('cloudtrail', credentials, region)
if not cloudtrail_client:
continue
# Get information about the S3 bucket that receives CloudTrail logs
trails = cloudtrail_client.describe_trails()
for trail in trails['trailList']:
bucket_name = trail['S3BucketName']
prefix = trail['S3KeyPrefix'] if 'S3KeyPrefix' in trail else ''
# Connect to S3
manage_dictionary(s3_clients, region, connect_service('s3', credentials, region))
target_bucket_region = get_s3_bucket_location(s3_clients[region], bucket_name)
manage_dictionary(s3_clients, target_bucket_region, connect_service('s3', credentials, target_bucket_region))
s3_client = s3_clients[target_bucket_region]
# Generate base path for files
log_path = os.path.join(prefix, cloudtrail_log_path.replace('REGION', region))
# Download files
printInfo('Downloading log files in %s... ' % region, False)
keys = []
for i in range(delta.days + 1):
day = from_date + timedelta(days=i)
folder_path = os.path.join(log_path, day.strftime("%Y/%m/%d"))
try:
objects = handle_truncated_response(s3_client.list_objects, {'Bucket': bucket_name, 'Prefix': folder_path}, ['Contents'])
for o in objects['Contents']:
keys.append([o['Key'], 0])
except Exception as e:
printException(e)
pass
thread_work(keys, download_object, params = {'Bucket': bucket_name, 'S3Client': s3_client}, num_threads = 100)
printInfo('Done')
# Iterate through files and gunzip 'em
printInfo('Decompressing files...')
gzlogs = []
for root, dirnames, filenames in os.walk(download_folder):
for filename in filenames:
gzlogs.append(filename)
thread_work(gzlogs, gunzip_file, num_threads = 30)