def scan_key_perms_v2(bucket, prefix, versions, make_private, workers):
startTime = time.time()
keys_proccessed = 0
s3 = boto3.resource( 's3' )
bkt = s3.Bucket( bucket )
objects = []
if not versions:
iterator = bkt.objects.filter(Prefix=prefix)
else:
iterator = bkt.object_versions.filter(Prefix=prefix)
processed = False
for k in iterator:
keys_proccessed += 1
processed = False
if versions:
obj, ver = k.object_key, k.id
else:
obj, ver = k.key, None
if len(objects) < 1000:
objects.append( [bucket, obj, ver, make_private] )
else:
multi_process(get_permission, objects, workers)
del objects[:]
processed = True
if not processed:
multi_process(get_permission, objects, workers)
elapsed = time.time() - startTime
end = round(elapsed, 2)
click.echo('\nTotal keys proccessed in total: %s in %ss' %(keys_proccessed, end))
def find_unencrypted_keys(bucket, prefix, versions, workers):
s3 = boto3.resource('s3')
bkt = s3.Bucket(bucket)
objects = []
if versions:
iterator = bkt.object_versions.filter(Prefix=prefix)
else:
iterator = bkt.objects.filter(Prefix=prefix)
processed = False
for k in iterator:
processed = False
if len(objects) < 1000:
if not versions:
data = {'bucket': bucket, 'key': k.key, 'version': 'null'}
else:
data = {'bucket': bucket, 'key': k.key, 'version': k.id}
objects.append(data)
# objects.append([bucket, k.key]) if not versions else objects.append([bucket, k.key, k.id])
else:
multi_process(get_encryption, objects, workers)
del objects[:]
processed = True
if not processed:
multi_process(get_encryption, objects, workers)
def scan_key_perms(scanperms, bucket, prefix, workers):
s3 = boto3.resource('s3')
bkt = s3.Bucket(bucket)
owner = bkt.Acl().owner['ID']
click.echo('>> Scanning bucket ACL')
click.echo(30 * '=')
click.echo('>> Scanning objects with PUBLIC ACL')
print(150 * "-")
print("Public {} | Other AWS Accounts {} | Owner {}| Key {}").format(
"".ljust(18), "".ljust(6), "".ljust(20), "".ljust(60))
print(150 * "-")
objects = []
iterator = bkt.objects.filter(Prefix=prefix)
processed = False
for k in iterator:
processed = False
if len(objects) < 1000:
objects.append([bucket, k.key, owner])
else:
multi_process(get_permission, objects, workers)
del objects[:]
processed = True
if not processed:
multi_process(get_permission, objects, workers)
def collect_keys(restore, bucket, prefix, days, type, versions,
permanent_restore, restore_to_bucket, storage_class,
update_restore_date, workers, include, exclude):
s3r = boto3.resource('s3')
startTime = time.time()
bkt = s3r.Bucket(bucket)
objects = []
keys_proccessed = 0
click.echo(
'Initiating %s restore for %s/%s...\nRestoring keys for %s days\nVersions: %s\n'
% (type, bucket, prefix, days, versions) + 30 * '=')
if versions:
iterator = bkt.object_versions.filter(Prefix=prefix)
else:
iterator = bkt.objects.filter(Prefix=prefix)
processed = False
for obj in iterator:
keys_proccessed += 1
processed = False
if len(objects) < 1000:
if versions:
data = [
bucket, obj.key, obj.id, days, type, update_restore_date
]
if include:
# if obj.key.endswith(include):
if any(x in obj.key for x in include):
objects.append(data)
else:
logging.warning(
'Not included because of filter (--include) to the key %s | Keys to include %s '
% (obj.key, include))
elif exclude:
if not any(x in obj.key for x in exclude):
objects.append(data)
else:
logging.warning(
'filter applied (exclude) to the key %s | Keys to exclude %s '
% (obj.key, exclude))
else:
objects.append(data)
else:
data = [
bucket, obj.key, days, type, permanent_restore,
restore_to_bucket, storage_class, update_restore_date
]
if include:
if any(x in obj.key for x in include):
objects.append(data)
else:
logging.warning(
' Not included because of filter (--include) to the key %s | Keys to include %s '
% (obj.key, include))
elif exclude:
if not any(x in obj.key for x in exclude):
objects.append(data)
else:
logging.warning(
'filter applied (exclude) to the key %s | Keys to exclude %s '
% (obj.key, exclude))
else:
objects.append(data)
else:
if versions:
multi_process(restore_versions, objects, workers)
else:
multi_process(restore_default, objects, workers)
del objects[:]
processed = True
if not processed:
if versions:
multi_process(restore_versions, objects, workers)
else:
multi_process(restore_default, objects, workers)
elapsed = time.time() - startTime
end = round(elapsed, 2)
click.echo('Total keys processed in total: %s in %ss' %
(keys_proccessed, end))