def list_backup_files(config: Config, timestamp: str):
s3_resource = Session(profile_name=config.aws_profile()).resource('s3')
backup_bucket = s3_resource.Bucket(config.s3_bucket())
for s3_object in backup_bucket.objects.filter(Prefix=timestamp):
base_filename = path.splitext(path.basename(s3_object.key))[0]
cli_library.echo('* {}'.format(base_filename))
class BucketWorker(Thread):
def __init__(self, q, *args, **kwargs):
self.q = q
self.use_aws = True
if CONFIG["aws_access_key"] and CONFIG["aws_secret"]:
self.session = Session(
aws_access_key_id=CONFIG["aws_access_key"],
aws_secret_access_key=CONFIG["aws_secret"]).resource("s3")
# boto3 is resilient in finding creds and does not require them passed in to the Session() call
# let boto3 try to create a session without needing creds in the config.yaml, and fall back to
# HTTP requests if that fails
else:
self.session = Session().resource("s3")
if self.session.meta.client.meta.region_name:
pass
else:
self.use_aws = False
self.session = requests.Session()
self.session.mount(
"http://",
HTTPAdapter(pool_connections=ARGS.threads,
pool_maxsize=QUEUE_SIZE,
max_retries=0))
super().__init__(*args, **kwargs)
def run(self):
global THREAD_EVENT
while not THREAD_EVENT.is_set():
try:
bucket_url = self.q.get()
self.__check_boto(
bucket_url) if self.use_aws else self.__check_http(
bucket_url)
except Exception as e:
print(e)
pass
finally:
self.q.task_done()
def __check_http(self, bucket_url):
check_response = self.session.head(S3_URL,
timeout=3,
headers={"Host": bucket_url})
if not ARGS.ignore_rate_limiting\
and (check_response.status_code == 503 and check_response.reason == "Slow Down"):
self.q.rate_limited = True
# add it back to the queue for re-processing
self.q.put(bucket_url)
elif check_response.status_code == 307: # valid bucket, lets check if its public
new_bucket_url = check_response.headers["Location"]
bucket_response = requests.request(
"GET" if ARGS.only_interesting else "HEAD",
new_bucket_url,
timeout=3)
if bucket_response.status_code == 200\
and (not ARGS.only_interesting or
(ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))):
self.__output("Found bucket '{}'".format(new_bucket_url),
"green")
self.__log(new_bucket_url)
def __check_boto(self, bucket_url):
bucket_name = bucket_url.replace(".s3.amazonaws.com", "")
try:
# just to check if the bucket exists. Throws NoSuchBucket exception if not
self.session.meta.client.head_bucket(Bucket=bucket_name)
if not ARGS.only_interesting or\
(ARGS.only_interesting and self.__bucket_contains_any_keywords(bucket_name)):
owner = None
acls = None
try:
# todo: also check IAM policy as it can override ACLs
acl = self.session.meta.client.get_bucket_acl(
Bucket=bucket_name)
owner = acl["Owner"]["DisplayName"]
acls = ". ACLs = {} | {}".format(
self.__get_group_acls(acl, "AllUsers"),
self.__get_group_acls(acl, "AuthenticatedUsers"))
except:
acls = ". ACLS = (could not read)"
color = "green" if not owner else "magenta"
self.__output(
"Found bucket '{}'. Owned by '{}'{}".format(
bucket_url, owner if owner else "(unknown)", acls),
color)
self.__log(bucket_url)
except Exception as e:
pass
def __get_group_acls(self, acl, group):
group_uri = "http://acs.amazonaws.com/groups/global/%s" % group
perms = [
g["Permission"] for g in acl["Grants"]
if g["Grantee"]["Type"] == "Group"
and g["Grantee"]["URI"] == group_uri
]
return "{}: {}".format(group, ", ".join(perms) if perms else "(none)")
def __bucket_contains_any_keywords(self, bucket_name):
try:
objects = [
o.key for o in self.session.Bucket(bucket_name).objects.all()
]
return any(keyword in ",".join(objects) for keyword in KEYWORDS)
except:
return False
def __log(self, new_bucket_url):
global FOUND_COUNT
FOUND_COUNT += 1
if ARGS.log_to_file:
with open("/data/buckets.log", "a+") as log:
log.write("%s%s" % (new_bucket_url, os.linesep))
def __output(self, line, color=None):
cprint(line, color, attrs=["bold"])
if CONFIG["slack_webhook"]:
resp = requests.post(CONFIG['slack_webhook'],
data=json.dumps({'text': line}),
headers={'Content-Type': 'application/json'})
if resp.status_code != 200:
cprint(
"Could not send to your Slack Webhook. Server returned: %s"
% resp.status_code, "red")
class BucketWorker(Thread):
def __init__(self, q, *args, **kwargs):
self.q = q
self.use_aws = CONFIG["aws_access_key"] and CONFIG["aws_secret"]
if self.use_aws:
self.session = Session(
aws_access_key_id=CONFIG["aws_access_key"], aws_secret_access_key=CONFIG["aws_secret"]).resource("s3")
else:
self.session = requests.Session()
self.session.mount(
"http://", HTTPAdapter(pool_connections=ARGS.threads, pool_maxsize=QUEUE_SIZE, max_retries=0))
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
bucket_url = self.q.get()
self.__check_boto(
bucket_url) if self.use_aws else self.__check_http(bucket_url)
except Exception as e:
print(e)
pass
finally:
self.q.task_done()
def __check_http(self, bucket_url):
check_response = self.session.head(
S3_URL, timeout=3, headers={"Host": bucket_url})
if not ARGS.ignore_rate_limiting\
and (check_response.status_code == 503 and check_response.reason == "Slow Down"):
self.q.rate_limited = True
# add it back to the bucket for re-processing
self.q.put(bucket_url)
elif check_response.status_code == 307: # valid bucket, lets check if its public
new_bucket_url = check_response.headers["Location"]
bucket_response = requests.request(
"GET" if ARGS.only_interesting else "HEAD", new_bucket_url, timeout=3)
if bucket_response.status_code == 200\
and (not ARGS.only_interesting or
(ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))):
cprint("Found bucket '{}'".format(new_bucket_url), "green", attrs=["bold"])
self.__log(new_bucket_url, "", "")
def __check_boto(self, bucket_url):
bucket_name = bucket_url.replace(".s3.amazonaws.com", "")
try:
# just to check if the bucket exists. Throws NoSuchBucket exception if not
self.session.meta.client.head_bucket(Bucket=bucket_name)
if not ARGS.only_interesting or\
(ARGS.only_interesting and self.__bucket_contains_any_keywords(bucket_name)):
owner = None
acls = None
try:
# todo: also check IAM policy as it can override ACLs
acl = self.session.meta.client.get_bucket_acl(
Bucket=bucket_name)
owner = acl["Owner"]["DisplayName"]
acls = "ACLS = {} | {}".format(self.__get_group_acls(acl, "AllUsers"),
self.__get_group_acls(acl, "AuthenticatedUsers"))
except:
acls = "ACLS = (could not read)"
color = "green" if not owner else "magenta"
cprint("Found bucket '{}'. Owned by '{}'. {}".format(
bucket_url, owner if owner else "(unknown)", acls), color, attrs=["bold"])
self.__log("{};OWNER:{};{}".format(
bucket_url, owner if owner else "(unknown)", acls))
except:
pass
def __get_group_acls(self, acl, group):
group_uri = "http://acs.amazonaws.com/groups/global/%s" % group
perms = [g["Permission"] for g in acl["Grants"]
if g["Grantee"]["Type"] == "Group" and g["Grantee"]["URI"] == group_uri]
return "{}: {}".format(group, ", ".join(perms) if perms else "(none)")
def __bucket_contains_any_keywords(self, bucket_name):
try:
objects = [o.key for o in self.session.Bucket(bucket_name).objects.all()]
return any(keyword in ",".join(objects) for keyword in KEYWORDS)
except:
return False
def __log(self, logentry):
global FOUND_COUNT
FOUND_COUNT += 1
if ARGS.log_to_file:
with open("buckets.log", "a+") as log:
log.write("%s%s" % (logentry, os.linesep))
