def list_backup_files(config: Config, timestamp: str): s3_resource = Session(profile_name=config.aws_profile()).resource('s3') backup_bucket = s3_resource.Bucket(config.s3_bucket()) for s3_object in backup_bucket.objects.filter(Prefix=timestamp): base_filename = path.splitext(path.basename(s3_object.key))[0] cli_library.echo('* {}'.format(base_filename))
class BucketWorker(Thread): def __init__(self, q, *args, **kwargs): self.q = q self.use_aws = True if CONFIG["aws_access_key"] and CONFIG["aws_secret"]: self.session = Session( aws_access_key_id=CONFIG["aws_access_key"], aws_secret_access_key=CONFIG["aws_secret"]).resource("s3") # boto3 is resilient in finding creds and does not require them passed in to the Session() call # let boto3 try to create a session without needing creds in the config.yaml, and fall back to # HTTP requests if that fails else: self.session = Session().resource("s3") if self.session.meta.client.meta.region_name: pass else: self.use_aws = False self.session = requests.Session() self.session.mount( "http://", HTTPAdapter(pool_connections=ARGS.threads, pool_maxsize=QUEUE_SIZE, max_retries=0)) super().__init__(*args, **kwargs) def run(self): global THREAD_EVENT while not THREAD_EVENT.is_set(): try: bucket_url = self.q.get() self.__check_boto( bucket_url) if self.use_aws else self.__check_http( bucket_url) except Exception as e: print(e) pass finally: self.q.task_done() def __check_http(self, bucket_url): check_response = self.session.head(S3_URL, timeout=3, headers={"Host": bucket_url}) if not ARGS.ignore_rate_limiting\ and (check_response.status_code == 503 and check_response.reason == "Slow Down"): self.q.rate_limited = True # add it back to the queue for re-processing self.q.put(bucket_url) elif check_response.status_code == 307: # valid bucket, lets check if its public new_bucket_url = check_response.headers["Location"] bucket_response = requests.request( "GET" if ARGS.only_interesting else "HEAD", new_bucket_url, timeout=3) if bucket_response.status_code == 200\ and (not ARGS.only_interesting or (ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))): self.__output("Found bucket '{}'".format(new_bucket_url), "green") self.__log(new_bucket_url) def __check_boto(self, bucket_url): bucket_name = bucket_url.replace(".s3.amazonaws.com", "") try: # just to check if the bucket exists. Throws NoSuchBucket exception if not self.session.meta.client.head_bucket(Bucket=bucket_name) if not ARGS.only_interesting or\ (ARGS.only_interesting and self.__bucket_contains_any_keywords(bucket_name)): owner = None acls = None try: # todo: also check IAM policy as it can override ACLs acl = self.session.meta.client.get_bucket_acl( Bucket=bucket_name) owner = acl["Owner"]["DisplayName"] acls = ". ACLs = {} | {}".format( self.__get_group_acls(acl, "AllUsers"), self.__get_group_acls(acl, "AuthenticatedUsers")) except: acls = ". ACLS = (could not read)" color = "green" if not owner else "magenta" self.__output( "Found bucket '{}'. Owned by '{}'{}".format( bucket_url, owner if owner else "(unknown)", acls), color) self.__log(bucket_url) except Exception as e: pass def __get_group_acls(self, acl, group): group_uri = "http://acs.amazonaws.com/groups/global/%s" % group perms = [ g["Permission"] for g in acl["Grants"] if g["Grantee"]["Type"] == "Group" and g["Grantee"]["URI"] == group_uri ] return "{}: {}".format(group, ", ".join(perms) if perms else "(none)") def __bucket_contains_any_keywords(self, bucket_name): try: objects = [ o.key for o in self.session.Bucket(bucket_name).objects.all() ] return any(keyword in ",".join(objects) for keyword in KEYWORDS) except: return False def __log(self, new_bucket_url): global FOUND_COUNT FOUND_COUNT += 1 if ARGS.log_to_file: with open("/data/buckets.log", "a+") as log: log.write("%s%s" % (new_bucket_url, os.linesep)) def __output(self, line, color=None): cprint(line, color, attrs=["bold"]) if CONFIG["slack_webhook"]: resp = requests.post(CONFIG['slack_webhook'], data=json.dumps({'text': line}), headers={'Content-Type': 'application/json'}) if resp.status_code != 200: cprint( "Could not send to your Slack Webhook. Server returned: %s" % resp.status_code, "red")
class BucketWorker(Thread): def __init__(self, q, *args, **kwargs): self.q = q self.use_aws = CONFIG["aws_access_key"] and CONFIG["aws_secret"] if self.use_aws: self.session = Session( aws_access_key_id=CONFIG["aws_access_key"], aws_secret_access_key=CONFIG["aws_secret"]).resource("s3") else: self.session = requests.Session() self.session.mount( "http://", HTTPAdapter(pool_connections=ARGS.threads, pool_maxsize=QUEUE_SIZE, max_retries=0)) super().__init__(*args, **kwargs) def run(self): while True: try: bucket_url = self.q.get() self.__check_boto( bucket_url) if self.use_aws else self.__check_http(bucket_url) except Exception as e: print(e) pass finally: self.q.task_done() def __check_http(self, bucket_url): check_response = self.session.head( S3_URL, timeout=3, headers={"Host": bucket_url}) if not ARGS.ignore_rate_limiting\ and (check_response.status_code == 503 and check_response.reason == "Slow Down"): self.q.rate_limited = True # add it back to the bucket for re-processing self.q.put(bucket_url) elif check_response.status_code == 307: # valid bucket, lets check if its public new_bucket_url = check_response.headers["Location"] bucket_response = requests.request( "GET" if ARGS.only_interesting else "HEAD", new_bucket_url, timeout=3) if bucket_response.status_code == 200\ and (not ARGS.only_interesting or (ARGS.only_interesting and any(keyword in bucket_response.text for keyword in KEYWORDS))): cprint("Found bucket '{}'".format(new_bucket_url), "green", attrs=["bold"]) self.__log(new_bucket_url, "", "") def __check_boto(self, bucket_url): bucket_name = bucket_url.replace(".s3.amazonaws.com", "") try: # just to check if the bucket exists. Throws NoSuchBucket exception if not self.session.meta.client.head_bucket(Bucket=bucket_name) if not ARGS.only_interesting or\ (ARGS.only_interesting and self.__bucket_contains_any_keywords(bucket_name)): owner = None acls = None try: # todo: also check IAM policy as it can override ACLs acl = self.session.meta.client.get_bucket_acl( Bucket=bucket_name) owner = acl["Owner"]["DisplayName"] acls = "ACLS = {} | {}".format(self.__get_group_acls(acl, "AllUsers"), self.__get_group_acls(acl, "AuthenticatedUsers")) except: acls = "ACLS = (could not read)" color = "green" if not owner else "magenta" cprint("Found bucket '{}'. Owned by '{}'. {}".format( bucket_url, owner if owner else "(unknown)", acls), color, attrs=["bold"]) self.__log("{};OWNER:{};{}".format( bucket_url, owner if owner else "(unknown)", acls)) except: pass def __get_group_acls(self, acl, group): group_uri = "http://acs.amazonaws.com/groups/global/%s" % group perms = [g["Permission"] for g in acl["Grants"] if g["Grantee"]["Type"] == "Group" and g["Grantee"]["URI"] == group_uri] return "{}: {}".format(group, ", ".join(perms) if perms else "(none)") def __bucket_contains_any_keywords(self, bucket_name): try: objects = [o.key for o in self.session.Bucket(bucket_name).objects.all()] return any(keyword in ",".join(objects) for keyword in KEYWORDS) except: return False def __log(self, logentry): global FOUND_COUNT FOUND_COUNT += 1 if ARGS.log_to_file: with open("buckets.log", "a+") as log: log.write("%s%s" % (logentry, os.linesep))