def test_config_version_socrata():
c = Config(f"{Path(__file__).parent}/data/socrata.yml")
uid = c.parsed_unrendered_template["dataset"]["source"]["socrata"]["uid"]
version = c.version_socrata(uid)
assert len(version) == 8 # format: YYYYMMDD
assert int(version[-2:]) <= 31 # check date
assert int(version[-4:-2]) <= 12 # check month
def collect_results(request_info, main_account):
security_features = request_info['request_params']['security_features']
regions = request_info['request_params']['regions']
scan_account_id = request_info['request_params']['account_id']
tags = request_info['request_params']['tags']
response = dict({'global': {}})
for region in regions:
response[region] = {}
for sec_feature in security_features:
if sec_feature not in GLOBAL_SECURITY_FEATURES:
response[region][sec_feature] = []
else:
response['global'][sec_feature] = []
config = Config()
for security_feature in security_features:
sec_feature_config = config.get_module_config_by_name(security_feature)
ddb_table = main_account.resource("dynamodb").Table(
sec_feature_config.ddb_table_name)
for issue in IssueOperations.get_account_open_issues(
ddb_table, scan_account_id):
if issue.contains_tags(tags) and (
issue.issue_details.region in regions
or security_feature in GLOBAL_SECURITY_FEATURES):
issue_region = issue.issue_details.region if issue.issue_details.region else 'global'
response[issue_region][security_feature].append({
'id':
issue.issue_id,
'issue_details':
issue.issue_details.as_dict()
})
return response
def lambda_handler(event, context):
""" Lambda handler to initiate to find public RDS snapshots """
set_logging(level=logging.INFO)
logging.debug("Initiating public RDS snapshots checking")
try:
sns_arn = os.environ["SNS_ARN"]
config = Config()
if not config.rdsSnapshot.enabled:
logging.debug("Public RDS snapshots checking disabled")
return
logging.debug("Iterating each account to initiate RDS snapshots checking")
for account_id, account_name in config.rdsSnapshot.accounts.items():
payload = {"account_id": account_id,
"account_name": account_name,
"regions": config.aws.regions,
"sns_arn": sns_arn
}
logging.debug(f"Initiating public RDS snapshots checking for '{account_name}'")
Sns.publish(sns_arn, payload)
except Exception:
logging.exception("Error occurred while initiation of public RDS snapshots checking")
return
logging.debug("Public RDS snapshot checking initiation done")
def lambda_handler(event, context):
""" Lambda handler to initiate to find SQS public access in policy """
set_logging(level=logging.INFO)
logging.debug("Initiating SQS policies checking")
try:
sns_arn = os.environ["SNS_ARN"]
config = Config()
if not config.sqspolicy.enabled:
logging.debug("SQS policies checking disabled")
return
logging.debug(
"Iterating over each account to initiate SQS policies check")
for account_id, account_name in config.sqspolicy.accounts.items():
payload = {
"account_id": account_id,
"account_name": account_name,
"regions": config.aws.regions,
"sns_arn": sns_arn
}
logging.debug(
f"Initiating SQS policies checking for '{account_name}'")
Sns.publish(sns_arn, payload)
except Exception:
logging.exception(
"Error occurred while initiation of SQS policy checking")
return
logging.debug("SQS policies checking initiation done")
def lambda_handler(event, context):
""" Lambda handler to initiate to find security groups unrestricted access """
set_logging(level=logging.INFO)
logging.debug("Initiating CloudTrail checking")
try:
sns_arn = os.environ["SNS_ARN"]
config = Config()
if not config.cloudtrails.enabled:
logging.debug("CloudTrail checking disabled")
return
logging.debug(
"Iterating over each account to initiate CloudTrail check")
for account_id, account_name in config.cloudtrails.accounts.items():
payload = {
"account_id": account_id,
"account_name": account_name,
"regions": config.aws.regions,
"sns_arn": sns_arn
}
logging.debug(
f"Initiating CloudTrail checking for '{account_name}'")
Sns.publish(sns_arn, payload)
except Exception:
logging.exception(
"Error occurred while initiation of CloudTrail checking")
return
logging.debug("CloudTrail checking initiation done")
def lambda_handler(event, context):
""" Lambda handler to initiate to find inactive keys for IAM users """
set_logging(level=logging.INFO)
logging.debug("Initiating IAM user inactive keys checking")
try:
sns_arn = os.environ["SNS_IAM_USER_INACTIVE_KEYS_ARN"]
config = Config()
if not config.iamUserInactiveKeys.enabled:
logging.debug("IAM user inactive keys checking disabled")
return
logging.debug(
"Iterating over each account to initiate IAM user inactive keys check"
)
for account_id, account_name in config.iamUserInactiveKeys.accounts.items(
):
payload = {
"account_id": account_id,
"account_name": account_name,
}
logging.debug(
f"Initiating IAM user inactive keys checking for '{account_name}'"
)
Sns.publish(sns_arn, payload)
except Exception:
logging.exception(
"Error occurred while initiation of IAM user inactive keys check")
return
logging.debug("IAM user inactive keys checking initiation done")
def get_error_msg(self, msg='E9999'):
"""
gets error message
"""
config = Config()
# READ XML FILE AND PARSE TO XML OBJECT
error_dir = config['error_code_dir']
exclusion = config['error_code_exclusion']
file_list = []
import os
for filename in os.listdir(error_dir):
if filename.endswith('yaml') and filename not in exclusion:
file_list.append(os.path.join(error_dir, filename))
# Read in all the files and assign them to an ET Root
error_config = {}
for filename in file_list:
config = yaml.load(open(os.path.join(error_dir, filename), 'rb'))
error_config.update(config['errors'])
# FIND THE ERROR CODE BY BREAKING DOWN THE msg
if len(msg.split()) == 1:
# THIS COULD BE A SPECIAL ERROR MESSAGE
for key in error_config.keys():
if key in msg.lower():
error_val = int(msg.split('_')[1])
return error_config[key][error_val]
return msg
def lambda_handler(event, context):
""" Lambda handler to initiate to find unencrypted EBS volumes """
set_logging(level=logging.INFO)
logging.debug("Initiating unencrypted EBS volumes checking")
try:
sns_arn = os.environ["SNS_EBS_VOLUMES_ARN"]
config = Config()
if not config.ebsVolume.enabled:
logging.debug("Unencrypted EBS volumes checking disabled")
return
logging.debug("Iterating over each account to initiate unencrypted EBS volumes checking")
for account_id, account_name in config.ebsVolume.accounts.items():
payload = {"account_id": account_id,
"account_name": account_name,
"regions": config.aws.regions,
"sns_arn": sns_arn
}
logging.debug(f"Initiating unencrypted EBS volume checking for '{account_name}'")
Sns.publish(sns_arn, payload)
except Exception:
logging.exception("Error occurred while initiation of unencrypted EBS volumes checking")
return
logging.debug("Unencrypted EBS volume checking initiation done")
def lambda_handler(event, context):
set_logging(level=logging.DEBUG)
config = Config()
#logging.debug("Client token: " + event['authorizationToken'])
logging.debug("Method ARN: " + event['methodArn'])
if event['authorizationToken'] != config.api.token:
raise Exception('Unauthorized')
principalId = 'hammer-api-user'
tmp = event['methodArn'].split(':')
apiGatewayArnTmp = tmp[5].split('/')
awsAccountId = tmp[4]
policy = AuthPolicy(principalId, awsAccountId)
policy.restApiId = apiGatewayArnTmp[0]
policy.region = tmp[3]
policy.stage = apiGatewayArnTmp[1]
# a quick hack to allow GET calls to /identify/{request_id}, request_id is hex string
# rewrite this solution to more generic variant
if len(apiGatewayArnTmp) == 5:
full_path = '/identify/' + apiGatewayArnTmp[4]
policy.allowMethod(HttpVerb.GET, full_path)
policy.allowMethod(HttpVerb.POST, '/identify')
policy.allowMethod(HttpVerb.POST, '/remediate')
authResponse = policy.build()
logging.debug(jsonDumps(authResponse))
return authResponse
def get_tc_list(module_name):
"""
Returns a list of test cases names (strings) to be executed by calling find_tc
@param module_name (string) - The module (dot py file) to pull all the test_cases from
@return tc_list (string List)
"""
# LOADING THE TC CONFIG
config = Config()
tc_list = list()
# REGEX TO FIND TEST CASES IN THE TEST CLASS
comp = re.compile("(test_)([a-z_0-9]+?)(\d{4}$)")
tmp_list = module_name.split('_')
# CONVERT THE MODULE NAME TO CLASS NAME, MUST FOLLOW THIS FORMAT
class_name = tmp_list[0].upper() + tmp_list[1][0].upper() + tmp_list[1][1:]
# LOAD THE TEST CASE YAML
with open(config['test_case_file'], 'rb') as f:
test_case_config = yaml.load(f)
# FIND THE PARENT MODULE THAT HAS THE TEST SUITE MODULE
for m_name in test_case_config:
if module_name in test_case_config[m_name]:
# THIS IS THE RIGHT MODULE THAT WE ARE LOOKING FOR SO IMPORT THE SUB MODULE
idx = test_case_config[m_name].index(module_name)
module = importlib.import_module(
f"test_cases.{m_name}.{test_case_config[m_name][idx]}")
test_class = getattr(module, class_name)
for name in test_class.__dict__:
if comp.match(name):
tc_list.append(name)
#tc_list.append(tmp_data)
return tc_list
def get_scan_results(request_id):
config = Config()
main_account = Account(region=config.aws.region)
api_table = main_account.resource("dynamodb").Table(
config.api.ddb_table_name)
request_info = DDB.get_request_data(api_table, request_id)
if not request_info:
status_code = 404
body = {"message": "Request id has not been found."}
elif request_info['progress'] == request_info['total']:
status_code = 200
body = {
"scan_status": "COMPLETE",
"scan_results": collect_results(request_info, main_account)
}
elif time.time() - request_info['updated'] <= 300:
status_code = 200
body = {"scan_status": "IN_PROGRESS"}
else:
status_code = 200
body = {"scan_status": "FAILED"}
return {
"statusCode": status_code,
"body": json.dumps(body, indent=4, default=utility.jsonEncoder)
}
def generate(
model_dir: Path,
model_iteration: Optional[int],
model_config: Optional[Path],
output_dir: Path,
use_gpu: bool,
):
if model_config is None:
model_config = model_dir / "config.yaml"
output_dir.mkdir(exist_ok=True)
save_arguments(output_dir / "arguments.yaml", generate, locals())
config = Config.from_dict(yaml.safe_load(model_config.open()))
model_path = _get_predictor_model_path(
model_dir=model_dir,
iteration=model_iteration,
)
generator = Generator(
config=config,
predictor=model_path,
use_gpu=use_gpu,
)
dataset = create_dataset(config.dataset)["test"]
for data in tqdm(dataset, desc="generate"):
target = data["target"]
output = generator.generate(data["feature"])
def lambda_handler(event, context):
""" Lambda handler to initiate to find S3 bucket public access in ACL """
set_logging(level=logging.INFO)
logging.debug("Initiating S3 acls checking")
try:
sns_arn = os.environ["SNS_S3_ACL_ARN"]
config = Config()
if not config.s3acl.enabled:
logging.debug("S3 acls checking disabled")
return
logging.debug("Iterating over each account to initiate s3 acls check")
for account_id, account_name in config.s3acl.accounts.items():
payload = {
"account_id": account_id,
"account_name": account_name,
}
logging.debug(f"Initiating s3 acls checking for '{account_name}'")
Sns.publish(sns_arn, payload)
except Exception:
logging.exception("Error occurred while initiation of S3 acl checking")
return
logging.debug("S3 acls checking initiation done")
def create_json_info():
"""
@return:
"""
config = Config()
# Get the OS, dist (7, 2012, minty, ubuntu), and bit (32 or 64) of the running system
(opsys, ip_addr, dist, bit, hostname) = get_os_info()
# Assign the values to the TestRun so that they are available in the xml report
config['os'] = opsys
config['ip'] = ip_addr
config['dist'] = dist
config['bit'] = bit
config['hostname'] = hostname
print("Hostname is {}".format(hostname))
output_file = config['json_file_path']
test_run_dict = dict()
test_run_dict['name'] = config['hostname']
test_run_dict['run_id'] = config['run_id']
test_run_dict['os'] = config['os']
test_run_dict['ip'] = config['ip']
test_run_dict['browser'] = config['browser']
test_run_dict['dist'] = config['dist']
test_run_dict['ip'] = config['ip']
test_run_dict['test_cases'] = []
if not os.path.exists(config['output_dir']):
os.mkdir(config['output_dir'])
with open(output_file, 'w') as outfile:
outfile.write(json.dumps(test_run_dict, sort_keys=True, indent=4))
def __init__(self):
self.config = Config()
self.enabled = self.config.aws.ddb_backup_enabled
self.retention_period = self.config.aws.ddb_backup_retention
self.account = Account(region=self.config.aws.region)
self.ddb_client = self.account.client('dynamodb')
self.ddb_resource = self.account.resource('dynamodb')
self.now = datetime.now(timezone.utc)
# used as a part of backup name
self.today = self.now.strftime("%Y-%m-%d")
def test_config_compute_parsed():
dataset, source, destination, info = Config(
f"{Path(__file__).parent}/data/socrata.yml").compute_parsed
assert dataset["source"] == source
assert dataset["info"] == info
assert dataset["destination"] == destination
assert "url" in list(source.keys())
assert "options" in list(source.keys())
assert "geometry" in list(source.keys())
assert "fields" in list(destination.keys())
assert "options" in list(destination.keys())
assert "geometry" in list(destination.keys())
def get_test_metadata(test_list):
"""
:param test_list:
:return:
"""
config = Config()
metadata_list = list()
# GET THE TEST ID
for test_name in test_list:
# APPEND test_ IF NOT PRESENT
test_name = test_name.replace("test_", "")
full_test_name = 'test_' + test_name if test_name[:5] != 'test_' else test_name
test_object = find_tc(full_test_name)
assert test_object, "Failed to find any test case named '{}'".format(
test_name)
test_id_list = re.findall("test_([a-zA-Z_]+?\d{4})", full_test_name)
assert len(test_id_list) == 1, "Test name does not follow pattern"
# GET THE DOC STRING INTO A DICTIONARY
doc_dict = test_object.doc_dict
test_cases = {"": ""}
try:
if 'note' in doc_dict and type(eval(doc_dict['note'])) is dict:
test_cases = eval(doc_dict['note'])
except Exception:
print("Note is not a dictionary")
# MAKE A TESTCASE FOR EACH TEST IF A DICTIONARY IS ADDED
for key, val in test_cases.iteritems():
test_metadata = test_object.doc_dict.copy()
test_metadata['name'] = "{}{}".format(test_metadata['title'],
" %s" % key if key else "")
test_metadata['script_id'] = "{}{}".format(
test_name, " %s" % val if val else "")
comp = re.compile("# (.+)\n")
test_metadata['steps'] = [
x.capitalize().replace('"', '') for x in (comp.findall(
inspect.getsource(getattr(test_object, full_test_name))))
]
if 'priority' not in test_metadata:
test_metadata['priority'] = 3
if 'result' not in test_metadata:
test_metadata['result'] = test_metadata['test'].replace(
'"', '')
metadata_list.append(test_metadata)
return metadata_list
def property_notifications():
if request.method == "GET":
return jsonify({'status': 'success', 'message': 'notification user successfully received'}), 200
else:
notifications_data = request.get_json()
print(notifications_data)
if "email" in notifications_data:
email = notifications_data['email']
else:
return jsonify({'status': 'failure', 'message': 'unable to located email'}), 500
if "name" in notifications_data:
name = notifications_data['name']
else:
return jsonify({'status': 'failure', 'message': 'unable to located name'}), 500
if "uid" in notifications_data:
uid = notifications_data['surname']
else:
return jsonify({'status': 'failure', 'message': 'unable to located uid'}), 500
return AdminView(config=Config()).add_notifications_user(email=email, name=name, uid=uid)
def find_tc(test_name):
"""
Finds the test case test from the test case object by name
@param: test_name(string) - the name of the test case, such as 'test_0001'
@param: config(Configuration Object) The config object to pass to the test case once found
@return: Unit Test test case object
@note: will find the test case based on the name regardless of where in the test_cases package it's located
"""
config = Config()
with open(config['test_case_file'], 'rb') as f:
tc_config = yaml.load(f)
for k, v in tc_config.items():
for package in v:
module = importlib.import_module(f'test_cases.{k}.{package}')
for _, oj in inspect.getmembers(module):
if inspect.isclass(oj):
if hasattr(oj, test_name):
return oj(test_name)
return None
def js_snippet(self, snippet_title, format_text=None, *args):
"""
Runs a javascript snippet
@param snippet_title (string) - the name of the snippet to execute
@param format_text (string) -
@param args (Variable arguments) - the arguments to pass into the execute script
@returns the return value of the javascript snippet
"""
config = Config()
logger.debug("Snippet: {} beginning...".format(snippet_title))
# Read in the JSSnippet
script = ""
snippet_path = config['js_snippet_dir']
with open(os.path.join(snippet_path, snippet_title), 'r') as f:
script += f.read()
if format_text is not None:
script = script % format_text
ret_val = self.execute_script(script, True, False, *args)
logger.debug("Snippet: {} complete...".format(snippet_title))
return ret_val
def __init__(self, name):
"""
Test Template Initialization initializes a TestTemplate object with some pre-configured values
@param name (string) - Name of the test to initialize
@returns TestTemplate Object
@note This setups up the verbs, assigns the config to it, calls the functions
docstring and then initializes the action
"""
config = Config()
super(TestTemplate, self).__init__(name)
self.doc_dict = None
func = getattr(self, name)
self.get_test_doc(name, func.__doc__)
# ASSIGN VALUES TO CONFIG
config['name'] = self.doc_dict['name']
logger.info("NAME: {}".format(config['name']))
config['title'] = self.doc_dict['title']
logger.info("TITLE: {}".format(config['title']))
config['test'] = self.doc_dict['test']
logger.info("TEST: {}".format(config['test']))
config['bug'] = self.doc_dict['bug']
logger.info("BUG: {}".format(config['bug']))
except Exception:
logging.exception(
f"Error occurred while updating bucket '{bucket_name}' policy "
f"in '{account_name} / {account_id}'")
else:
logging.debug(
f"Skipping '{bucket_name}' "
f"({retention_period - no_of_days_issue_created} days before remediation)"
)
if __name__ == "__main__":
module_name = sys.modules[__name__].__loader__.name
set_logging(level=logging.DEBUG,
logfile=f"/var/log/hammer/{module_name}.log")
config = Config()
add_cw_logging(config.local.log_group,
log_stream=module_name,
level=logging.DEBUG,
region=config.aws.region)
try:
si = SingletonInstance(module_name)
except SingletonInstanceException:
logging.error(
f"Another instance of '{module_name}' is already running, quitting"
)
sys.exit(1)
parser = argparse.ArgumentParser()
parser.add_argument('--batch',
action='store_true',
def lambda_handler(event, context):
""" Lambda handler to evaluate public ami issues"""
set_logging(level=logging.INFO)
try:
payload = json.loads(event["Records"][0]["Sns"]["Message"])
account_id = payload['account_id']
account_name = payload['account_name']
# get the last region from the list to process
region = payload['regions'].pop()
# if request_id is present in payload then this lambda was called from the API
request_id = payload.get('request_id', None)
except Exception:
logging.exception(f"Failed to parse event\n{event}")
return
try:
config = Config()
main_account = Account(region=config.aws.region)
ddb_table = main_account.resource("dynamodb").Table(
config.publicAMIs.ddb_table_name)
account = Account(id=account_id,
name=account_name,
region=region,
role_name=config.aws.role_name_identification)
if account.session is None:
return
logging.debug(f"Checking for Public AMI issues for {account}")
# existing open issues for account to check if resolved
open_issues = IssueOperations.get_account_open_issues(
ddb_table, account_id, PublicAMIIssue)
# make dictionary for fast search by id
# and filter by current region
open_issues = {
issue.issue_id: issue
for issue in open_issues if issue.issue_details.region == region
}
logging.debug(f"Public AMIs in DDB:\n{open_issues.keys()}")
checker = PublicAMIChecker(account=account)
if checker.check():
for ami in checker.amis:
logging.debug(f"Checking {ami.id}")
if ami.public_access:
issue = PublicAMIIssue(account_id, ami.id)
issue.issue_details.tags = ami.tags
issue.issue_details.name = ami.name
issue.issue_details.region = region
if config.publicAMIs.in_whitelist(account_id, ami.id):
issue.status = IssueStatus.Whitelisted
else:
issue.status = IssueStatus.Open
logging.debug(
f"Setting {ami.id}/{ami.id} status {issue.status}")
IssueOperations.update(ddb_table, issue)
# remove issue id from issues_list_from_db (if exists)
# as we already checked it
open_issues.pop(ami.id, None)
logging.debug(f"Public AMIs in DDB:\n{open_issues.keys()}")
# all other unresolved issues in DDB are for removed/remediated keys
for issue in open_issues.values():
IssueOperations.set_status_resolved(ddb_table, issue)
# track the progress of API request to scan specific account/region/feature
if request_id:
api_table = main_account.resource("dynamodb").Table(
config.api.ddb_table_name)
DDB.track_progress(api_table, request_id)
except Exception:
logging.exception(
f"Failed to check AMI public access for '{account_id} ({account_name})'"
)
return
# push SNS messages until the list with regions to check is empty
if len(payload['regions']) > 0:
try:
Sns.publish(payload["sns_arn"], payload)
except Exception:
logging.exception("Failed to chain public AMI checking")
logging.debug(
f"Checked AMI public access for '{account_id} ({account_name})'")
def lambda_handler(event, context):
""" Lambda handler to evaluate public EBS snapshots """
set_logging(level=logging.INFO)
try:
payload = json.loads(event["Records"][0]["Sns"]["Message"])
account_id = payload['account_id']
account_name = payload['account_name']
# get the last region from the list to process
region = payload['regions'].pop()
except Exception:
logging.exception(f"Failed to parse event\n{event}")
return
try:
config = Config()
main_account = Account(region=config.aws.region)
ddb_table = main_account.resource("dynamodb").Table(
config.ebsSnapshot.ddb_table_name)
account = Account(id=account_id,
name=account_name,
region=region,
role_name=config.aws.role_name_identification)
if account.session is None:
return
logging.debug(f"Checking for public EBS snapshots in {account}")
# existing open issues for account to check if resolved
open_issues = IssueOperations.get_account_open_issues(
ddb_table, account_id, EBSPublicSnapshotIssue)
# make dictionary for fast search by id
# and filter by current region
open_issues = {
issue.issue_id: issue
for issue in open_issues if issue.issue_details.region == region
}
logging.debug(f"Public EBS snapshots in DDB:\n{open_issues.keys()}")
checker = EBSPublicSnapshotsChecker(account=account)
if checker.check():
for snapshot in checker.snapshots:
if snapshot.public:
issue = EBSPublicSnapshotIssue(account_id, snapshot.id)
issue.issue_details.region = snapshot.account.region
issue.issue_details.volume_id = snapshot.volume_id
issue.issue_details.tags = snapshot.tags
if config.ebsSnapshot.in_whitelist(account_id,
snapshot.id):
issue.status = IssueStatus.Whitelisted
else:
issue.status = IssueStatus.Open
logging.debug(
f"Setting {snapshot.id} status {issue.status}")
IssueOperations.update(ddb_table, issue)
# remove issue id from issues_list_from_db (if exists)
# as we already checked it
open_issues.pop(snapshot.id, None)
logging.debug(
f"Public EBS snapshots in DDB:\n{open_issues.keys()}")
# all other unresolved issues in DDB are for removed/remediated EBS snapshots
for issue in open_issues.values():
IssueOperations.set_status_resolved(ddb_table, issue)
except Exception:
logging.exception(
f"Failed to check public EBS snapshots in '{region}' for '{account_id} ({account_name})'"
)
# push SNS messages until the list with regions to check is empty
if len(payload['regions']) > 0:
try:
Sns.publish(payload["sns_arn"], payload)
except Exception:
logging.exception("Failed to chain public EBS snapshots checking")
logging.debug(
f"Checked public EBS snapshots in '{region}' for '{account_id} ({account_name})'"
)
import logging
import sys
from library.logger import set_logging, add_cw_logging
from library.config import Config
from slackbot import settings
from slackbot.bot import Bot
def main():
bot = Bot()
bot.run()
if __name__ == "__main__":
module_name = sys.modules[__name__].__loader__.name
set_logging(level=logging.WARNING, logfile=f"/var/log/hammer/{module_name}.log")
settings.config = Config()
if not settings.config.slack.enabled:
sys.exit(0)
settings.API_TOKEN = settings.config.slack.api_token
settings.PLUGINS = ['bot.commands']
add_cw_logging(settings.config.local.log_group,
log_stream=module_name,
level=logging.WARNING,
region=settings.config.aws.region)
main()
def __init__(self):
self.config = Config()
def lambda_handler(event, context):
""" Lambda handler to evaluate insecure services """
set_logging(level=logging.INFO)
try:
payload = json.loads(event["Records"][0]["Sns"]["Message"])
account_id = payload['account_id']
account_name = payload['account_name']
# get the last region from the list to process
region = payload['regions'].pop()
except Exception:
logging.exception(f"Failed to parse event\n{event}")
return
try:
config = Config()
main_account = Account(region=config.aws.region)
ddb_table = main_account.resource("dynamodb").Table(
config.sg.ddb_table_name)
account = Account(id=account_id,
name=account_name,
region=region,
role_name=config.aws.role_name_identification)
if account.session is None:
return
logging.debug(f"Checking for insecure services in {account}")
# existing open issues for account to check if resolved
open_issues = IssueOperations.get_account_open_issues(
ddb_table, account_id, SecurityGroupIssue)
# make dictionary for fast search by id
# and filter by current region
open_issues = {
issue.issue_id: issue
for issue in open_issues if issue.issue_details.region == region
}
logging.debug(f"Security groups in DDB:\n{open_issues.keys()}")
checker = SecurityGroupsChecker(
account=account, restricted_ports=config.sg.restricted_ports)
if checker.check():
for sg in checker.groups:
logging.debug(f"Checking {sg.name} ({sg.id})")
if not sg.restricted:
# TODO: move instances detection for security group from reporting to identification
#ec2_instances = EC2Operations.get_instance_details_of_sg_associated(account.client("ec2"), sg.id)
#logging.debug(f"associated ec2 instances: {ec2_instances}")
issue = SecurityGroupIssue(account_id, sg.id)
issue.issue_details.name = sg.name
issue.issue_details.region = sg.account.region
issue.issue_details.tags = sg.tags
issue.issue_details.status = sg.status.value
for perm in sg.permissions:
for ip_range in perm.ip_ranges:
if not ip_range.restricted:
issue.add_perm(perm.protocol, perm.from_port,
perm.to_port, ip_range.cidr,
ip_range.status)
if config.sg.in_whitelist(
account_id, sg.name) or config.sg.in_whitelist(
account_id, sg.id):
issue.status = IssueStatus.Whitelisted
else:
issue.status = IssueStatus.Open
logging.debug(f"Setting {sg.id} status {issue.status}")
IssueOperations.update(ddb_table, issue)
# remove issue id from issues_list_from_db (if exists)
# as we already checked it
open_issues.pop(sg.id, None)
logging.debug(f"Security groups in DDB:\n{open_issues.keys()}")
# all other unresolved issues in DDB are for removed/remediated security groups
for issue in open_issues.values():
IssueOperations.set_status_resolved(ddb_table, issue)
except Exception:
logging.exception(
f"Failed to check insecure services in '{region}' for '{account_id} ({account_name})'"
)
# push SNS messages until the list with regions to check is empty
if len(payload['regions']) > 0:
try:
Sns.publish(payload["sns_arn"], payload)
except Exception:
logging.exception("Failed to chain insecure services checking")
logging.debug(
f"Checked insecure services in '{region}' for '{account_id} ({account_name})'"
)
def create_trainer(
config_dict: Dict[str, Any],
output: Path,
):
# config
config = Config.from_dict(config_dict)
config.add_git_info()
output.mkdir(exist_ok=True, parents=True)
with (output / "config.yaml").open(mode="w") as f:
yaml.safe_dump(config.to_dict(), f)
# model
networks = create_network(config.network)
model = Model(model_config=config.model, networks=networks)
if config.train.weight_initializer is not None:
init_weights(model, name=config.train.weight_initializer)
device = torch.device("cuda") if config.train.use_gpu else torch.device(
"cpu")
model.to(device)
# dataset
_create_iterator = partial(
create_iterator,
batch_size=config.train.batch_size,
eval_batch_size=config.train.eval_batch_size,
num_processes=config.train.num_processes,
use_multithread=config.train.use_multithread,
)
datasets = create_dataset(config.dataset)
train_iter = _create_iterator(datasets["train"], for_train=True)
test_iter = _create_iterator(datasets["test"], for_train=False)
eval_iter = _create_iterator(datasets["eval"], for_train=False)
warnings.simplefilter("error", MultiprocessIterator.TimeoutWarning)
# optimizer
optimizer = make_optimizer(config_dict=config.train.optimizer, model=model)
# updater
if not config.train.use_amp:
updater = StandardUpdater(
iterator=train_iter,
optimizer=optimizer,
model=model,
device=device,
)
else:
updater = AmpUpdater(
iterator=train_iter,
optimizer=optimizer,
model=model,
device=device,
)
# trainer
trigger_log = (config.train.log_iteration, "iteration")
trigger_eval = (config.train.eval_iteration, "iteration")
trigger_snapshot = (config.train.snapshot_iteration, "iteration")
trigger_stop = ((config.train.stop_iteration, "iteration")
if config.train.stop_iteration is not None else None)
trainer = Trainer(updater, stop_trigger=trigger_stop, out=output)
ext = extensions.Evaluator(test_iter, model, device=device)
trainer.extend(ext, name="test", trigger=trigger_log)
if config.train.stop_iteration is not None:
saving_model_num = int(config.train.stop_iteration /
config.train.eval_iteration / 10)
else:
saving_model_num = 10
ext = extensions.snapshot_object(
networks.predictor,
filename="predictor_{.updater.iteration}.pth",
n_retains=saving_model_num,
)
trainer.extend(
ext,
trigger=LowValueTrigger("test/main/loss", trigger=trigger_eval),
)
trainer.extend(extensions.FailOnNonNumber(), trigger=trigger_log)
trainer.extend(extensions.observe_lr(), trigger=trigger_log)
trainer.extend(extensions.LogReport(trigger=trigger_log))
trainer.extend(
extensions.PrintReport(["iteration", "main/loss", "test/main/loss"]),
trigger=trigger_log,
)
ext = TensorboardReport(writer=SummaryWriter(Path(output)))
trainer.extend(ext, trigger=trigger_log)
if config.project.category is not None:
ext = WandbReport(
config_dict=config.to_dict(),
project_category=config.project.category,
project_name=config.project.name,
output_dir=output.joinpath("wandb"),
)
trainer.extend(ext, trigger=trigger_log)
(output / "struct.txt").write_text(repr(model))
if trigger_stop is not None:
trainer.extend(extensions.ProgressBar(trigger_stop))
ext = extensions.snapshot_object(
trainer,
filename="trainer_{.updater.iteration}.pth",
n_retains=1,
autoload=True,
)
trainer.extend(ext, trigger=trigger_snapshot)
return trainer
def __init__(self, config=None):
self.config = Config() if config is None else config
self.sc = SlackClient(self.config.slack.api_token)
self.slackUser = "******"
def lambda_handler(event, context):
""" Lambda handler to evaluate s3 buckets acl """
set_logging(level=logging.INFO)
try:
payload = json.loads(event["Records"][0]["Sns"]["Message"])
account_id = payload['account_id']
account_name = payload['account_name']
# if request_id is present in payload then this lambda was called from the API
request_id = payload.get('request_id', None)
except Exception:
logging.exception(f"Failed to parse event\n{event}")
return
try:
config = Config()
main_account = Account(region=config.aws.region)
ddb_table = main_account.resource("dynamodb").Table(
config.s3acl.ddb_table_name)
account = Account(id=account_id,
name=account_name,
role_name=config.aws.role_name_identification)
if account.session is None:
return
logging.debug(f"Checking for public S3 ACLs in {account}")
# existing open issues for account to check if resolved
open_issues = IssueOperations.get_account_open_issues(
ddb_table, account_id, S3AclIssue)
# make dictionary for fast search by id
# and filter by current region
open_issues = {issue.issue_id: issue for issue in open_issues}
logging.debug(f"S3 in DDB:\n{open_issues.keys()}")
checker = S3BucketsAclChecker(account=account)
if not checker.check():
return
for bucket in checker.buckets:
logging.debug(f"Checking {bucket.name}")
if bucket.public:
issue = S3AclIssue(account_id, bucket.name)
issue.issue_details.owner = bucket.owner
issue.issue_details.public_acls = bucket.get_public_acls()
issue.issue_details.tags = bucket.tags
if config.s3acl.in_whitelist(account_id, bucket.name):
issue.status = IssueStatus.Whitelisted
else:
issue.status = IssueStatus.Open
logging.debug(f"Setting {bucket.name} status {issue.status}")
IssueOperations.update(ddb_table, issue)
# remove issue id from issues_list_from_db (if exists)
# as we already checked it
open_issues.pop(bucket.name, None)
logging.debug(f"S3 in DDB:\n{open_issues.keys()}")
# all other unresolved issues in DDB are for removed/remediated buckets
for issue in open_issues.values():
IssueOperations.set_status_resolved(ddb_table, issue)
if request_id:
api_table = main_account.resource("dynamodb").Table(
config.api.ddb_table_name)
DDB.track_progress(api_table, request_id)
except Exception:
logging.exception(
f"Failed to check s3 acls for '{account_id} ({account_name})'")
return
logging.debug(f"Checked s3 acls for '{account_id} ({account_name})'")
