def __init__(self, accounts=None, alert_accounts=None, debug=False):
self.account_watchers = {}
self.account_alerters = {}
if not alert_accounts:
alert_accounts = accounts
for account in accounts:
self.account_watchers[account] = [
(SQS(accounts=[account], debug=debug), None),
(ELB(accounts=[account], debug=debug), None),
(IAMSSL(accounts=[account], debug=debug), None),
(RDSSecurityGroup(accounts=[account], debug=debug),
RDSSecurityGroupAuditor(accounts=[account], debug=debug)),
(SecurityGroup(accounts=[account], debug=debug),
SecurityGroupAuditor(accounts=[account], debug=debug)),
(S3(accounts=[account],
debug=debug), S3Auditor(accounts=[account], debug=debug)),
(IAMUser(accounts=[account], debug=debug),
IAMUserAuditor(accounts=[account], debug=debug)),
(IAMGroup(accounts=[account], debug=debug), None),
(IAMRole(accounts=[account], debug=debug), None),
(Keypair(accounts=[account], debug=debug), None),
(SNS(accounts=[account],
debug=debug), SNSAuditor(accounts=[account], debug=debug))
]
if account in alert_accounts:
self.account_alerters[account] = Alerter(
watchers_auditors=self.account_watchers[account],
account=account)
def __init__(self, account=None, alert_accounts=None, debug=False):
self.account_watchers = {}
self.account_alerters = {}
if not alert_accounts:
alert_accounts = [account]
self.account_watchers[account] = []
for monitor in all_monitors(account, debug):
self.account_watchers[account].append((monitor))
if account in alert_accounts:
self.account_alerters[account] = Alerter(watchers_auditors=self.account_watchers[account], account=account)
def __init__(self, accounts=None, alert_accounts=None, debug=False):
self.account_watchers = {}
self.account_alerters = {}
if not alert_accounts:
alert_accounts = accounts
for account in accounts:
self.account_watchers[account] = []
for monitor in all_monitors():
watcher = monitor.watcher_class(accounts=[account],
debug=debug)
auditor = monitor.auditor_class(
accounts=[account],
debug=debug) if monitor.has_auditor() else None
self.account_watchers[account].append((watcher, auditor))
if account in alert_accounts:
self.account_alerters[account] = Alerter(
watchers_auditors=self.account_watchers[account],
account=account)
def reporter_logic(account_name, technology_name):
"""Logic for the run change reporter"""
try:
# Before doing anything... Look for orphaned items for this given technology. If they exist, then delete them:
fix_orphaned_deletions(account_name, technology_name)
# Watch and Audit:
monitors = find_changes(account_name, technology_name)
# Alert:
app.logger.info("[ ] Sending alerts (if applicable) for account: {}, technology: {}".format(account_name,
technology_name))
Alerter(monitors, account=account_name).report()
except (OperationalError, InvalidRequestError, StatementError) as e:
app.logger.exception("[X] Database error processing account %s - technology %s cleaning up session.",
account_name, technology_name)
db.session.remove()
store_exception("scheduler-task-account-tech", None, e)
raise e
def __init__(self, account=None, debug=False):
self.all_monitors = all_monitors(account, debug)
self.account_alerter = Alerter(watchers_auditors=self.all_monitors,
account=account)
class Reporter(object):
"""Sets up all watchers and auditors and the alerters"""
def __init__(self, account=None, debug=False):
self.all_monitors = all_monitors(account, debug)
self.account_alerter = Alerter(watchers_auditors=self.all_monitors,
account=account)
def run(self, account, interval=None):
"""Starts the process of watchers -> auditors -> alerters """
app.logger.info("Starting work on account {}.".format(account))
time1 = time.time()
mons = self.get_monitors_to_run(account, interval)
watchers_with_changes = set()
for monitor in mons:
app.logger.info(
"Running slurp {} for {} ({} minutes interval)".format(
monitor.watcher.i_am_singular, account, interval))
# Batch logic needs to be handled differently:
if monitor.batch_support:
from security_monkey.scheduler import batch_logic
batch_logic(monitor, monitor.watcher, account, False)
else:
(items, exception_map) = monitor.watcher.slurp()
monitor.watcher.find_changes(items, exception_map)
if (len(monitor.watcher.created_items) > 0) or (len(
monitor.watcher.changed_items) > 0):
watchers_with_changes.add(monitor.watcher.index)
monitor.watcher.save()
db_account = get_account_by_name(account)
for monitor in self.all_monitors:
# Skip over batched items, since they are done:
if monitor.batch_support:
continue
for auditor in monitor.auditors:
if auditor.applies_to_account(db_account):
items_to_audit = self.get_items_to_audit(
monitor.watcher, auditor, watchers_with_changes)
app.logger.info("Running audit {} for {}".format(
monitor.watcher.index, account))
try:
auditor.items = items_to_audit
auditor.audit_objects()
auditor.save_issues()
except Exception as e:
store_exception('reporter-run-auditor',
(auditor.index, account), e)
continue
time2 = time.time()
app.logger.info('Run Account %s took %0.1f s' % (account,
(time2 - time1)))
self.account_alerter.report()
db.session.close()
def get_monitors_to_run(self, account, interval=None):
"""
Return a list of (watcher, auditor) enabled for a specific account,
optionally filtered by interval time
"""
mons = []
if interval:
for monitor in self.all_monitors:
if monitor.watcher and interval == monitor.watcher.get_interval(
):
mons.append(monitor)
else:
mons = self.all_monitors
return mons
def get_intervals(self, account):
""" Returns current intervals for watchers """
buckets = []
for monitor in self.all_monitors:
if monitor.watcher:
interval = monitor.watcher.get_interval()
if interval not in buckets:
buckets.append(interval)
return buckets
def get_items_to_audit(self, watcher, auditor, watchers_with_changes):
"""
Returns the items that have changed if there are no changes in dependencies,
otherwise returns all slurped items for reauditing
"""
watcher.full_audit_list = None
if auditor.support_watcher_indexes:
for support_watcher_index in auditor.support_watcher_indexes:
if support_watcher_index in watchers_with_changes:
app.logger.debug(
"Upstream watcher changed {}. reauditing {}".format(
support_watcher_index, watcher.index))
watcher.full_audit_list = auditor.read_previous_items()
if auditor.support_auditor_indexes:
for support_auditor_index in auditor.support_auditor_indexes:
if support_auditor_index in watchers_with_changes:
app.logger.debug(
"Upstream auditor changed {}. reauditing {}".format(
support_auditor_index, watcher.index))
watcher.full_audit_list = auditor.read_previous_items()
if watcher.full_audit_list:
return watcher.full_audit_list
return [item for item in watcher.created_items + watcher.changed_items]
def __init__(self, account=None, debug=False):
self.all_monitors = all_monitors(account, debug)
self.account_alerter = Alerter(watchers_auditors=self.all_monitors, account=account)
class Reporter(object):
"""Sets up all watchers and auditors and the alerters"""
def __init__(self, account=None, debug=False):
self.all_monitors = all_monitors(account, debug)
self.account_alerter = Alerter(watchers_auditors=self.all_monitors, account=account)
def run(self, account, interval=None):
"""Starts the process of watchers -> auditors -> alerters """
app.logger.info("Starting work on account {}.".format(account))
time1 = time.time()
mons = self.get_monitors_to_run(account, interval)
watchers_with_changes = set()
for monitor in mons:
app.logger.info("Running slurp {} for {} ({} minutes interval)".format(monitor.watcher.i_am_singular, account, interval))
# Batch logic needs to be handled differently:
if monitor.batch_support:
from security_monkey.scheduler import batch_logic
batch_logic(monitor, monitor.watcher, account, False)
else:
(items, exception_map) = monitor.watcher.slurp()
monitor.watcher.find_changes(items, exception_map)
if (len(monitor.watcher.created_items) > 0) or (len(monitor.watcher.changed_items) > 0):
watchers_with_changes.add(monitor.watcher.index)
monitor.watcher.save()
db_account = get_account_by_name(account)
for monitor in self.all_monitors:
# Skip over batched items, since they are done:
if monitor.batch_support:
continue
for auditor in monitor.auditors:
if auditor.applies_to_account(db_account):
items_to_audit = self.get_items_to_audit(monitor.watcher, auditor, watchers_with_changes)
app.logger.info("Running audit {} for {}".format(
monitor.watcher.index,
account))
try:
auditor.items = items_to_audit
auditor.audit_objects()
auditor.save_issues()
except Exception as e:
store_exception('reporter-run-auditor', (auditor.index, account), e)
continue
time2 = time.time()
app.logger.info('Run Account %s took %0.1f s' % (account, (time2-time1)))
self.account_alerter.report()
db.session.close()
def get_monitors_to_run(self, account, interval=None):
"""
Return a list of (watcher, auditor) enabled for a specific account,
optionally filtered by interval time
"""
mons = []
if interval:
for monitor in self.all_monitors:
if monitor.watcher and interval == monitor.watcher.get_interval():
mons.append(monitor)
else:
mons = self.all_monitors
return mons
def get_intervals(self, account):
""" Returns current intervals for watchers """
buckets = []
for monitor in self.all_monitors:
if monitor.watcher:
interval = monitor.watcher.get_interval()
if interval not in buckets:
buckets.append(interval)
return buckets
def get_items_to_audit(self, watcher, auditor, watchers_with_changes):
"""
Returns the items that have changed if there are no changes in dependencies,
otherwise returns all slurped items for reauditing
"""
watcher.full_audit_list = None
if auditor.support_watcher_indexes:
for support_watcher_index in auditor.support_watcher_indexes:
if support_watcher_index in watchers_with_changes:
app.logger.debug("Upstream watcher changed {}. reauditing {}".format(
support_watcher_index, watcher.index))
watcher.full_audit_list = auditor.read_previous_items()
if auditor.support_auditor_indexes:
for support_auditor_index in auditor.support_auditor_indexes:
if support_auditor_index in watchers_with_changes:
app.logger.debug("Upstream auditor changed {}. reauditing {}".format(
support_auditor_index, watcher.index))
watcher.full_audit_list = auditor.read_previous_items()
if watcher.full_audit_list:
return watcher.full_audit_list
return [item for item in watcher.created_items + watcher.changed_items]
def test_report_batch_changes(self):
from security_monkey.alerter import Alerter
from security_monkey.reporter import Reporter
from security_monkey.datastore import Item, ItemRevision, ItemAudit
from security_monkey.monitors import Monitor
from security_monkey.watchers.iam.iam_role import IAMRole
from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
account_type_result = AccountType.query.filter(
AccountType.name == "AWS").one()
db.session.add(account_type_result)
db.session.commit()
test_account = Account(identifier="012345678910",
name="TEST_ACCOUNT",
account_type_id=account_type_result.id,
notes="TEST_ACCOUNT1",
third_party=False,
active=True)
watcher = IAMRole(accounts=[test_account.name])
db.session.commit()
watcher.batched_size = 3 # should loop 4 times
self.add_roles()
# Set up the monitor:
batched_monitor = Monitor(IAMRole, test_account)
batched_monitor.watcher = watcher
batched_monitor.auditors = [
IAMRoleAuditor(accounts=[test_account.name])
]
# Set up the Reporter:
import security_monkey.reporter
old_all_monitors = security_monkey.reporter.all_monitors
security_monkey.reporter.all_monitors = lambda x, y: []
test_reporter = Reporter()
test_reporter.all_monitors = [batched_monitor]
test_reporter.account_alerter = Alerter(
watchers_auditors=test_reporter.all_monitors,
account=test_account.name)
import security_monkey.scheduler
# import security_monkey.monitors
# old_get_monitors = security_monkey.scheduler.get_monitors
security_monkey.scheduler.get_monitors = lambda x, y, z: [
batched_monitor
]
# Moto screws up the IAM Role ARN -- so we need to fix it:
original_slurp_list = watcher.slurp_list
original_slurp = watcher.slurp
def mock_slurp_list():
items, exception_map = original_slurp_list()
for item in watcher.total_list:
item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(
item["RoleName"])
return items, exception_map
def mock_slurp():
batched_items, exception_map = original_slurp()
for item in batched_items:
item.arn = "arn:aws:iam::012345678910:role/{}".format(
item.name)
item.config["Arn"] = item.arn
item.config["RoleId"] = item.name # Need this to stay the same
return batched_items, exception_map
watcher.slurp_list = mock_slurp_list
watcher.slurp = mock_slurp
test_reporter.run(account=test_account.name)
# Check that all items were added to the DB:
assert len(Item.query.all()) == 11
# Check that we have exactly 11 item revisions:
assert len(ItemRevision.query.all()) == 11
# Check that there are audit issues for all 11 items:
assert len(ItemAudit.query.all()) == 11
mock_iam().stop()
mock_sts().stop()
# Something isn't cleaning itself up properly and causing other core tests to fail.
# This is the solution:
security_monkey.reporter.all_monitors = old_all_monitors
import monitor_mock
security_monkey.scheduler.get_monitors = monitor_mock.mock_get_monitors