def __init__(self, accounts=None, alert_accounts=None, debug=False): self.account_watchers = {} self.account_alerters = {} if not alert_accounts: alert_accounts = accounts for account in accounts: self.account_watchers[account] = [ (SQS(accounts=[account], debug=debug), None), (ELB(accounts=[account], debug=debug), None), (IAMSSL(accounts=[account], debug=debug), None), (RDSSecurityGroup(accounts=[account], debug=debug), RDSSecurityGroupAuditor(accounts=[account], debug=debug)), (SecurityGroup(accounts=[account], debug=debug), SecurityGroupAuditor(accounts=[account], debug=debug)), (S3(accounts=[account], debug=debug), S3Auditor(accounts=[account], debug=debug)), (IAMUser(accounts=[account], debug=debug), IAMUserAuditor(accounts=[account], debug=debug)), (IAMGroup(accounts=[account], debug=debug), None), (IAMRole(accounts=[account], debug=debug), None), (Keypair(accounts=[account], debug=debug), None), (SNS(accounts=[account], debug=debug), SNSAuditor(accounts=[account], debug=debug)) ] if account in alert_accounts: self.account_alerters[account] = Alerter( watchers_auditors=self.account_watchers[account], account=account)
def __init__(self, account=None, alert_accounts=None, debug=False): self.account_watchers = {} self.account_alerters = {} if not alert_accounts: alert_accounts = [account] self.account_watchers[account] = [] for monitor in all_monitors(account, debug): self.account_watchers[account].append((monitor)) if account in alert_accounts: self.account_alerters[account] = Alerter(watchers_auditors=self.account_watchers[account], account=account)
def __init__(self, accounts=None, alert_accounts=None, debug=False): self.account_watchers = {} self.account_alerters = {} if not alert_accounts: alert_accounts = accounts for account in accounts: self.account_watchers[account] = [] for monitor in all_monitors(): watcher = monitor.watcher_class(accounts=[account], debug=debug) auditor = monitor.auditor_class( accounts=[account], debug=debug) if monitor.has_auditor() else None self.account_watchers[account].append((watcher, auditor)) if account in alert_accounts: self.account_alerters[account] = Alerter( watchers_auditors=self.account_watchers[account], account=account)
def reporter_logic(account_name, technology_name): """Logic for the run change reporter""" try: # Before doing anything... Look for orphaned items for this given technology. If they exist, then delete them: fix_orphaned_deletions(account_name, technology_name) # Watch and Audit: monitors = find_changes(account_name, technology_name) # Alert: app.logger.info("[ ] Sending alerts (if applicable) for account: {}, technology: {}".format(account_name, technology_name)) Alerter(monitors, account=account_name).report() except (OperationalError, InvalidRequestError, StatementError) as e: app.logger.exception("[X] Database error processing account %s - technology %s cleaning up session.", account_name, technology_name) db.session.remove() store_exception("scheduler-task-account-tech", None, e) raise e
def __init__(self, account=None, debug=False): self.all_monitors = all_monitors(account, debug) self.account_alerter = Alerter(watchers_auditors=self.all_monitors, account=account)
class Reporter(object): """Sets up all watchers and auditors and the alerters""" def __init__(self, account=None, debug=False): self.all_monitors = all_monitors(account, debug) self.account_alerter = Alerter(watchers_auditors=self.all_monitors, account=account) def run(self, account, interval=None): """Starts the process of watchers -> auditors -> alerters """ app.logger.info("Starting work on account {}.".format(account)) time1 = time.time() mons = self.get_monitors_to_run(account, interval) watchers_with_changes = set() for monitor in mons: app.logger.info( "Running slurp {} for {} ({} minutes interval)".format( monitor.watcher.i_am_singular, account, interval)) # Batch logic needs to be handled differently: if monitor.batch_support: from security_monkey.scheduler import batch_logic batch_logic(monitor, monitor.watcher, account, False) else: (items, exception_map) = monitor.watcher.slurp() monitor.watcher.find_changes(items, exception_map) if (len(monitor.watcher.created_items) > 0) or (len( monitor.watcher.changed_items) > 0): watchers_with_changes.add(monitor.watcher.index) monitor.watcher.save() db_account = get_account_by_name(account) for monitor in self.all_monitors: # Skip over batched items, since they are done: if monitor.batch_support: continue for auditor in monitor.auditors: if auditor.applies_to_account(db_account): items_to_audit = self.get_items_to_audit( monitor.watcher, auditor, watchers_with_changes) app.logger.info("Running audit {} for {}".format( monitor.watcher.index, account)) try: auditor.items = items_to_audit auditor.audit_objects() auditor.save_issues() except Exception as e: store_exception('reporter-run-auditor', (auditor.index, account), e) continue time2 = time.time() app.logger.info('Run Account %s took %0.1f s' % (account, (time2 - time1))) self.account_alerter.report() db.session.close() def get_monitors_to_run(self, account, interval=None): """ Return a list of (watcher, auditor) enabled for a specific account, optionally filtered by interval time """ mons = [] if interval: for monitor in self.all_monitors: if monitor.watcher and interval == monitor.watcher.get_interval( ): mons.append(monitor) else: mons = self.all_monitors return mons def get_intervals(self, account): """ Returns current intervals for watchers """ buckets = [] for monitor in self.all_monitors: if monitor.watcher: interval = monitor.watcher.get_interval() if interval not in buckets: buckets.append(interval) return buckets def get_items_to_audit(self, watcher, auditor, watchers_with_changes): """ Returns the items that have changed if there are no changes in dependencies, otherwise returns all slurped items for reauditing """ watcher.full_audit_list = None if auditor.support_watcher_indexes: for support_watcher_index in auditor.support_watcher_indexes: if support_watcher_index in watchers_with_changes: app.logger.debug( "Upstream watcher changed {}. reauditing {}".format( support_watcher_index, watcher.index)) watcher.full_audit_list = auditor.read_previous_items() if auditor.support_auditor_indexes: for support_auditor_index in auditor.support_auditor_indexes: if support_auditor_index in watchers_with_changes: app.logger.debug( "Upstream auditor changed {}. reauditing {}".format( support_auditor_index, watcher.index)) watcher.full_audit_list = auditor.read_previous_items() if watcher.full_audit_list: return watcher.full_audit_list return [item for item in watcher.created_items + watcher.changed_items]
class Reporter(object): """Sets up all watchers and auditors and the alerters""" def __init__(self, account=None, debug=False): self.all_monitors = all_monitors(account, debug) self.account_alerter = Alerter(watchers_auditors=self.all_monitors, account=account) def run(self, account, interval=None): """Starts the process of watchers -> auditors -> alerters """ app.logger.info("Starting work on account {}.".format(account)) time1 = time.time() mons = self.get_monitors_to_run(account, interval) watchers_with_changes = set() for monitor in mons: app.logger.info("Running slurp {} for {} ({} minutes interval)".format(monitor.watcher.i_am_singular, account, interval)) # Batch logic needs to be handled differently: if monitor.batch_support: from security_monkey.scheduler import batch_logic batch_logic(monitor, monitor.watcher, account, False) else: (items, exception_map) = monitor.watcher.slurp() monitor.watcher.find_changes(items, exception_map) if (len(monitor.watcher.created_items) > 0) or (len(monitor.watcher.changed_items) > 0): watchers_with_changes.add(monitor.watcher.index) monitor.watcher.save() db_account = get_account_by_name(account) for monitor in self.all_monitors: # Skip over batched items, since they are done: if monitor.batch_support: continue for auditor in monitor.auditors: if auditor.applies_to_account(db_account): items_to_audit = self.get_items_to_audit(monitor.watcher, auditor, watchers_with_changes) app.logger.info("Running audit {} for {}".format( monitor.watcher.index, account)) try: auditor.items = items_to_audit auditor.audit_objects() auditor.save_issues() except Exception as e: store_exception('reporter-run-auditor', (auditor.index, account), e) continue time2 = time.time() app.logger.info('Run Account %s took %0.1f s' % (account, (time2-time1))) self.account_alerter.report() db.session.close() def get_monitors_to_run(self, account, interval=None): """ Return a list of (watcher, auditor) enabled for a specific account, optionally filtered by interval time """ mons = [] if interval: for monitor in self.all_monitors: if monitor.watcher and interval == monitor.watcher.get_interval(): mons.append(monitor) else: mons = self.all_monitors return mons def get_intervals(self, account): """ Returns current intervals for watchers """ buckets = [] for monitor in self.all_monitors: if monitor.watcher: interval = monitor.watcher.get_interval() if interval not in buckets: buckets.append(interval) return buckets def get_items_to_audit(self, watcher, auditor, watchers_with_changes): """ Returns the items that have changed if there are no changes in dependencies, otherwise returns all slurped items for reauditing """ watcher.full_audit_list = None if auditor.support_watcher_indexes: for support_watcher_index in auditor.support_watcher_indexes: if support_watcher_index in watchers_with_changes: app.logger.debug("Upstream watcher changed {}. reauditing {}".format( support_watcher_index, watcher.index)) watcher.full_audit_list = auditor.read_previous_items() if auditor.support_auditor_indexes: for support_auditor_index in auditor.support_auditor_indexes: if support_auditor_index in watchers_with_changes: app.logger.debug("Upstream auditor changed {}. reauditing {}".format( support_auditor_index, watcher.index)) watcher.full_audit_list = auditor.read_previous_items() if watcher.full_audit_list: return watcher.full_audit_list return [item for item in watcher.created_items + watcher.changed_items]
def test_report_batch_changes(self): from security_monkey.alerter import Alerter from security_monkey.reporter import Reporter from security_monkey.datastore import Item, ItemRevision, ItemAudit from security_monkey.monitors import Monitor from security_monkey.watchers.iam.iam_role import IAMRole from security_monkey.auditors.iam.iam_role import IAMRoleAuditor account_type_result = AccountType.query.filter( AccountType.name == "AWS").one() db.session.add(account_type_result) db.session.commit() test_account = Account(identifier="012345678910", name="TEST_ACCOUNT", account_type_id=account_type_result.id, notes="TEST_ACCOUNT1", third_party=False, active=True) watcher = IAMRole(accounts=[test_account.name]) db.session.commit() watcher.batched_size = 3 # should loop 4 times self.add_roles() # Set up the monitor: batched_monitor = Monitor(IAMRole, test_account) batched_monitor.watcher = watcher batched_monitor.auditors = [ IAMRoleAuditor(accounts=[test_account.name]) ] # Set up the Reporter: import security_monkey.reporter old_all_monitors = security_monkey.reporter.all_monitors security_monkey.reporter.all_monitors = lambda x, y: [] test_reporter = Reporter() test_reporter.all_monitors = [batched_monitor] test_reporter.account_alerter = Alerter( watchers_auditors=test_reporter.all_monitors, account=test_account.name) import security_monkey.scheduler # import security_monkey.monitors # old_get_monitors = security_monkey.scheduler.get_monitors security_monkey.scheduler.get_monitors = lambda x, y, z: [ batched_monitor ] # Moto screws up the IAM Role ARN -- so we need to fix it: original_slurp_list = watcher.slurp_list original_slurp = watcher.slurp def mock_slurp_list(): items, exception_map = original_slurp_list() for item in watcher.total_list: item["Arn"] = "arn:aws:iam::012345678910:role/{}".format( item["RoleName"]) return items, exception_map def mock_slurp(): batched_items, exception_map = original_slurp() for item in batched_items: item.arn = "arn:aws:iam::012345678910:role/{}".format( item.name) item.config["Arn"] = item.arn item.config["RoleId"] = item.name # Need this to stay the same return batched_items, exception_map watcher.slurp_list = mock_slurp_list watcher.slurp = mock_slurp test_reporter.run(account=test_account.name) # Check that all items were added to the DB: assert len(Item.query.all()) == 11 # Check that we have exactly 11 item revisions: assert len(ItemRevision.query.all()) == 11 # Check that there are audit issues for all 11 items: assert len(ItemAudit.query.all()) == 11 mock_iam().stop() mock_sts().stop() # Something isn't cleaning itself up properly and causing other core tests to fail. # This is the solution: security_monkey.reporter.all_monitors = old_all_monitors import monitor_mock security_monkey.scheduler.get_monitors = monitor_mock.mock_get_monitors