Ejemplo n.º 1
0
    def test_iamrole_trust_policy(self):
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        trust_policy = {
            "Version": "2008-10-17",
            "Statement": [
                {
                    "Action": "sts:AssumeRole",
                    "Principal": {
                        "AWS": ["arn:aws:iam::222222222222:role/SomeRole"]
                    },
                    "Effect": "Allow",
                    "Sid": ""
                }
            ]
        }

        auditor = IAMRoleAuditor(accounts=['TEST_ACCOUNT'])
        auditor.prep_for_audit()

        iamobj = MockIAMObj()
        iamobj.account = 'TEST_ACCOUNT'
        iamobj.config = {'InlinePolicies': None, 'AssumeRolePolicyDocument': trust_policy}

        auditor.check_friendly_cross_account(iamobj)
        self.assertIs(len(iamobj.audit_issues), 1, "Cross Account Trust Policy not Flagged")
        self.assertEquals(iamobj.audit_issues[0].issue, 'Friendly Cross Account')
        self.assertEquals(iamobj.audit_issues[0].notes, 'Account: [222222222222/TEST_ACCOUNT_TWO] Entity: [principal:arn:aws:iam::222222222222:role/SomeRole] Actions: ["sts:AssumeRole"]')
Ejemplo n.º 2
0
    def test_celery_beat(self, mock_expired_exceptions, mock_account_tech,
                         mock_purge, mock_setup):
        from security_monkey.task_scheduler.beat import setup_the_tasks
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        # Set up the monitor:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        watcher = IAMRole(accounts=[test_account.name])
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        setup_the_tasks(mock.Mock())

        assert mock_setup.called
        assert mock_purge.called

        # "apply_async" where the immediately scheduled tasks
        assert mock_account_tech.apply_async.called

        # The ".s" are the scheduled tasks. Too lazy to grab the intervals out.
        assert mock_account_tech.s.called
        assert mock_expired_exceptions.s.called
        #assert mock_task_audit.s.called

        # Build the expected mock results:
        scheduled_tech_result_list = []
        async_result_list = []
        # audit_result_list = []

        import security_monkey.watcher
        import security_monkey.auditor

        for account in Account.query.filter(
                Account.third_party == False).filter(
                    Account.active == True).all():  # noqa
            for w in security_monkey.watcher.watcher_registry.iterkeys():
                scheduled_tech_result_list.append(((account.name, w), ))
                async_result_list.append((((account.name, w), ), ))

            # It's just policy for IAM:
            # audit_result_list.append(((account.name, "policy"),))

        assert mock_account_tech.s.call_args_list == scheduled_tech_result_list
        assert async_result_list == mock_account_tech.apply_async.call_args_list
        # assert audit_result_list == mock_task_audit.s.call_args_list

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
Ejemplo n.º 3
0
    def test_iamrole_trust_policy(self):
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        trust_policy = {
            "Version":
            "2008-10-17",
            "Statement": [{
                "Action": "sts:AssumeRole",
                "Principal": {
                    "AWS": ["arn:aws:iam::222222222222:role/SomeRole"]
                },
                "Effect": "Allow",
                "Sid": ""
            }]
        }

        auditor = IAMRoleAuditor(accounts=['TEST_ACCOUNT'])
        auditor.prep_for_audit()

        iamobj = MockIAMObj()
        iamobj.account = 'TEST_ACCOUNT'
        iamobj.config = {
            'InlinePolicies': None,
            'AssumeRolePolicyDocument': trust_policy
        }

        auditor.check_friendly_cross_account(iamobj)
        self.assertIs(len(iamobj.audit_issues), 1,
                      "Cross Account Trust Policy not Flagged")
        self.assertEquals(iamobj.audit_issues[0].issue,
                          'Friendly Cross Account')
        self.assertEquals(
            iamobj.audit_issues[0].notes,
            'Account: [222222222222/TEST_ACCOUNT_TWO] Entity: [principal:arn:aws:iam::222222222222:role/SomeRole] Actions: ["sts:AssumeRole"]'
        )
Ejemplo n.º 4
0
    def test_audit_specific_changes(self):
        from security_monkey.task_scheduler.tasks import _audit_specific_changes
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.cloudaux_watcher import CloudAuxChangeItem
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        # Set up the monitor:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher = Watcher(accounts=[test_account.name])
        watcher.current_account = (test_account, 0)
        watcher.technology = technology

        # Create some IAM roles for testing:
        items = []
        for x in range(0, 3):
            role_policy = dict(ROLE_CONF)
            role_policy[
                "Arn"] = ARN_PREFIX + ":iam::012345678910:role/roleNumber{}".format(
                    x)
            role_policy["RoleName"] = "roleNumber{}".format(x)
            role = CloudAuxChangeItem.from_item(name=role_policy['RoleName'],
                                                item=role_policy,
                                                record_region='universal',
                                                account_name=test_account.name,
                                                index='iamrole',
                                                source_watcher=watcher)
            items.append(role)

        audit_items = watcher.find_changes_batch(items, {})
        assert len(audit_items) == 3

        # Perform the audit:
        _audit_specific_changes(batched_monitor, audit_items, False)

        # Check all the issues are there:
        assert len(ItemAudit.query.all()) == 3
Ejemplo n.º 5
0
    def test_audit_specific_changes(self):
        from security_monkey.scheduler import _audit_specific_changes
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole, IAMRoleItem
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        # Set up the monitor:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher = Watcher(accounts=[test_account.name])
        watcher.current_account = (test_account, 0)
        watcher.technology = technology

        # Create some IAM roles for testing:
        items = []
        for x in range(0, 3):
            role_policy = dict(ROLE_CONF)
            role_policy[
                "Arn"] = "arn:aws:iam::012345678910:role/roleNumber{}".format(
                    x)
            role_policy["RoleName"] = "roleNumber{}".format(x)
            role = IAMRoleItem.from_slurp(role_policy,
                                          account_name=test_account.name)
            items.append(role)

        audit_items = watcher.find_changes_batch(items, {})
        assert len(audit_items) == 3

        # Perform the audit:
        _audit_specific_changes(batched_monitor, audit_items, False)

        # Check all the issues are there:
        assert len(ItemAudit.query.all()) == 3
Ejemplo n.º 6
0
    def pre_test_setup(self):
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        from security_monkey.datastore import Account, AccountType
        from security_monkey import db

        IAMRoleAuditor(accounts=['TEST_ACCOUNT']).OBJECT_STORE.clear()
        account_type_result = AccountType(name='AWS')
        db.session.add(account_type_result)
        db.session.commit()

        # main
        account = Account(identifier="012345678910",
                          name="TEST_ACCOUNT",
                          account_type_id=account_type_result.id,
                          notes="TEST_ACCOUNT",
                          third_party=False,
                          active=True)
        # friendly
        account2 = Account(identifier="222222222222",
                           name="TEST_ACCOUNT_TWO",
                           account_type_id=account_type_result.id,
                           notes="TEST_ACCOUNT_TWO",
                           third_party=False,
                           active=True)
        # third party
        account3 = Account(identifier="333333333333",
                           name="TEST_ACCOUNT_THREE",
                           account_type_id=account_type_result.id,
                           notes="TEST_ACCOUNT_THREE",
                           third_party=True,
                           active=True)

        db.session.add(account)
        db.session.add(account2)
        db.session.add(account3)
        db.session.commit()
Ejemplo n.º 7
0
    def test_find_batch_changes(self, mock_fix_orphaned):
        """
        Runs through a full find job via the IAMRole watcher, as that supports batching.

        However, this is mostly testing the logic through each function call -- this is
        not going to do any boto work and that will instead be mocked out.
        :return:
        """
        from security_monkey.task_scheduler.tasks import manual_run_change_finder
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        test_account = Account(name="TEST_ACCOUNT1")
        watcher = IAMRole(accounts=[test_account.name])

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher.batched_size = 3  # should loop 4 times

        self.add_roles()

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        manual_run_change_finder([test_account.name], [watcher.index])
        assert mock_fix_orphaned.called

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        # Delete one of the items:
        # Moto lacks implementation for "delete_role" (and I'm too lazy to submit a PR :D) -- so need to create again...
        mock_iam().stop()
        mock_sts().stop()
        self.add_roles(initial=False)

        # Run the it again:
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        # Check that nothing new was added:
        assert len(Item.query.all()) == 11

        # There should be the same number of issues and 2 more revisions:
        assert len(ItemAudit.query.all()) == 11
        assert len(ItemRevision.query.all()) == 13

        # Check that the deleted roles show as being inactive:
        ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
            .filter(Item.arn.in_(
                [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                 ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()

        assert len(ir) == 2
        assert not ir[0].active
        assert not ir[1].active

        # Finally -- test with a slurp list exception (just checking that things don't blow up):
        import security_monkey.watchers.iam.iam_role
        old_list_roles = security_monkey.watchers.iam.iam_role.list_roles

        def mock_slurp_list_with_exception():
            security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0

            items, exception_map = original_slurp_list()

            assert len(exception_map) > 0
            return items, exception_map

        watcher.slurp_list = mock_slurp_list_with_exception
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
        security_monkey.watchers.iam.iam_role.list_roles = old_list_roles

        mock_iam().stop()
        mock_sts().stop()
Ejemplo n.º 8
0
    def test_report_batch_changes(self, mock_fix_orphaned):
        from security_monkey.task_scheduler.tasks import manual_run_change_reporter
        from security_monkey.datastore import Item, ItemRevision, ItemAudit
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()

        watcher = IAMRole(accounts=[test_account.name])

        watcher.batched_size = 3  # should loop 4 times

        self.add_roles()

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        # Set up the Reporter:
        import security_monkey.reporter
        old_all_monitors = security_monkey.reporter.all_monitors
        security_monkey.reporter.all_monitors = lambda x, y: [batched_monitor]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        manual_run_change_reporter([test_account.name])

        assert mock_fix_orphaned.called

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        mock_iam().stop()
        mock_sts().stop()

        security_monkey.reporter.all_monitors = old_all_monitors
        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
Ejemplo n.º 9
0
    def test_celery_ignore_tech(self, mock_store_exception,
                                mock_expired_exceptions, mock_account_tech,
                                mock_purge, mock_setup):
        import security_monkey.celeryconfig
        security_monkey.celeryconfig.security_monkey_watcher_ignore = {
            "policy"
        }

        from security_monkey.task_scheduler.beat import setup_the_tasks
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.watchers.iam.managed_policy import ManagedPolicy
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        from security_monkey.auditors.iam.iam_policy import IAMPolicyAuditor

        # Stop the watcher registry from stepping on everyone's toes:
        import security_monkey.watcher
        import security_monkey.monitors
        security_monkey.watcher.watcher_registry = {
            IAMRole.index: IAMRole,
            ManagedPolicy.index: ManagedPolicy
        }
        security_monkey.monitors.watcher_registry = security_monkey.watcher.watcher_registry

        # Set up the monitors:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        role_watcher = IAMRole(accounts=[test_account.name])
        mp_watcher = ManagedPolicy(accounts=[test_account.name])
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = role_watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]
        normal_monitor = Monitor(ManagedPolicy, test_account)
        normal_monitor.watcher = mp_watcher
        normal_monitor.auditors = [
            IAMPolicyAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor, normal_monitor
        ]

        setup_the_tasks(mock.Mock())

        assert mock_setup.called
        assert mock_purge.called
        assert not mock_store_exception.called

        # "apply_async" where the immediately scheduled tasks
        assert mock_account_tech.apply_async.called

        # The ".s" are the scheduled tasks. Too lazy to grab the intervals out.
        assert mock_account_tech.s.called
        assert mock_expired_exceptions.s.called
        assert mock_expired_exceptions.apply_async.called

        # Policy should not be called at all:
        for mocked_call in mock_account_tech.s.call_args_list:
            assert mocked_call[0][1] == "iamrole"

        for mocked_call in mock_account_tech.apply_async.call_args_list:
            assert mocked_call[0][0][1] == "iamrole"

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
Ejemplo n.º 10
0
    def test_report_batch_changes(self):
        from security_monkey.alerter import Alerter
        from security_monkey.reporter import Reporter
        from security_monkey.datastore import Item, ItemRevision, ItemAudit
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        account_type_result = AccountType.query.filter(
            AccountType.name == "AWS").one()
        db.session.add(account_type_result)
        db.session.commit()

        test_account = Account(identifier="012345678910",
                               name="TEST_ACCOUNT",
                               account_type_id=account_type_result.id,
                               notes="TEST_ACCOUNT1",
                               third_party=False,
                               active=True)
        watcher = IAMRole(accounts=[test_account.name])
        db.session.commit()

        watcher.batched_size = 3  # should loop 4 times

        self.add_roles()

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        # Set up the Reporter:
        import security_monkey.reporter
        old_all_monitors = security_monkey.reporter.all_monitors
        security_monkey.reporter.all_monitors = lambda x, y: []

        test_reporter = Reporter()
        test_reporter.all_monitors = [batched_monitor]
        test_reporter.account_alerter = Alerter(
            watchers_auditors=test_reporter.all_monitors,
            account=test_account.name)

        import security_monkey.scheduler
        # import security_monkey.monitors
        # old_get_monitors = security_monkey.scheduler.get_monitors
        security_monkey.scheduler.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = "arn:aws:iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = "arn:aws:iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        test_reporter.run(account=test_account.name)

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        mock_iam().stop()
        mock_sts().stop()

        # Something isn't cleaning itself up properly and causing other core tests to fail.
        # This is the solution:
        security_monkey.reporter.all_monitors = old_all_monitors
        import monitor_mock
        security_monkey.scheduler.get_monitors = monitor_mock.mock_get_monitors
Ejemplo n.º 11
0
    def test_find_batch_changes(self, mock_fix_orphaned):
        """
        Runs through a full find job via the IAMRole watcher, as that supports batching.

        However, this is mostly testing the logic through each function call -- this is
        not going to do any boto work and that will instead be mocked out.
        :return:
        """
        from security_monkey.task_scheduler.tasks import manual_run_change_finder
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        test_account = Account(name="TEST_ACCOUNT1")
        watcher = IAMRole(accounts=[test_account.name])

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher.batched_size = 3  # should loop 4 times

        ## CREATE MOCK IAM ROLES ##
        client = boto3.client("iam")
        aspd = {
            "Version":
            "2012-10-17",
            "Statement": [{
                "Effect": "Allow",
                "Action": "sts:AssumeRole",
                "Principal": {
                    "Service": "ec2.amazonaws.com"
                }
            }]
        }
        for x in range(0, 11):
            # Create the IAM Role via Moto:
            aspd["Statement"][0][
                "Resource"] = ARN_PREFIX + ":iam:012345678910:role/roleNumber{}".format(
                    x)
            client.create_role(Path="/",
                               RoleName="roleNumber{}".format(x),
                               AssumeRolePolicyDocument=json.dumps(aspd,
                                                                   indent=4))
            client.put_role_policy(RoleName="roleNumber{}".format(x),
                                   PolicyName="testpolicy",
                                   PolicyDocument=json.dumps(OPEN_POLICY,
                                                             indent=4))

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        manual_run_change_finder([test_account.name], [watcher.index])
        assert mock_fix_orphaned.called

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        # Delete two of the items:
        managedPolicy = client.list_attached_role_policies(
            RoleName="roleNumber9")
        for each in managedPolicy['AttachedPolicies']:
            print("Detaching ", each)
            client.detach_role_policy(RoleName="roleNumber9",
                                      PolicyArn=each['PolicyArn'])

        inlinePolicy = client.list_role_policies(RoleName="roleNumber9")
        for each in inlinePolicy['PolicyNames']:
            print("Deleting ", each)
            client.delete_role_policy(RoleName="roleNumber9", PolicyName=each)

        instanceProfiles = client.list_instance_profiles_for_role(
            RoleName="roleNumber9")
        for each in instanceProfiles['InstanceProfiles']:
            print("Removing role from instance profile ", each)
            client.remove_role_from_instance_profile(
                RoleName="roleNumber9",
                InstanceProfileName=each['InstanceProfileName'])
        client.delete_role(RoleName="roleNumber9")

        managedPolicy = client.list_attached_role_policies(
            RoleName="roleNumber10")
        for each in managedPolicy['AttachedPolicies']:
            print("Detaching ", each)
            client.detach_role_policy(RoleName="roleNumber10",
                                      PolicyArn=each['PolicyArn'])

        inlinePolicy = client.list_role_policies(RoleName="roleNumber10")
        for each in inlinePolicy['PolicyNames']:
            print("Deleting ", each)
            client.delete_role_policy(RoleName="roleNumber10", PolicyName=each)

        instanceProfiles = client.list_instance_profiles_for_role(
            RoleName="roleNumber10")
        for each in instanceProfiles['InstanceProfiles']:
            print("Removing role from instance profile ", each)
            client.remove_role_from_instance_profile(
                RoleName="roleNumber10",
                InstanceProfileName=each['InstanceProfileName'])
        client.delete_role(RoleName="roleNumber10")

        # Run the it again:
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        # Check that nothing new was added:
        assert len(Item.query.all()) == 11

        # There should be the same number of issues and 2 more revisions:
        assert len(ItemAudit.query.all()) == 11
        assert len(ItemRevision.query.all()) == 13

        # Check that the deleted roles show as being inactive:
        ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
            .filter(Item.arn.in_(
                [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                 ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()

        assert len(ir) == 2
        assert not ir[0].active
        assert not ir[1].active

        # Finally -- test with a slurp list exception (just checking that things don't blow up):
        import security_monkey.watchers.iam.iam_role
        old_list_roles = security_monkey.watchers.iam.iam_role.list_roles

        def mock_slurp_list_with_exception():
            security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0

            items, exception_map = original_slurp_list()

            assert len(exception_map) > 0
            return items, exception_map

        watcher.slurp_list = mock_slurp_list_with_exception
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
        security_monkey.watchers.iam.iam_role.list_roles = old_list_roles