Пример #1
0
    def pre_test_setup(self):
        account_type_result = AccountType.query.filter(
            AccountType.name == 'AWS').first()
        if not account_type_result:
            account_type_result = AccountType(name='AWS')
            db.session.add(account_type_result)
            db.session.commit()

        self.account = Account(identifier="012345678910",
                               name="testing",
                               active=True,
                               third_party=False,
                               account_type_id=account_type_result.id)
        self.technology = Technology(name="s3")
        self.item = Item(region="us-west-2",
                         name="somebucket",
                         arn=ARN_PREFIX + ":s3:::somebucket",
                         technology=self.technology,
                         account=self.account)

        db.session.add(self.account)
        db.session.add(self.technology)
        db.session.add(self.item)

        db.session.commit()

        mock_s3().start()
        client = boto3.client("s3")
        client.create_bucket(Bucket="somebucket")
        client.create_bucket(Bucket="someotherbucket")
        client.create_bucket(Bucket="someotherbucket2")
Пример #2
0
    def pre_test_setup(self):
        self.account_type = AccountType(name="GitHub")
        db.session.add(self.account_type)
        db.session.commit()

        db.session.add(
            Account(name="octocat",
                    account_type_id=self.account_type.id,
                    identifier="octocat",
                    active=True,
                    third_party=False))
        self.technology = Technology(name="repository")
        db.session.add(self.technology)
        db.session.commit()

        self.gh_items = [
            GitHubOrgItem(account="octocat",
                          name="Hello-World",
                          arn="octocat/Hello-World",
                          config=CONFIG_ONE),
            GitHubOrgItem(account="octocat",
                          name="Repo-Private",
                          arn="octocat/Repo-Private",
                          config=CONFIG_TWO),
        ]
Пример #3
0
    def prep_for_batch_slurp(self):
        """
        Should be run before batching slurps to set the current account (and region).

        This will load the DB objects for account and technology for where we are currently at in the process.
        :return:
        """
        self.prep_for_slurp()

        # Which account are we currently on?
        if not self.current_account:
            index = 0

            # Get the Technology
            # If technology doesn't exist, then create it:
            technology = Technology.query.filter(Technology.name == self.index).first()
            if not technology:
                technology = Technology(name=self.index)
                db.session.add(technology)
                db.session.commit()
                app.logger.info("Technology: {} did not exist... created it...".format(self.index))

            self.technology = technology
        else:
            index = self.current_account[1] + 1

        self.current_account = (Account.query.filter(Account.name == self.accounts[index]).one(), index)

        # We will not be using CloudAux's iter_account_region for multi-account -- we want
        # to have per-account level of batching
        self.total_list = []    # Reset the total list for a new account to run against.
        self.done_slurping = False
        self.batch_counter = 0
    def _setup_one_two_revisions(self):
        account_type_result = AccountType.query.filter(AccountType.name == 'AWS').first()
        if not account_type_result:
            account_type_result = AccountType(name='AWS')
            db.session.add(account_type_result)
            db.session.commit()

        account = Account(number="012345678910", name="testing",
                          s3_name="testing", role_name="SecurityMonkey",
                          account_type_id=account_type_result.id)

        technology = Technology(name="iamrole")
        item = Item(region="us-west-2", name="testrole",
                    arn="arn:aws:iam::012345678910:role/testrole", technology=technology,
                    account=account)

        self.now = datetime(2016, 11, 3)
        self.yesterday = self.now - timedelta(days=1)
        item.revisions.append(ItemRevision(active=True, config={}, date_created=self.now))
        item.revisions.append(ItemRevision(active=True, config={}, date_created=self.yesterday))

        db.session.add(account)
        db.session.add(technology)
        db.session.add(item)

        db.session.commit()

        items = Item.query.all()
        for item in items:
            latest_revision = item.revisions.first()
            item.latest_revision_id = latest_revision.id
            db.session.add(item)
            db.session.commit()
Пример #5
0
    def test_fix_orphaned_deletions(self):
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        technology = Technology(name="orphaned")

        db.session.add(technology)
        db.session.commit()

        orphaned_item = Item(name="orphaned",
                             region="us-east-1",
                             tech_id=technology.id,
                             account_id=test_account.id)
        db.session.add(orphaned_item)
        db.session.commit()

        assert not orphaned_item.latest_revision_id
        assert not orphaned_item.revisions.count()
        assert len(
            Item.query.filter(Item.account_id == test_account.id, Item.tech_id
                              == technology.id, Item.latest_revision_id
                              == None).all()) == 1  # noqa

        from security_monkey.task_scheduler.tasks import fix_orphaned_deletions
        fix_orphaned_deletions(test_account.name, technology.name)

        assert not Item.query.filter(
            Item.account_id == test_account.id, Item.tech_id == technology.id,
            Item.latest_revision_id == None).all()  # noqa

        assert orphaned_item.latest_revision_id
        assert orphaned_item.revisions.count() == 1
        assert orphaned_item.latest_config == {}
    def pre_test_setup(self):
        self.gh_items = [
            GitHubTeamItem(account="Org-one",
                           name="Org-one",
                           arn="Org-one",
                           config=CONFIG_ONE),
            GitHubTeamItem(account="Org-one",
                           name="Org-one",
                           arn="Org-one",
                           config=CONFIG_TWO),
        ]

        self.account_type = AccountType(name="GitHub")
        db.session.add(self.account_type)
        db.session.commit()

        db.session.add(
            Account(name="Org-one",
                    account_type_id=self.account_type.id,
                    identifier="Org-one",
                    active=True,
                    third_party=False))
        self.technology = Technology(name="team")
        db.session.add(self.technology)
        db.session.commit()
Пример #7
0
    def pre_test_setup(self):
        self.gh_items = [
            GitHubOrgItem(account="Netflix",
                          name="Netflix",
                          arn="Netflix",
                          config=CONFIG_ONE),
            GitHubOrgItem(account="Netflix-PRIVATE",
                          name="Netflix-PRIVATE",
                          arn="Netflix-PRIVATE",
                          config=CONFIG_TWO),
        ]

        self.account_type = AccountType(name="GitHub")
        db.session.add(self.account_type)
        db.session.commit()

        db.session.add(
            Account(name="Netflix",
                    account_type_id=self.account_type.id,
                    identifier="Netflix",
                    active=True,
                    third_party=False))
        db.session.add(
            Account(name="Netflix-PRIVATE",
                    account_type_id=self.account_type.id,
                    identifier="Netflix-PRIVATE",
                    active=True,
                    third_party=False))
        self.technology = Technology(name="organization")
        db.session.add(self.technology)
        db.session.commit()
Пример #8
0
    def pre_test_setup(self):
        account_type_result = AccountType(name='AWS')
        db.session.add(account_type_result)
        db.session.commit()

        self.account = Account(identifier="012345678910",
                               name="testing",
                               third_party=False,
                               active=True,
                               account_type_id=account_type_result.id)
        self.technology = Technology(name="iamrole")

        self.total_roles = 75

        db.session.add(self.account)
        db.session.add(self.technology)
        db.session.commit()
        mock_iam().start()
        client = boto3.client("iam")

        aspd = {
            "Version":
            "2012-10-17",
            "Statement": [{
                "Effect": "Allow",
                "Action": "sts:AssumeRole",
                "Principal": {
                    "Service": "ec2.amazonaws.com"
                }
            }]
        }

        policy = {
            "Version": "2012-10-17",
            "Statement": [{
                "Effect": "Deny",
                "Action": "*",
                "Resource": "*"
            }]
        }

        for x in range(0, self.total_roles):
            # Create the IAM Role via Moto:
            aspd["Statement"][0][
                "Resource"] = ARN_PREFIX + "arn:aws:iam:012345678910:role/roleNumber{}".format(
                    x)
            client.create_role(Path="/",
                               RoleName="roleNumber{}".format(x),
                               AssumeRolePolicyDocument=json.dumps(aspd,
                                                                   indent=4))
            client.put_role_policy(RoleName="roleNumber{}".format(x),
                                   PolicyName="testpolicy",
                                   PolicyDocument=json.dumps(policy, indent=4))
Пример #9
0
    def setup_db(self):
        account_type_result = AccountType(name='AWS')
        db.session.add(account_type_result)
        db.session.commit()

        self.account = Account(identifier="012345678910", name="testing",
                               account_type_id=account_type_result.id)
        self.technology = Technology(name="iamrole")

        db.session.add(self.account)
        db.session.add(self.technology)
        db.session.commit()
Пример #10
0
def fix_orphaned_deletions(account_name, technology_name):
    """
    Possible issue with orphaned items. This will check if there are any, and will assume that the item
    was deleted. This will create a deletion change record to it.

    :param account_name:
    :param technology_name:
    :return:
    """
    # If technology doesn't exist, then create it:
    technology = Technology.query.filter(
        Technology.name == technology_name).first()
    if not technology:
        technology = Technology(name=technology_name)
        db.session.add(technology)
        db.session.commit()
        app.logger.info("Technology: {} did not exist... created it...".format(
            technology_name))

    account = Account.query.filter(Account.name == account_name).one()

    # Query for orphaned items of the given technology/account pair:
    orphaned_items = Item.query.filter(
        Item.account_id == account.id, Item.tech_id == technology.id,
        Item.latest_revision_id == None).all()  # noqa

    if not orphaned_items:
        app.logger.info(
            "[@] No orphaned items have been found. (This is good)")
        return

    # Fix the orphaned items:
    for oi in orphaned_items:
        app.logger.error(
            "[?] Found an orphaned item: {}. Creating a deletion record for it"
            .format(oi.name))
        revision = ItemRevision(active=False, config={})
        oi.revisions.append(revision)
        db.session.add(revision)
        db.session.add(oi)
        db.session.commit()

        # Update the latest revision id:
        db.session.refresh(revision)
        oi.latest_revision_id = revision.id
        db.session.add(oi)

        db.session.commit()
        app.logger.info("[-] Created deletion record for item: {}.".format(
            oi.name))
    def pre_test_setup(self):
        self.account_type = AccountType.query.filter(
            AccountType.name == 'AWS').first()
        if not self.account_type:
            self.account_type = AccountType(name='AWS')
            db.session.add(self.account_type)
            db.session.commit()
        self.test_account = Account(type=self.account_type,
                                    name="test_account",
                                    identifier="012345678910")
        self.technology = Technology(name="testtech")

        db.session.add(self.test_account)
        db.session.add(self.technology)
        db.session.commit()
Пример #12
0
    def test_audit_specific_changes(self):
        from security_monkey.task_scheduler.tasks import _audit_specific_changes
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.cloudaux_watcher import CloudAuxChangeItem
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        # Set up the monitor:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher = Watcher(accounts=[test_account.name])
        watcher.current_account = (test_account, 0)
        watcher.technology = technology

        # Create some IAM roles for testing:
        items = []
        for x in range(0, 3):
            role_policy = dict(ROLE_CONF)
            role_policy[
                "Arn"] = ARN_PREFIX + ":iam::012345678910:role/roleNumber{}".format(
                    x)
            role_policy["RoleName"] = "roleNumber{}".format(x)
            role = CloudAuxChangeItem.from_item(name=role_policy['RoleName'],
                                                item=role_policy,
                                                record_region='universal',
                                                account_name=test_account.name,
                                                index='iamrole',
                                                source_watcher=watcher)
            items.append(role)

        audit_items = watcher.find_changes_batch(items, {})
        assert len(audit_items) == 3

        # Perform the audit:
        _audit_specific_changes(batched_monitor, audit_items, False)

        # Check all the issues are there:
        assert len(ItemAudit.query.all()) == 3
    def pre_test_setup(self):
        self.account_type = AccountType(name="GitHub")
        db.session.add(self.account_type)
        db.session.commit()

        app.config["GITHUB_CREDENTIALS"] = {
            "Org-one": "token-one",
            "FAILURE": "FAILURE",
            "Netflix": "token-two"
        }

        db.session.add(Account(name="Org-one", account_type_id=self.account_type.id,
                               identifier="Org-one", active=True, third_party=False))
        self.technology = Technology(name="repository")
        db.session.add(self.technology)

        db.session.commit()
Пример #14
0
    def pre_test_setup(self):
        account_type_result = AccountType.query.filter(AccountType.name == 'AWS').first()
        if not account_type_result:
            account_type_result = AccountType(name='AWS')
            db.session.add(account_type_result)
            db.session.commit()

        self.account = Account(identifier="012345678910", name="testing",
                               account_type_id=account_type_result.id)

        self.technology = Technology(name="iamrole")
        item = Item(region="us-west-2", name="testrole",
                    arn=ARN_PREFIX + ":iam::012345678910:role/testrole", technology=self.technology,
                    account=self.account)

        db.session.add(self.account)
        db.session.add(self.technology)
        db.session.add(item)
        db.session.commit()
Пример #15
0
    def test_audit_specific_changes(self):
        from security_monkey.scheduler import _audit_specific_changes
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole, IAMRoleItem
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor

        # Set up the monitor:
        test_account = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher = Watcher(accounts=[test_account.name])
        watcher.current_account = (test_account, 0)
        watcher.technology = technology

        # Create some IAM roles for testing:
        items = []
        for x in range(0, 3):
            role_policy = dict(ROLE_CONF)
            role_policy[
                "Arn"] = "arn:aws:iam::012345678910:role/roleNumber{}".format(
                    x)
            role_policy["RoleName"] = "roleNumber{}".format(x)
            role = IAMRoleItem.from_slurp(role_policy,
                                          account_name=test_account.name)
            items.append(role)

        audit_items = watcher.find_changes_batch(items, {})
        assert len(audit_items) == 3

        # Perform the audit:
        _audit_specific_changes(batched_monitor, audit_items, False)

        # Check all the issues are there:
        assert len(ItemAudit.query.all()) == 3
Пример #16
0
    def test_find_batch_changes(self, mock_fix_orphaned):
        """
        Runs through a full find job via the IAMRole watcher, as that supports batching.

        However, this is mostly testing the logic through each function call -- this is
        not going to do any boto work and that will instead be mocked out.
        :return:
        """
        from security_monkey.task_scheduler.tasks import manual_run_change_finder
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        test_account = Account(name="TEST_ACCOUNT1")
        watcher = IAMRole(accounts=[test_account.name])

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher.batched_size = 3  # should loop 4 times

        self.add_roles()

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        manual_run_change_finder([test_account.name], [watcher.index])
        assert mock_fix_orphaned.called

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        # Delete one of the items:
        # Moto lacks implementation for "delete_role" (and I'm too lazy to submit a PR :D) -- so need to create again...
        mock_iam().stop()
        mock_sts().stop()
        self.add_roles(initial=False)

        # Run the it again:
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        # Check that nothing new was added:
        assert len(Item.query.all()) == 11

        # There should be the same number of issues and 2 more revisions:
        assert len(ItemAudit.query.all()) == 11
        assert len(ItemRevision.query.all()) == 13

        # Check that the deleted roles show as being inactive:
        ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
            .filter(Item.arn.in_(
                [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                 ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()

        assert len(ir) == 2
        assert not ir[0].active
        assert not ir[1].active

        # Finally -- test with a slurp list exception (just checking that things don't blow up):
        import security_monkey.watchers.iam.iam_role
        old_list_roles = security_monkey.watchers.iam.iam_role.list_roles

        def mock_slurp_list_with_exception():
            security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0

            items, exception_map = original_slurp_list()

            assert len(exception_map) > 0
            return items, exception_map

        watcher.slurp_list = mock_slurp_list_with_exception
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
        security_monkey.watchers.iam.iam_role.list_roles = old_list_roles

        mock_iam().stop()
        mock_sts().stop()
Пример #17
0
    def test_celery_skipabeat(self, mock_store_exception,
                              mock_expired_exceptions, mock_account_tech,
                              mock_purge, mock_setup):
        from security_monkey.task_scheduler.beat import setup_the_tasks
        from security_monkey.watchers.github.org import GitHubOrg
        from security_monkey.auditors.github.org import GitHubOrgAuditor

        # Stop the watcher registry from stepping on everyone's toes:
        import security_monkey.watcher
        import security_monkey.monitors
        security_monkey.watcher.watcher_registry = {GitHubOrg.index: GitHubOrg}
        security_monkey.monitors.watcher_registry = security_monkey.watcher.watcher_registry

        app.config["GITHUB_CREDENTIALS"] = {}

        # Set up the monitor:
        self.account_type = AccountType(name="GitHub")
        db.session.add(self.account_type)
        db.session.commit()
        app.config["GITHUB_CREDENTIALS"] = {"Org-one": "token-one"}
        db.session.add(
            Account(name="Org-one",
                    account_type_id=self.account_type.id,
                    identifier="Org-one",
                    active=True,
                    third_party=False))
        self.technology = Technology(name="organization")
        db.session.add(self.technology)
        db.session.commit()

        # Disable the other accounts:
        disable_account_1 = Account.query.filter(
            Account.name == "TEST_ACCOUNT1").one()
        disable_account_2 = Account.query.filter(
            Account.name == "TEST_ACCOUNT2").one()
        disable_account_1.active = False
        disable_account_2.active = False
        db.session.add(disable_account_1)
        db.session.add(disable_account_2)
        db.session.commit()

        test_account = Account.query.filter(Account.name == "Org-one").one()
        watcher = GitHubOrg(accounts=[test_account.name])
        monitor = Monitor(GitHubOrg, test_account)
        monitor.watcher = watcher
        monitor.auditors = [GitHubOrgAuditor(accounts=[test_account.name])]

        # This is externally executed (as in not with Celery):
        db.session.add(
            WatcherConfig(index=GitHubOrg.index, active=True, interval=0))
        db.session.commit()

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            monitor
        ]

        get_interval = mock.Mock()
        monitor.watcher.get_interval = get_interval

        setup_the_tasks(mock.Mock())

        assert mock_setup.called
        assert mock_purge.called
        assert not mock_store_exception.called

        # "apply_async" will NOT be called...
        assert not mock_account_tech.apply_async.called

        # The ".s" are the scheduled tasks. Too lazy to grab the intervals out.
        assert not mock_account_tech.s.called  # Will not be called
        assert mock_expired_exceptions.s.called
        assert mock_expired_exceptions.apply_async.called

        # Cleanup:
        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
        disable_account_1.active = True
        disable_account_2.active = True
        test_account.active = False
        db.session.add(disable_account_1)
        db.session.add(disable_account_2)
        db.session.add(test_account)
        db.session.commit()
Пример #18
0
    def test_find_batch_changes(self, mock_fix_orphaned):
        """
        Runs through a full find job via the IAMRole watcher, as that supports batching.

        However, this is mostly testing the logic through each function call -- this is
        not going to do any boto work and that will instead be mocked out.
        :return:
        """
        from security_monkey.task_scheduler.tasks import manual_run_change_finder
        from security_monkey.monitors import Monitor
        from security_monkey.watchers.iam.iam_role import IAMRole
        from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
        test_account = Account(name="TEST_ACCOUNT1")
        watcher = IAMRole(accounts=[test_account.name])

        technology = Technology(name="iamrole")
        db.session.add(technology)
        db.session.commit()

        watcher.batched_size = 3  # should loop 4 times

        ## CREATE MOCK IAM ROLES ##
        client = boto3.client("iam")
        aspd = {
            "Version":
            "2012-10-17",
            "Statement": [{
                "Effect": "Allow",
                "Action": "sts:AssumeRole",
                "Principal": {
                    "Service": "ec2.amazonaws.com"
                }
            }]
        }
        for x in range(0, 11):
            # Create the IAM Role via Moto:
            aspd["Statement"][0][
                "Resource"] = ARN_PREFIX + ":iam:012345678910:role/roleNumber{}".format(
                    x)
            client.create_role(Path="/",
                               RoleName="roleNumber{}".format(x),
                               AssumeRolePolicyDocument=json.dumps(aspd,
                                                                   indent=4))
            client.put_role_policy(RoleName="roleNumber{}".format(x),
                                   PolicyName="testpolicy",
                                   PolicyDocument=json.dumps(OPEN_POLICY,
                                                             indent=4))

        # Set up the monitor:
        batched_monitor = Monitor(IAMRole, test_account)
        batched_monitor.watcher = watcher
        batched_monitor.auditors = [
            IAMRoleAuditor(accounts=[test_account.name])
        ]

        import security_monkey.task_scheduler.tasks
        old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
        security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [
            batched_monitor
        ]

        # Moto screws up the IAM Role ARN -- so we need to fix it:
        original_slurp_list = watcher.slurp_list
        original_slurp = watcher.slurp

        def mock_slurp_list():
            items, exception_map = original_slurp_list()

            for item in watcher.total_list:
                item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item["RoleName"])

            return items, exception_map

        def mock_slurp():
            batched_items, exception_map = original_slurp()

            for item in batched_items:
                item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(
                    item.name)
                item.config["Arn"] = item.arn
                item.config["RoleId"] = item.name  # Need this to stay the same

            return batched_items, exception_map

        watcher.slurp_list = mock_slurp_list
        watcher.slurp = mock_slurp

        manual_run_change_finder([test_account.name], [watcher.index])
        assert mock_fix_orphaned.called

        # Check that all items were added to the DB:
        assert len(Item.query.all()) == 11

        # Check that we have exactly 11 item revisions:
        assert len(ItemRevision.query.all()) == 11

        # Check that there are audit issues for all 11 items:
        assert len(ItemAudit.query.all()) == 11

        # Delete two of the items:
        managedPolicy = client.list_attached_role_policies(
            RoleName="roleNumber9")
        for each in managedPolicy['AttachedPolicies']:
            print("Detaching ", each)
            client.detach_role_policy(RoleName="roleNumber9",
                                      PolicyArn=each['PolicyArn'])

        inlinePolicy = client.list_role_policies(RoleName="roleNumber9")
        for each in inlinePolicy['PolicyNames']:
            print("Deleting ", each)
            client.delete_role_policy(RoleName="roleNumber9", PolicyName=each)

        instanceProfiles = client.list_instance_profiles_for_role(
            RoleName="roleNumber9")
        for each in instanceProfiles['InstanceProfiles']:
            print("Removing role from instance profile ", each)
            client.remove_role_from_instance_profile(
                RoleName="roleNumber9",
                InstanceProfileName=each['InstanceProfileName'])
        client.delete_role(RoleName="roleNumber9")

        managedPolicy = client.list_attached_role_policies(
            RoleName="roleNumber10")
        for each in managedPolicy['AttachedPolicies']:
            print("Detaching ", each)
            client.detach_role_policy(RoleName="roleNumber10",
                                      PolicyArn=each['PolicyArn'])

        inlinePolicy = client.list_role_policies(RoleName="roleNumber10")
        for each in inlinePolicy['PolicyNames']:
            print("Deleting ", each)
            client.delete_role_policy(RoleName="roleNumber10", PolicyName=each)

        instanceProfiles = client.list_instance_profiles_for_role(
            RoleName="roleNumber10")
        for each in instanceProfiles['InstanceProfiles']:
            print("Removing role from instance profile ", each)
            client.remove_role_from_instance_profile(
                RoleName="roleNumber10",
                InstanceProfileName=each['InstanceProfileName'])
        client.delete_role(RoleName="roleNumber10")

        # Run the it again:
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        # Check that nothing new was added:
        assert len(Item.query.all()) == 11

        # There should be the same number of issues and 2 more revisions:
        assert len(ItemAudit.query.all()) == 11
        assert len(ItemRevision.query.all()) == 13

        # Check that the deleted roles show as being inactive:
        ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
            .filter(Item.arn.in_(
                [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                 ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()

        assert len(ir) == 2
        assert not ir[0].active
        assert not ir[1].active

        # Finally -- test with a slurp list exception (just checking that things don't blow up):
        import security_monkey.watchers.iam.iam_role
        old_list_roles = security_monkey.watchers.iam.iam_role.list_roles

        def mock_slurp_list_with_exception():
            security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0

            items, exception_map = original_slurp_list()

            assert len(exception_map) > 0
            return items, exception_map

        watcher.slurp_list = mock_slurp_list_with_exception
        watcher.current_account = None  # Need to reset the watcher
        manual_run_change_finder([test_account.name], [watcher.index])

        security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
        security_monkey.watchers.iam.iam_role.list_roles = old_list_roles