def crawl_account_hierarchy(provider_uuid=None): """Crawl top level accounts to discover hierarchy.""" if provider_uuid: _, polling_accounts = Orchestrator.get_accounts( provider_uuid=provider_uuid) else: _, polling_accounts = Orchestrator.get_accounts() LOG.info("Account hierarchy crawler found %s accounts to scan" % len(polling_accounts)) processed = 0 skipped = 0 for account in polling_accounts: crawler = None # Look for a known crawler class to handle this provider if account.get("provider_type") == Provider.PROVIDER_AWS: crawler = AWSOrgUnitCrawler(account) if crawler: LOG.info( "Starting account hierarchy crawler for type {} with provider_uuid: {}" .format(account.get("provider_type"), account.get("provider_uuid"))) crawler.crawl_account_hierarchy() processed += 1 else: LOG.info( "No known crawler for account with provider_uuid: {} of type {}" .format(account.get("provider_uuid"), account.get("provider_type"))) skipped += 1 LOG.info( f"Account hierarchy crawler finished. {processed} processed and {skipped} skipped" )
def test_general_client_error_denied(self): """Test botocore general ClientError.""" logging.disable(logging.NOTSET) mock_error = MagicMock() mock_error.list_roots.side_effect = _mock_boto3_general_client_error unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._client = mock_error unit_crawler._check_if_crawlable() self.assertEqual(False, unit_crawler.crawlable)
def test_crawl_list_root_access_denied(self): """Test botocore list roots access denied.""" # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html logging.disable(logging.NOTSET) mock_error = MagicMock() mock_error.list_roots.side_effect = _mock_boto3_access_denied unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._client = mock_error unit_crawler._check_if_crawlable() self.assertEqual(False, unit_crawler.crawlable)
def test_org_unit_deleted_state(self): """Test that an org unit that is in a deleted state is fixed when it is found again.""" unit_crawler = AWSOrgUnitCrawler(self.account) with schema_context(self.schema): AWSAccountAlias.objects.create(account_id="A_001", account_alias="Root Account") unit_crawler._build_accout_alias_map() unit_crawler._structure_yesterday = {} root_ou = {"Id": "R_001", "Name": "root"} root_account = {"Id": "A_001", "Name": "Root Account"} unit_crawler._save_aws_org_method(root_ou, "unit_path", 0, root_account) # simulate an org unit getting into a deleted state and ensure that the crawler # nullifies the deleted_timestamp with schema_context(self.schema): ou_to_update = AWSOrganizationalUnit.objects.filter( org_unit_id="R_001") ou_to_update.update( deleted_timestamp=unit_crawler._date_accessor.today()) updated_ou = unit_crawler._save_aws_org_method(root_ou, "unit_path", 0, root_account) with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 1) self.assertEqual(updated_ou.deleted_timestamp, None)
def test_build_account_alias_map(self): """Test function that builds account alias map.""" unit_crawler = AWSOrgUnitCrawler(self.account) with schema_context(self.schema): account_1 = AWSAccountAlias.objects.create(account_id="12345", account_alias="a1") account_2 = AWSAccountAlias.objects.create(account_id="67890", account_alias="a2") unit_crawler._build_accout_alias_map() self.assertIsNotNone(unit_crawler._account_alias_map) self.assertIn(account_1.account_id, unit_crawler._account_alias_map.keys()) self.assertEqual(unit_crawler._account_alias_map.get(account_1.account_id), account_1) self.assertIn(account_2.account_id, unit_crawler._account_alias_map.keys()) self.assertEqual(unit_crawler._account_alias_map.get(account_2.account_id), account_2)
def test_initializer(self): """Test AWSOrgUnitCrawler initializer.""" unit_crawler = AWSOrgUnitCrawler(self.account) result_auth_cred = unit_crawler._auth_cred expected_auth_cred = self.account.get("authentication") self.assertEqual(result_auth_cred, expected_auth_cred) self.assertIsNone(unit_crawler._client)
def test_no_delete_on_exceptions(self, mock_crawl, mock_session): """Test that when things go wrong we don't delete.""" mock_crawl.side_effect = Exception() mock_session.client = MagicMock() account_side_effect = [] paginator_side_effect = [] ou_ids = ["r-0", "ou-0", "ou-1", "ou-2", "sou-0"] for ou_id in ou_ids: parent_acts = _generate_act_for_parent_side_effect( self.schema, ou_id) account_side_effect.extend(parent_acts) paginator = MagicMock() paginator.paginate( ParentId=ou_id ).build_full_result.return_value = self.paginator_dict[ou_id] paginator_side_effect.append(paginator) unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.return_value = { "Roots": [{ "Id": "r-0", "Arn": "arn-0", "Name": "root_0" }] } unit_crawler._client.list_accounts_for_parent.side_effect = account_side_effect unit_crawler._client.get_paginator.side_effect = paginator_side_effect with patch( "masu.external.accounts.hierarchy.aws.aws_org_unit_crawler.AWSOrgUnitCrawler._mark_nodes_deleted" ) as mock_deleted: unit_crawler.crawl_account_hierarchy() self.assertEqual(True, unit_crawler.errors_raised) self.assertEqual(False, mock_deleted.called)
def test_crawl_org_for_acts(self, mock_session): "Test that if an exception is raised the crawl continues" mock_session.client = MagicMock() account_side_effect = [] paginator_side_effect = [] ou_ids = ["r-0", "ou-0", "ou-1", "ou-2", "sou-0"] for ou_id in ou_ids: parent_acts = _generate_act_for_parent_side_effect( self.schema, ou_id) account_side_effect.extend(parent_acts) paginator = MagicMock() paginator.paginate( ParentId=ou_id ).build_full_result.return_value = self.paginator_dict[ou_id] paginator_side_effect.append(paginator) unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.return_value = { "Roots": [{ "Id": "r-0", "Arn": "arn-0", "Name": "root_0" }] } unit_crawler._client.list_accounts_for_parent.side_effect = account_side_effect unit_crawler._client.get_paginator.side_effect = paginator_side_effect unit_crawler.crawl_account_hierarchy() with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() total_entries = (len(ou_ids) * GEN_NUM_ACT_DEFAULT) + len(ou_ids) self.assertEqual(cur_count, total_entries)
def test_crawl_accounts_per_id(self, mock_session): """Test that the accounts are depaginated and saved to db.""" mock_session.client = MagicMock() parent_id = "big_sub_org" unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() side_effect_list = _generate_act_for_parent_side_effect(self.schema, parent_id, 3) unit_crawler._build_accout_alias_map() unit_crawler._client.list_accounts_for_parent.side_effect = side_effect_list unit_crawler._structure_yesterday = {} prefix = f"root&{parent_id}" ou = {"Id": parent_id, "Name": "Big Org Unit"} unit_crawler._crawl_accounts_per_id(ou, prefix, level=1) with schema_context(self.schema): acts_in_db = AWSOrganizationalUnit.objects.filter(account_alias__isnull=False) self.assertIsNotNone(acts_in_db) self.assertEqual(acts_in_db.count(), 3)
def test_crawl_boto_param_exception(self, mock_session): """Test botocore parameter exception is caught properly.""" logging.disable(logging.NOTSET) mock_session.client = MagicMock() unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.side_effect = ParamValidationError(report="Bad Param") with self.assertLogs(logger=crawler_log, level=logging.WARNING): unit_crawler.crawl_account_hierarchy()
def test_general_client_error_denied(self, mock_session): """Test botocore general ClientError.""" logging.disable(logging.NOTSET) mock_session.client = MagicMock() unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.side_effect = _mock_boto3_general_client_error with self.assertLogs(logger=crawler_log, level=logging.WARNING): unit_crawler.crawl_account_hierarchy()
def test_unknown_exception(self, mock_session): """Test botocore general ClientError.""" logging.disable(logging.NOTSET) mock_session.client = MagicMock() unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.side_effect = Exception("unknown error") with self.assertLogs(logger=crawler_log, level=logging.raiseExceptions): unit_crawler.crawl_account_hierarchy()
def test_crawl_list_root_access_denied(self, mock_session): """Test botocore list roots access denied.""" # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html logging.disable(logging.NOTSET) mock_session.client = MagicMock() unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() unit_crawler._client.list_roots.side_effect = _mock_boto3_access_denied with self.assertLogs(logger=crawler_log, level=logging.WARNING): unit_crawler.crawl_account_hierarchy()
def test_depaginate(self, mock_session): """Test the aws account info is depaginated""" mock_session.client = MagicMock() parent_id = "TestDepaginate" unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() side_effect_list = _generate_act_for_parent_side_effect(self.schema, parent_id, 3) unit_crawler._build_accout_alias_map() unit_crawler._client.list_accounts_for_parent.side_effect = side_effect_list accounts = unit_crawler._depaginate_account_list( function=unit_crawler._client.list_accounts_for_parent, resource_key="Accounts", ParentId=parent_id ) self.assertIsNotNone(accounts) self.assertEqual(len(accounts), len(side_effect_list)) for side_effect in side_effect_list: expected_account = side_effect["Accounts"][0] self.assertIn(expected_account, accounts)
def test_compute_org_structure_interval(self): """Test function that computes org structure for an interval.""" unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._build_accout_alias_map() with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 0) unit_crawler._structure_yesterday = {} # Add root node with 1 account created_nodes = [] root = {"Id": "R_001", "Name": "root"} root_account = {"Id": "A_001", "Name": "Root Account"} created_nodes.append( unit_crawler._save_aws_org_method(root, "R_001", 0, None)) created_nodes.append( unit_crawler._save_aws_org_method(root, "R_001", 0, root_account)) # Add sub_org_unit_1 with 2 accounts sub_org_unit_1 = {"Id": "OU_1000", "Name": "sub_org_unit_1"} created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_1, "R_001&OU_1000", 1, None)) created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_1, "R_001&OU_1000", 1, { "Id": "A_002", "Name": "Sub Org Account 2" })) created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_1, "R_001&OU_1000", 1, { "Id": "A_003", "Name": "Sub Org Account 3" })) # Change created date to two_days_ago with schema_context(self.schema): two_days_ago = (unit_crawler._date_accessor.today() - timedelta(2)).strftime("%Y-%m-%d") for node in created_nodes: node.created_timestamp = two_days_ago node.save() curr_count = AWSOrganizationalUnit.objects.filter( created_timestamp__lte=two_days_ago).count() self.assertEqual(curr_count, 5) expected_count_2_days_ago = curr_count # # Add sub_org_unit_2 and move sub_org_unit_1 2 accounts here created_nodes = [] sub_org_unit_2 = {"Id": "OU_2000", "Name": "sub_org_unit_2"} created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_2, "R_001&OU_2000", 1, None)) created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_2, "R_001&OU_2000", 1, { "Id": "A_002", "Name": "Sub Org Account 2" })) created_nodes.append( unit_crawler._save_aws_org_method(sub_org_unit_2, "R_001&OU_2000", 1, { "Id": "A_003", "Name": "Sub Org Account 3" })) deleted_nodes = unit_crawler._delete_aws_org_unit("OU_1000") # Test fake node delete unit_crawler._delete_aws_org_unit("sub_org_unit_1_Fake") with schema_context(self.schema): yesterday = (unit_crawler._date_accessor.today() - timedelta(1)).strftime("%Y-%m-%d") for node in created_nodes: node.created_timestamp = yesterday node.save() for node in deleted_nodes: node.deleted_timestamp = yesterday node.save() curr_count = AWSOrganizationalUnit.objects.filter( created_timestamp__lte=yesterday).count() deleted_count = AWSOrganizationalUnit.objects.filter( deleted_timestamp__lte=yesterday).count() self.assertEqual(curr_count, 8) self.assertEqual(deleted_count, 3) expected_yesterday_count = curr_count - deleted_count unit_crawler._delete_aws_account("A_002") sub_org_unit_2 = {"Id": "OU_3000", "Name": "sub_org_unit_3"} unit_crawler._save_aws_org_method(sub_org_unit_2, "R_001&OU_3000", 1, None) with schema_context(self.schema): today = unit_crawler._date_accessor.today().strftime("%Y-%m-%d") curr_count = AWSOrganizationalUnit.objects.filter( created_timestamp__lte=today).count() deleted_count = AWSOrganizationalUnit.objects.filter( deleted_timestamp__lte=today).count() self.assertEqual(curr_count, 9) expected_today_count = curr_count - deleted_count # 2 days ago count matches structure_2_days_ago = unit_crawler._compute_org_structure_interval( two_days_ago) self.assertEqual(expected_count_2_days_ago, len(structure_2_days_ago)) # Yesterday count matches unit_crawler._compute_org_structure_yesterday() self.assertEqual(expected_yesterday_count, len(unit_crawler._structure_yesterday)) # today structure_today = unit_crawler._compute_org_structure_interval(today) self.assertEqual(len(structure_today), expected_today_count)
def test_save_aws_org_method(self): """Test that saving to the database works.""" unit_crawler = AWSOrgUnitCrawler(self.account) with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 0) AWSAccountAlias.objects.create(account_id="A_001", account_alias="Root Account") unit_crawler._build_accout_alias_map() unit_crawler._structure_yesterday = {} # Test that we are using a get or create so only one entry should be found. root_ou = {"Id": "R_001", "Name": "root"} root_account = {"Id": "A_001", "Name": "Root Account"} unit_crawler._save_aws_org_method(root_ou, "unit_path", 0, root_account) unit_crawler._save_aws_org_method(root_ou, "unit_path", 0, root_account) with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 1) # simulate an account being moved into a big org big_ou = {"Id": "R_001", "Name": "Big0"} unit_crawler._save_aws_org_method(big_ou, "unit_path&big_org", 1, root_account) with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 2) # simulate a leaf node being added without an account_id unit_crawler._save_aws_org_method(root_ou, "unit_path", 0) with schema_context(self.schema): cur_count = AWSOrganizationalUnit.objects.count() self.assertEqual(cur_count, 3)
def test_init_session(self, mock_session): """Test the method that retrieves of a aws client.""" unit_crawler = AWSOrgUnitCrawler(self.account) unit_crawler._init_session() mock_session.assert_called()