def test_crawling_from_folder(self):
"""Crawl from folder, verify expected resources crawled."""
config = InventoryConfig(
'folders/1032',
'',
{},
'',
{})
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = {
'appengine_app': {'resource': 1},
'appengine_instance': {'resource': 3},
'appengine_service': {'resource': 1},
'appengine_version': {'resource': 1},
'bucket': {'gcs_policy': 1, 'iam_policy': 1, 'resource': 1},
'folder': {'iam_policy': 2, 'resource': 2},
'project': {'billing_info': 1, 'enabled_apis': 1, 'iam_policy': 1,
'resource': 1},
'role': {'resource': 1},
'sink': {'resource': 1},
}
self.assertEqual(expected_counts, result_counts)
def test_crawling_to_memory_storage_exclude_all_folders_and_projects_using_projectNumber(self):
"""Crawl mock environment, test that all the folders are excluded."""
config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID,
'',
{},
'',
{},
excluded_resources=['folders/1031', 'folders/1032',
'projects/1041', 'projects/1042'])
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = {
'billing_account': {'iam_policy': 2, 'resource': 2},
'crm_org_policy': {'resource': 2},
'gsuite_group': {'resource': 4},
'gsuite_group_member': {'resource': 1},
'gsuite_groups_settings': {'resource': 4},
'gsuite_user': {'resource': 4},
'gsuite_user_member': {'resource': 3},
'organization': {'iam_policy': 1, 'resource': 1},
'role': {'resource': 19},
'sink': {'resource': 2}
}
self.assertEqual(expected_counts, result_counts)
def setUp(self):
"""Setup method."""
CrawlerBase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
session_maker = sessionmaker()
self.session = session_maker(bind=self.engine)
initialize(self.engine)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(FakeServerConfig(self.engine))
# Ensure test data doesn't get deleted
self.mock_unlink = mock.patch.object(os, 'unlink',
autospec=True).start()
self.mock_copy_file_from_gcs = mock.patch.object(
file_loader, 'copy_file_from_gcs', autospec=True).start()
self.maxDiff = None
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
def test_crawling_no_org_access(self):
"""Crawl with no access to organization, only child projects."""
config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID,
'',
{},
'',
{})
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config, has_org_access=False)
# The crawl should be the same as test_crawling_to_memory_storage, but
# without organization iam_policy, org_policy (needs Org access) or
# gsuite_* resources (needs directoryCustomerId from Organization).
expected_counts = copy.deepcopy(GCP_API_RESOURCES)
expected_counts['organization'].pop('iam_policy')
expected_counts['crm_org_policy']['resource'] -= 2
expected_counts.pop('gsuite_group')
expected_counts.pop('gsuite_groups_settings')
expected_counts.pop('gsuite_group_member')
expected_counts.pop('gsuite_user')
expected_counts.pop('gsuite_user_member')
self.assertEqual(expected_counts, result_counts)
class TestServiceConfig(MockServerConfig):
"""ServiceConfig Stub."""
def __init__(self):
self.engine = create_test_engine()
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(10)
self.inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID,
'', {}, '', {})
self.inventory_config.set_service_config(self)
def run_in_background(self, func):
"""Stub."""
self.workers.add_func(func)
def get_engine(self):
return self.engine
def scoped_session(self):
return self.sessionmaker()
def client(self):
return ClientComposition(self.endpoint)
def get_storage_class(self):
return Storage
def get_inventory_config(self):
return self.inventory_config
def test_crawling_with_apis_disabled(self):
"""Crawl with the appengine and cloudsql APIs disabled."""
config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {
'appengine': {
'disable_polling': True
},
'sqladmin': {
'disable_polling': True
},
}, '', {})
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config, has_org_access=True)
# The crawl should be the same as test_crawling_to_memory_storage, but
# without appengine and cloudsql resources.
expected_counts = copy.deepcopy(GCP_API_RESOURCES)
expected_counts.pop('appengine_app')
expected_counts.pop('appengine_instance')
expected_counts.pop('appengine_service')
expected_counts.pop('appengine_version')
expected_counts.pop('cloudsqlinstance')
self.assertEqual(expected_counts, result_counts)
def __init__(self):
self.engine = create_test_engine()
initialize(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.inventory_config = InventoryConfig('organizations/1', '', {}, '',
{})
self.inventory_config.set_service_config(self)
def test_crawling_from_project(self):
"""Crawl from project, verify expected resources crawled."""
config = InventoryConfig(
'projects/1041',
'',
{},
'',
{})
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = {
'backendservice': {'resource': 1},
'compute_project': {'resource': 1},
'crm_org_policy': {'resource': 1},
'disk': {'resource': 3},
'firewall': {'resource': 3},
'forwardingrule': {'resource': 1},
'instance': {'resource': 3},
'instancegroup': {'resource': 2},
'instancegroupmanager': {'resource': 2},
'instancetemplate': {'resource': 2},
'kubernetes_cluster': {'resource': 1, 'service_config': 1},
'lien': {'resource': 1},
'network': {'resource': 1},
'project': {'billing_info': 1, 'enabled_apis': 1, 'iam_policy': 1,
'resource': 1},
'serviceaccount': {'iam_policy': 1, 'resource': 1},
'sink': {'resource': 2},
'snapshot': {'resource': 2},
'subnetwork': {'resource': 12},
}
self.assertEqual(expected_counts, result_counts)
def main():
"""Create CAI dump files from fake data."""
logger.enable_console_log()
config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '', {}, '',
{'enabled': False})
service_config = TestServiceConfig('sqlite', config)
config.set_service_config(service_config)
resources = []
iam_policies = []
with MemoryStorage() as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp():
run_crawler(storage, progresser, config, parallel=False)
for item in storage.mem.values():
(resource, iam_policy) = convert_item_to_assets(item)
if resource:
resources.append(resource)
if iam_policy:
iam_policies.append(iam_policy)
with open(ADDITIONAL_RESOURCES_FILE, 'r') as f:
for line in f:
if line.startswith('#'):
continue
resources.append(line.strip())
with open(ADDITIONAL_IAM_POLCIIES_FILE, 'r') as f:
for line in f:
if line.startswith('#'):
continue
iam_policies.append(line.strip())
write_data(resources, RESOURCE_DUMP_FILE)
write_data(iam_policies, IAM_POLICY_DUMP_FILE)
def __init__(self):
self.engine = create_test_engine()
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(10)
self.inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID,
'', {}, '', {})
self.inventory_config.set_service_config(self)
def __init__(self, engine):
self.engine = engine
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(1)
self.inventory_config = InventoryConfig(
None, '', {}, 0, {'enabled': False},
['folders/1032', 'projects/1041'])
self.inventory_config.set_service_config(self)
def test_crawling_to_memory_storage(self):
"""Crawl mock environment, test that there are items in storage."""
config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '', {}, '', {})
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = GCP_API_RESOURCES
self.assertEqual(expected_counts, result_counts)
def __init__(self, engine):
self.engine = engine
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(1)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(self)
def setUp(self):
"""Setup method."""
CrawlerBase.setUp(self)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(
FakeServerConfig('mock_engine'))
self.maxDiff = None
def setUp(self):
"""Setup method."""
unittest_utils.ForsetiTestCase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
_session_maker = sessionmaker()
self.session = _session_maker(bind=self.engine)
storage.initialize(self.engine)
self.inventory_config = InventoryConfig(
'organizations/987654321', '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
# Ensure test data doesn't get deleted
self.mock_unlink = mock.patch.object(os, 'unlink',
autospec=True).start()
self.mock_export_assets = mock.patch.object(
cloudasset_api.CloudAssetClient, 'export_assets',
autospec=True).start()
self.mock_copy_file_from_gcs = mock.patch.object(
file_loader, 'copy_file_from_gcs', autospec=True).start()
self.mock_auth = mock.patch.object(
google.auth,
'default',
return_value=(mock.Mock(spec_set=credentials.Credentials),
'test-project')).start()
def test_load_cloudasset_data_composite_root(self):
"""Validate load_cloudasset_data correctly works with composite root."""
composite_root_resources = ['projects/1043', 'projects/1044']
inventory_config = InventoryConfig(None, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
}, composite_root_resources)
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock download to return correct test data file
def _fake_download(self, full_bucket_path, output_file):
"""Fake copy_file_from_gcs."""
if 'resource' in full_bucket_path:
if 'projects-1043' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project3_resources.dump')
if 'projects-1044' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project4_resources.dump')
elif 'iam_policy' in full_bucket_path:
if 'projects-1043' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project3_iam_policies.dump')
if 'projects-1044' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project4_iam_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
self.mock_download.side_effect = _fake_download
results = cloudasset.load_cloudasset_data(self.engine,
inventory_config,
self.inventory_index_id)
expected_results = 12 # Total of resources and IAM policies in dumps.
self.assertEqual(expected_results, results)
# Validate data from both projects in database.
for root_id in composite_root_resources:
for content_type in [
cai_temporary_storage.ContentTypes.resource,
cai_temporary_storage.ContentTypes.iam_policy
]:
expected_resource_name = (
'//cloudresourcemanager.googleapis.com/%s' % root_id)
resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
content_type,
'cloudresourcemanager.googleapis.com/Project',
expected_resource_name, self.engine)
self.assertTrue(resource,
msg=('Resource %s type %s is missing' %
(root_id, content_type)))
def test_crawling_from_composite_root(self):
"""Crawl from composite_root with folder and project."""
config = InventoryConfig(
None,
'',
{},
'',
{},
['folders/1032', 'projects/1041'])
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = {
'appengine_app': {'resource': 1},
'appengine_instance': {'resource': 3},
'appengine_service': {'resource': 1},
'appengine_version': {'resource': 1},
'backendservice': {'resource': 1},
'bucket': {'gcs_policy': 1, 'iam_policy': 1, 'resource': 1},
'composite_root': {'resource': 1},
'compute_project': {'resource': 1},
'crm_org_policy': {'resource': 1},
'disk': {'resource': 3},
'firewall': {'resource': 3},
'folder': {'iam_policy': 2, 'resource': 2},
'forwardingrule': {'resource': 1},
'instance': {'resource': 3},
'instancegroup': {'resource': 2},
'instancegroupmanager': {'resource': 2},
'instancetemplate': {'resource': 2},
'kubernetes_cluster': {'resource': 1, 'service_config': 1},
'lien': {'resource': 1},
'network': {'resource': 1},
'project': {'billing_info': 2, 'enabled_apis': 2, 'iam_policy': 2,
'resource': 2},
'role': {'resource': 1},
'serviceaccount': {'iam_policy': 1, 'resource': 1},
'serviceaccount_key': {'resource': 1},
'sink': {'resource': 3},
'snapshot': {'resource': 2},
'subnetwork': {'resource': 12},
}
self.assertEqual(expected_counts, result_counts)
def test_crawling_from_folder_exclude_folder(self):
"""Crawl from folder, and skip one folder, verify
expected resources crawled."""
config = InventoryConfig(
'folders/1032',
'',
{},
'',
{},
excluded_resources=['folders/1033'])
config.set_service_config(FakeServerConfig('mock_engine'))
result_counts = self._run_crawler(config)
expected_counts = {
'folder': {'iam_policy': 1, 'resource': 1},
'sink': {'resource': 1},
}
self.assertEqual(expected_counts, result_counts)
class TestServiceConfig(MockServerConfig):
"""ServiceConfig Stub."""
def __init__(self):
self.engine = create_test_engine()
initialize(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.inventory_config = InventoryConfig('organizations/1', '', {}, '',
{})
self.inventory_config.set_service_config(self)
def get_engine(self):
return self.engine
def scoped_session(self):
return self.sessionmaker()
def get_storage_class(self):
return Storage
def get_inventory_config(self):
return self.inventory_config
def test_explain_is_not_supported(self):
"""Test explain is not supported."""
inventory_config_with_folder_root = (InventoryConfig(
gcp_api_mocks.FOLDER_ID, '', {}, '', {}))
setup = create_tester(inventory_config_with_folder_root)
def test(client):
"""Test implementation with API client."""
self.assertRaisesRegexp(
_Rendezvous, 'FAILED_PRECONDITION',
lambda r: list(client.explain.list_roles('')), '')
setup.run(test)
def test_load_cloudasset_data_cai_valueerror(self):
"""Validate load_cloud_asset raises exception on bad root resource."""
inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0,
{
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.mock_export_assets.side_effect = (ValueError(
'parent must start with folders/, projects/, or '
'organizations/'))
with self.assertRaises(ValueError):
cloudasset.load_cloudasset_data(self.engine, inventory_config)
self.assertFalse(self.mock_download.called)
self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_valueerror(self):
"""Validate load_cloud_asset handles a bad root resource id."""
inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0,
{
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.mock_export_assets.side_effect = (ValueError(
'parent must start with folders/, projects/, or '
'organizations/'))
results = cloudasset.load_cloudasset_data(self.session,
inventory_config)
self.assertIsNone(results)
self.assertFalse(self.mock_copy_file_from_gcs.called)
self.validate_no_data_in_table()
def test_inventory_config_exclude_resource_invalid_ids(self):
self.inventory_config = InventoryConfig(
'test-org-id',
'', {},
0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
},
excluded_resources=[
'asd', 'projects/aqwedas', 'projects/222', 'folders/badfolder',
'organizations/testing', 'organizations/12345', 'folders/12345'
])
self.assertSetEqual(set(self.inventory_config.excluded_resources), {
'project/aqwedas', 'project/222', 'organization/12345',
'folder/12345'
})
def setUp(self):
"""Setup method."""
unittest_utils.ForsetiTestCase.setUp(self)
self.engine, self.dbfile = cai_temporary_storage.create_sqlite_db()
self.inventory_config = InventoryConfig(
'organizations/987654321', '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.mock_auth = mock.patch.object(
google.auth,
'default',
return_value=(mock.Mock(spec_set=credentials.Credentials),
'test-project')).start()
self.mock_export_assets = mock.patch.object(
cloudasset_api.CloudAssetClient, 'export_assets',
autospec=True).start()
self.mock_download = mock.patch.object(storage.StorageClient,
'download',
autospec=True).start()
def test_explain_is_supported(self):
"""Test explain is supported."""
inventory_config_with_organization_root = (InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, '', {}))
setup = create_tester(inventory_config_with_organization_root)
def test(client):
"""Test implementation with API client."""
expected_reply = set([
'role/a',
'role/b',
'role/c',
'role/d',
'roles/owner',
'roles/editor',
'roles/viewer',
])
actual_reply = set(r.role_name
for r in client.explain.list_roles(''))
self.assertEqual(expected_reply, actual_reply)
setup.run(test)
class CloudAssetCrawlerTest(CrawlerBase):
"""Test CloudAsset integration with crawler."""
def setUp(self):
"""Setup method."""
CrawlerBase.setUp(self)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(
FakeServerConfig('mock_engine'))
self.maxDiff = None
def _run_crawler(self, config):
"""Runs the crawler with a specific InventoryConfig.
Args:
config (InventoryConfig): The configuration to test.
Returns:
dict: the resource counts returned by the crawler.
"""
# Mock download to return correct test data file
def _fake_download(full_bucket_path, output_file):
if 'resource' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
elif 'org_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_org_policies.dump')
elif 'access_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_access_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
with MemoryStorage() as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
gcp_mocks.mock_storage.download.side_effect = _fake_download
run_crawler(storage, progresser, config, parallel=True)
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
return self._get_resource_counts_from_storage(storage)
def tearDown(self):
"""tearDown."""
CrawlerBase.tearDown(self)
mock.patch.stopall()
def test_cai_crawl_to_memory(self):
"""Crawl mock environment, test that there are items in storage."""
result_counts = self._run_crawler(self.inventory_config)
expected_counts = copy.deepcopy(GCP_API_RESOURCES)
expected_counts.update({
'backendservice': {
'resource': 2
},
'bigquery_table': {
'resource': 1
},
'bigtable_cluster': {
'resource': 1
},
'bigtable_instance': {
'resource': 1
},
'bigtable_table': {
'resource': 1
},
'cloudsqlinstance': {
'resource': 2
},
'compute_address': {
'resource': 2
},
'compute_autoscaler': {
'resource': 1
},
'compute_backendbucket': {
'resource': 1
},
'compute_healthcheck': {
'resource': 1
},
'compute_httphealthcheck': {
'resource': 1
},
'compute_httpshealthcheck': {
'resource': 1
},
'compute_interconnect': {
'resource': 1
},
'compute_interconnect_attachment': {
'resource': 1
},
'compute_license': {
'resource': 1
},
'compute_router': {
'resource': 1
},
'compute_securitypolicy': {
'resource': 1
},
'compute_sslcertificate': {
'resource': 1
},
'compute_targethttpproxy': {
'resource': 1
},
'compute_targethttpsproxy': {
'resource': 1
},
'compute_targetinstance': {
'resource': 1
},
'compute_targetpool': {
'resource': 1
},
'compute_targetsslproxy': {
'resource': 1
},
'compute_targettcpproxy': {
'resource': 1
},
'compute_targetvpngateway': {
'resource': 1
},
'compute_urlmap': {
'resource': 1
},
'compute_vpntunnel': {
'resource': 1
},
'crm_access_level': {
'resource': 3
},
'crm_access_policy': {
'resource': 1
},
'crm_org_policy': {
'resource': 3
},
'crm_service_perimeter': {
'resource': 1
},
'dataproc_cluster': {
'resource': 2,
'iam_policy': 1
},
'dataset': {
'dataset_policy': 2,
'iam_policy': 2,
'resource': 3
},
'disk': {
'resource': 5
},
'dns_managedzone': {
'resource': 1
},
'dns_policy': {
'resource': 1
},
'forwardingrule': {
'resource': 2
},
'kms_cryptokey': {
'iam_policy': 1,
'resource': 1
},
'kms_cryptokeyversion': {
'resource': 1
},
'kms_keyring': {
'iam_policy': 1,
'resource': 1
},
'kubernetes_cluster': {
'resource': 1,
'service_config': 1
},
'kubernetes_clusterrole': {
'resource': 1
},
'kubernetes_clusterrolebinding': {
'resource': 1
},
'kubernetes_namespace': {
'resource': 1
},
'kubernetes_node': {
'resource': 1
},
'kubernetes_pod': {
'resource': 1
},
'kubernetes_role': {
'resource': 1
},
'kubernetes_rolebinding': {
'resource': 1
},
'kubernetes_service': {
'resource': 1
},
'pubsub_subscription': {
'iam_policy': 1,
'resource': 1
},
'pubsub_topic': {
'iam_policy': 1,
'resource': 1
},
'service': {
'resource': 1
},
'serviceaccount': {
'iam_policy': 2,
'resource': 3
},
'serviceaccount_key': {
'resource': 1
},
'spanner_database': {
'resource': 1
},
'spanner_instance': {
'resource': 1
},
})
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_api_polling_disabled(self):
"""Validate using only CAI and no API polling works."""
self.inventory_config.api_quota_configs = {
'admin': {
'disable_polling': True
},
'appengine': {
'disable_polling': True
},
'bigquery': {
'disable_polling': True
},
'cloudbilling': {
'disable_polling': True
},
'compute': {
'disable_polling': True
},
'container': {
'disable_polling': True
},
'crm': {
'disable_polling': True
},
'iam': {
'disable_polling': True
},
'logging': {
'disable_polling': True
},
'servicemanagement': {
'disable_polling': True
},
'serviceusage': {
'disable_polling': True
},
'sqladmin': {
'disable_polling': True
},
'storage': {
'disable_polling': True
},
}
result_counts = self._run_crawler(self.inventory_config)
# Any resource not included in Cloud Asset export will not be in the
# inventory.
expected_counts = {
'appengine_app': {
'resource': 2
},
'appengine_service': {
'resource': 1
},
'appengine_version': {
'resource': 1
},
'backendservice': {
'resource': 2
},
'bigtable_cluster': {
'resource': 1
},
'bigtable_instance': {
'resource': 1
},
'bigtable_table': {
'resource': 1
},
'billing_account': {
'iam_policy': 2,
'resource': 2
},
'bigquery_table': {
'resource': 1
},
'bucket': {
'gcs_policy': 2,
'iam_policy': 2,
'resource': 2
},
'cloudsqlinstance': {
'resource': 2
},
'compute_address': {
'resource': 2
},
'compute_autoscaler': {
'resource': 1
},
'compute_backendbucket': {
'resource': 1
},
'compute_healthcheck': {
'resource': 1
},
'compute_httphealthcheck': {
'resource': 1
},
'compute_httpshealthcheck': {
'resource': 1
},
'compute_interconnect': {
'resource': 1
},
'compute_interconnect_attachment': {
'resource': 1
},
'compute_license': {
'resource': 1
},
'compute_project': {
'resource': 2
},
'compute_router': {
'resource': 1
},
'compute_securitypolicy': {
'resource': 1
},
'compute_sslcertificate': {
'resource': 1
},
'compute_targethttpproxy': {
'resource': 1
},
'compute_targethttpsproxy': {
'resource': 1
},
'compute_targetinstance': {
'resource': 1
},
'compute_targetpool': {
'resource': 1
},
'compute_targetsslproxy': {
'resource': 1
},
'compute_targettcpproxy': {
'resource': 1
},
'compute_targetvpngateway': {
'resource': 1
},
'compute_urlmap': {
'resource': 1
},
'compute_vpntunnel': {
'resource': 1
},
'crm_access_level': {
'resource': 3
},
'crm_access_policy': {
'resource': 1
},
'crm_org_policy': {
'resource': 3
},
'crm_service_perimeter': {
'resource': 1
},
'dataproc_cluster': {
'resource': 2,
'iam_policy': 1
},
'dataset': {
'dataset_policy': 2,
'iam_policy': 2,
'resource': 3
},
'disk': {
'resource': 5
},
'dns_managedzone': {
'resource': 1
},
'dns_policy': {
'resource': 1
},
'firewall': {
'resource': 7
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'forwardingrule': {
'resource': 2
},
'image': {
'resource': 2
},
'instance': {
'resource': 4
},
'instancegroup': {
'resource': 2
},
'instancegroupmanager': {
'resource': 2
},
'instancetemplate': {
'resource': 2
},
'kms_cryptokey': {
'iam_policy': 1,
'resource': 1
},
'kms_cryptokeyversion': {
'resource': 1
},
'kms_keyring': {
'iam_policy': 1,
'resource': 1
},
'kubernetes_cluster': {
'resource': 1
},
'kubernetes_clusterrole': {
'resource': 1
},
'kubernetes_clusterrolebinding': {
'resource': 1
},
'kubernetes_namespace': {
'resource': 1
},
'kubernetes_node': {
'resource': 1
},
'kubernetes_pod': {
'resource': 1
},
'kubernetes_role': {
'resource': 1
},
'kubernetes_rolebinding': {
'resource': 1
},
'kubernetes_service': {
'resource': 1
},
'network': {
'resource': 2
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'iam_policy': 4,
'resource': 4
},
'pubsub_subscription': {
'iam_policy': 1,
'resource': 1
},
'pubsub_topic': {
'iam_policy': 1,
'resource': 1
},
'role': {
'resource': 2
},
'service': {
'resource': 1
},
'serviceaccount': {
'iam_policy': 2,
'resource': 3
},
'serviceaccount_key': {
'resource': 1
},
'snapshot': {
'resource': 3
},
'spanner_database': {
'resource': 1
},
'spanner_instance': {
'resource': 1
},
'subnetwork': {
'resource': 24
}
}
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig('fake_engine'))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
filtered_org = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_org_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_org.append(line)
filtered_org = ''.join(filtered_org)
filtered_access = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_access_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_access.append(line)
filtered_access = ''.join(filtered_access)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
with unittest_utils.create_temp_file(
filtered_org) as org_policies:
with unittest_utils.create_temp_file(
filtered_access) as access_policies:
# Mock download to return correct test data file
def _fake_download(full_bucket_path, output_file):
if 'resource' in full_bucket_path:
fake_file = resources
elif 'iam_policy' in full_bucket_path:
fake_file = iam_policies
elif 'org_policy' in full_bucket_path:
fake_file = org_policies
elif 'access_policy' in full_bucket_path:
fake_file = access_policies
with open(fake_file, 'rb') as f:
output_file.write(f.read())
with MemoryStorage() as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
gcp_mocks.mock_storage.download.side_effect = (
_fake_download)
run_crawler(storage, progresser,
inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ORG_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ACCESS_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls,
any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_access_level': {
'resource': 3
},
'crm_access_policy': {
'resource': 1
},
'crm_org_policy': {
'resource': 3
},
'crm_service_perimeter': {
'resource': 1
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig('fake_engine'))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
filtered_org = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_org_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_org.append(line)
filtered_org = ''.join(filtered_org)
filtered_access = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_access_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_access.append(line)
filtered_access = ''.join(filtered_access)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
with unittest_utils.create_temp_file(
filtered_org) as org_policies:
with unittest_utils.create_temp_file(
filtered_access) as access_policies:
# Mock download to return correct test data file
def _fake_download(full_bucket_path, output_file):
if 'resource' in full_bucket_path:
fake_file = resources
elif 'iam_policy' in full_bucket_path:
fake_file = iam_policies
elif 'org_policy' in full_bucket_path:
fake_file = org_policies
elif 'access_policy' in full_bucket_path:
fake_file = access_policies
with open(fake_file, 'rb') as f:
output_file.write(f.read())
with MemoryStorage() as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
gcp_mocks.mock_storage.download.side_effect = (
_fake_download)
run_crawler(storage, progresser,
inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ORG_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ACCESS_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls,
any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_access_level': {
'resource': 3
},
'crm_access_policy': {
'resource': 1
},
'crm_org_policy': {
'resource': 3
},
'crm_service_perimeter': {
'resource': 1
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig(self.engine))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return resources
elif 'iam_policy' in file_path:
return iam_policies
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
with MemoryStorage(session=self.session) as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
run_crawler(storage, progresser, inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls, any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_org_policy': {
'resource': 5
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'kubernetes_cluster': {
'resource': 1,
'service_config': 1
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)
class CloudAssetCrawlerTest(CrawlerBase):
"""Test CloudAsset integration with crawler."""
def setUp(self):
"""Setup method."""
CrawlerBase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
session_maker = sessionmaker()
self.session = session_maker(bind=self.engine)
initialize(self.engine)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(FakeServerConfig(self.engine))
# Ensure test data doesn't get deleted
self.mock_unlink = mock.patch.object(os, 'unlink',
autospec=True).start()
self.mock_copy_file_from_gcs = mock.patch.object(
file_loader, 'copy_file_from_gcs', autospec=True).start()
self.maxDiff = None
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
def tearDown(self):
"""tearDown."""
CrawlerBase.tearDown(self)
mock.patch.stopall()
# Stop mocks before unlinking the database file.
os.unlink(self.dbfile)
def test_cai_crawl_to_memory(self):
"""Crawl mock environment, test that there are items in storage."""
result_counts = self._run_crawler(self.inventory_config,
session=self.session)
expected_counts = copy.deepcopy(GCP_API_RESOURCES)
expected_counts.update({
'backendservice': {
'resource': 2
},
'cloudsqlinstance': {
'resource': 2
},
'compute_autoscaler': {
'resource': 1
},
'compute_backendbucket': {
'resource': 1
},
'compute_healthcheck': {
'resource': 1
},
'compute_httphealthcheck': {
'resource': 1
},
'compute_httpshealthcheck': {
'resource': 1
},
'compute_license': {
'resource': 1
},
'compute_router': {
'resource': 1
},
'compute_sslcertificate': {
'resource': 1
},
'compute_targethttpproxy': {
'resource': 1
},
'compute_targethttpsproxy': {
'resource': 1
},
'compute_targetinstance': {
'resource': 1
},
'compute_targetpool': {
'resource': 1
},
'compute_targetsslproxy': {
'resource': 1
},
'compute_targettcpproxy': {
'resource': 1
},
'compute_targetvpngateway': {
'resource': 1
},
'compute_urlmap': {
'resource': 1
},
'compute_vpntunnel': {
'resource': 1
},
'dataproc_cluster': {
'resource': 2,
'iam_policy': 1
},
'dataset': {
'dataset_policy': 2,
'iam_policy': 2,
'resource': 3
},
'dns_managedzone': {
'resource': 1
},
'dns_policy': {
'resource': 1
},
'forwardingrule': {
'resource': 2
},
'kms_cryptokey': {
'iam_policy': 1,
'resource': 1
},
'kms_cryptokeyversion': {
'resource': 1
},
'kms_keyring': {
'iam_policy': 1,
'resource': 1
},
'kubernetes_cluster': {
'resource': 1,
'service_config': 1
},
'kubernetes_clusterrole': {
'resource': 1
},
'kubernetes_clusterrolebinding': {
'resource': 1
},
'kubernetes_namespace': {
'resource': 1
},
'kubernetes_node': {
'resource': 1
},
'kubernetes_pod': {
'resource': 1
},
'kubernetes_role': {
'resource': 1
},
'kubernetes_rolebinding': {
'resource': 1
},
'pubsub_subscription': {
'iam_policy': 1,
'resource': 1
},
'pubsub_topic': {
'iam_policy': 1,
'resource': 1
},
'spanner_database': {
'resource': 1
},
'spanner_instance': {
'resource': 1
},
})
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_api_polling_disabled(self):
"""Validate using only CAI and no API polling works."""
self.inventory_config.api_quota_configs = {
'admin': {
'disable_polling': True
},
'appengine': {
'disable_polling': True
},
'bigquery': {
'disable_polling': True
},
'cloudbilling': {
'disable_polling': True
},
'compute': {
'disable_polling': True
},
'container': {
'disable_polling': True
},
'crm': {
'disable_polling': True
},
'iam': {
'disable_polling': True
},
'logging': {
'disable_polling': True
},
'servicemanagement': {
'disable_polling': True
},
'sqladmin': {
'disable_polling': True
},
'storage': {
'disable_polling': True
},
}
result_counts = self._run_crawler(self.inventory_config,
session=self.session)
# Any resource not included in Cloud Asset export will not be in the
# inventory.
expected_counts = {
'appengine_app': {
'resource': 2
},
'appengine_service': {
'resource': 1
},
'appengine_version': {
'resource': 1
},
'backendservice': {
'resource': 2
},
'billing_account': {
'iam_policy': 2,
'resource': 2
},
'bucket': {
'gcs_policy': 2,
'iam_policy': 2,
'resource': 2
},
'cloudsqlinstance': {
'resource': 2
},
'compute_autoscaler': {
'resource': 1
},
'compute_backendbucket': {
'resource': 1
},
'compute_healthcheck': {
'resource': 1
},
'compute_httphealthcheck': {
'resource': 1
},
'compute_httpshealthcheck': {
'resource': 1
},
'compute_license': {
'resource': 1
},
'compute_project': {
'resource': 2
},
'compute_router': {
'resource': 1
},
'compute_sslcertificate': {
'resource': 1
},
'compute_targethttpproxy': {
'resource': 1
},
'compute_targethttpsproxy': {
'resource': 1
},
'compute_targetinstance': {
'resource': 1
},
'compute_targetpool': {
'resource': 1
},
'compute_targetsslproxy': {
'resource': 1
},
'compute_targettcpproxy': {
'resource': 1
},
'compute_targetvpngateway': {
'resource': 1
},
'compute_urlmap': {
'resource': 1
},
'compute_vpntunnel': {
'resource': 1
},
'dataproc_cluster': {
'resource': 2,
'iam_policy': 1
},
'dataset': {
'dataset_policy': 2,
'iam_policy': 2,
'resource': 3
},
'disk': {
'resource': 4
},
'dns_managedzone': {
'resource': 1
},
'dns_policy': {
'resource': 1
},
'firewall': {
'resource': 7
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'forwardingrule': {
'resource': 2
},
'image': {
'resource': 2
},
'instance': {
'resource': 4
},
'instancegroup': {
'resource': 2
},
'instancegroupmanager': {
'resource': 2
},
'instancetemplate': {
'resource': 2
},
'kms_cryptokey': {
'iam_policy': 1,
'resource': 1
},
'kms_cryptokeyversion': {
'resource': 1
},
'kms_keyring': {
'iam_policy': 1,
'resource': 1
},
# 'kubernetes_cluster': {'resource': 1},
'network': {
'resource': 2
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'iam_policy': 4,
'resource': 4
},
'pubsub_subscription': {
'iam_policy': 1,
'resource': 1
},
'pubsub_topic': {
'iam_policy': 1,
'resource': 1
},
'role': {
'resource': 2
},
'serviceaccount': {
'iam_policy': 2,
'resource': 2
},
'snapshot': {
'resource': 3
},
'spanner_database': {
'resource': 1
},
'spanner_instance': {
'resource': 1
},
'subnetwork': {
'resource': 24
}
}
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig(self.engine))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return resources
elif 'iam_policy' in file_path:
return iam_policies
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
with MemoryStorage(session=self.session) as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
run_crawler(storage, progresser, inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls, any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_org_policy': {
'resource': 5
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'kubernetes_cluster': {
'resource': 1,
'service_config': 1
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)
def __init__(self):
self.engine = create_test_engine()
self.model_manager = ModelManager(self.engine)
self.inventory_config = (
InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '', {}, '', {}))
