def test_load_cloudasset_data_cai_valueerror(self):
"""Validate load_cloud_asset raises exception on bad root resource."""
inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0,
{
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.mock_export_assets.side_effect = (ValueError(
'parent must start with folders/, projects/, or '
'organizations/'))
with self.assertRaises(ValueError):
cloudasset.load_cloudasset_data(self.engine, inventory_config)
self.assertFalse(self.mock_download.called)
self.validate_no_data_in_table()
def test_load_cloudasset_data(self):
"""Validate load_cloudasset_data correctly dumps and imports data."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock download to return correct test data file
def _fake_download(self, full_bucket_path, output_file):
"""Fake copy_file_from_gcs."""
if 'resource' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
elif 'org_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_org_policies.dump')
elif 'access_policy' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_access_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
self.mock_download.side_effect = _fake_download
results = cloudasset.load_cloudasset_data(self.engine,
self.inventory_config,
self.inventory_index_id)
self.assertTrue(results)
self.validate_data_in_table()
def _api_client_factory(storage, config, parallel):
"""Creates the proper initialized API client based on the configuration.
Args:
storage (object): Storage implementation to use.
config (object): Inventory configuration on server.
parallel (bool): If true, use the parallel crawler implementation.
Returns:
Union[gcp.ApiClientImpl, cai_gcp_client.CaiApiClientImpl]:
The initialized api client implementation class.
"""
client_config = config.get_api_quota_configs()
client_config['domain_super_admin_email'] = config.get_gsuite_admin_email()
client_config['excluded_resources'] = config.get_excluded_resources()
asset_count = 0
if config.get_cai_enabled():
# TODO: When CAI supports resource exclusion, update the following
# method to handle resource exclusion during export time.
asset_count = cloudasset.load_cloudasset_data(storage.session, config)
LOGGER.info('%s total assets loaded from Cloud Asset data.',
asset_count)
if asset_count:
engine = config.get_service_config().get_engine()
return cai_gcp_client.CaiApiClientImpl(client_config, engine, parallel,
storage.session)
# Default to the non-CAI implementation
return gcp.ApiClientImpl(client_config)
def _api_client_factory(config, threads):
"""Creates the proper initialized API client based on the configuration.
Args:
config (object): Inventory configuration on server.
threads (int): how many threads to use.
Returns:
Union[gcp.ApiClientImpl, cai_gcp_client.CaiApiClientImpl]:
The initialized api client implementation class.
"""
client_config = config.get_api_quota_configs()
client_config['domain_super_admin_email'] = config.get_gsuite_admin_email()
client_config['excluded_resources'] = config.get_excluded_resources()
if config.get_cai_enabled():
# TODO: When CAI supports resource exclusion, update the following
# method to handle resource exclusion during export time.
engine, tmpfile = cai_temporary_storage.create_sqlite_db(threads)
asset_count = cloudasset.load_cloudasset_data(engine, config)
LOGGER.info('%s total assets loaded from Cloud Asset data.',
asset_count)
if asset_count:
return cai_gcp_client.CaiApiClientImpl(client_config, engine,
tmpfile)
# Default to the non-CAI implementation
return gcp.ApiClientImpl(client_config)
def test_load_cloudasset_data_composite_root(self):
"""Validate load_cloudasset_data correctly works with composite root."""
composite_root_resources = ['projects/1043', 'projects/1044']
inventory_config = InventoryConfig(None, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
}, composite_root_resources)
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock download to return correct test data file
def _fake_download(self, full_bucket_path, output_file):
"""Fake copy_file_from_gcs."""
if 'resource' in full_bucket_path:
if 'projects-1043' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project3_resources.dump')
if 'projects-1044' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project4_resources.dump')
elif 'iam_policy' in full_bucket_path:
if 'projects-1043' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project3_iam_policies.dump')
if 'projects-1044' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_project4_iam_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
self.mock_download.side_effect = _fake_download
results = cloudasset.load_cloudasset_data(self.engine,
inventory_config,
self.inventory_index_id)
expected_results = 12 # Total of resources and IAM policies in dumps.
self.assertEqual(expected_results, results)
# Validate data from both projects in database.
for root_id in composite_root_resources:
for content_type in [
cai_temporary_storage.ContentTypes.resource,
cai_temporary_storage.ContentTypes.iam_policy
]:
expected_resource_name = (
'//cloudresourcemanager.googleapis.com/%s' % root_id)
resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
content_type,
'cloudresourcemanager.googleapis.com/Project',
expected_resource_name, self.engine)
self.assertTrue(resource,
msg=('Resource %s type %s is missing' %
(root_id, content_type)))
def test_load_cloudasset_data_cai_error_response(self):
"""Validate load_cloud_asset handles an error result from CAI."""
self.mock_export_assets.return_value = EXPORT_ASSETS_ERROR
results = cloudasset.load_cloudasset_data(self.engine,
self.inventory_config)
self.assertIsNone(results)
self.assertFalse(self.mock_download.called)
self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_timeout(self):
"""Validate load_cloud_asset handles a timeout error."""
self.mock_export_assets.side_effect = (
api_errors.OperationTimeoutError('organizations/987654321', {}))
results = cloudasset.load_cloudasset_data(self.engine,
self.inventory_config)
self.assertIsNone(results)
self.assertFalse(self.mock_download.called)
self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_valueerror(self):
"""Validate load_cloud_asset handles a bad root resource id."""
inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0,
{
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.mock_export_assets.side_effect = (ValueError(
'parent must start with folders/, projects/, or '
'organizations/'))
results = cloudasset.load_cloudasset_data(self.session,
inventory_config)
self.assertIsNone(results)
self.assertFalse(self.mock_copy_file_from_gcs.called)
self.validate_no_data_in_table()
def test_load_cloudasset_data_download_error(self):
"""Validate load_cloud_asset handles an error downloading from GCS."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
response = httplib2.Response(
{'status': '403', 'content-type': 'application/json'})
content = PERMISSION_DENIED.encode()
error_403 = errors.HttpError(response, content)
self.mock_copy_file_from_gcs.side_effect = error_403
results = cloudasset.load_cloudasset_data(self.session,
self.inventory_config)
self.assertIsNone(results)
self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_apierror(self):
"""Validate load_cloud_asset handles an API error from CAI."""
response = httplib2.Response(
{'status': '403', 'content-type': 'application/json'})
content = PERMISSION_DENIED.encode()
error_403 = errors.HttpError(response, content)
self.mock_export_assets.side_effect = (
api_errors.ApiExecutionError('organizations/987654321', error_403)
)
results = cloudasset.load_cloudasset_data(self.session,
self.inventory_config)
self.assertIsNone(results)
self.assertFalse(self.mock_copy_file_from_gcs.called)
self.validate_no_data_in_table()
def test_long_resource_name(self):
"""Validate load_cloudasset_data handles resources with long names."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock download to return correct test data file
def _fake_download(self, full_bucket_path, output_file):
"""Fake copy_file_from_gcs."""
if 'resource' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_long_resource_name.dump')
elif 'iam_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_iam_policies.dump')
elif 'org_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_org_policies.dump')
elif 'access_policy' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_access_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
self.mock_download.side_effect = _fake_download
results = cloudasset.load_cloudasset_data(self.engine,
self.inventory_config,
self.inventory_index_id)
# Expect both resources got imported.
expected_results = 2
self.assertEqual(results, expected_results)
cai_type = 'spanner.googleapis.com/Instance'
cai_name = '//spanner.googleapis.com/projects/project2/instances/test123'
# Validate resource with short name is in database.
resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
cai_temporary_storage.ContentTypes.resource, cai_type, cai_name,
self.engine)
expected_resource = ({
'config': 'projects/project2/instanceConfigs/regional-us-east1',
'displayName': 'Test123',
'name': 'projects/project2/instances/test123',
'nodeCount': 1,
'state': 'READY'
}, AssetMetadata(cai_type=cai_type, cai_name=cai_name))
self.assertEqual(expected_resource, resource)
def test_load_cloudasset_data(self):
"""Validate load_cloudasset_data correctly dumps and imports data."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
if 'resource' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
results = cloudasset.load_cloudasset_data(self.session,
self.inventory_config)
self.assertTrue(results)
self.validate_data_in_table()
def test_long_resource_name(self):
"""Validate load_cloudasset_data handles resources with long names."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
if 'resource' in file_path:
return os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_long_resource_name.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
results = cloudasset.load_cloudasset_data(self.session,
self.inventory_config)
# Expect only the resource with the short name got imported.
expected_results = 1
self.assertEqual(results, expected_results)
cai_type = 'spanner.googleapis.com/Instance'
cai_name = '//spanner.googleapis.com/projects/project2/instances/test123'
# Validate resource with short name is in database.
resource = storage.CaiDataAccess.fetch_cai_asset(
storage.ContentTypes.resource,
cai_type,
cai_name,
self.session)
expected_resource = ({
'config': 'projects/project2/instanceConfigs/regional-us-east1',
'displayName': 'Test123',
'name': 'projects/project2/instances/test123',
'nodeCount': 1,
'state': 'READY'}, AssetMetadata(cai_type=cai_type, cai_name=cai_name))
self.assertEqual(expected_resource, resource)