def test_credential_from_keyfile_raises(self):
"""Validate that an invalid credential file raises exception."""
with unittest_utils.create_temp_file(b'{}') as f:
with self.assertRaises(api_errors.ApiInitializationError):
api_helpers.credential_from_keyfile(
f, fake_key_file.FAKE_REQUIRED_SCOPES,
'*****@*****.**')
def test_credential_from_keyfile(self, signer_factory):
"""Validate with a valid test credential file."""
test_delegate = '*****@*****.**'
with unittest_utils.create_temp_file(fake_key_file.FAKE_KEYFILE) as f:
credentials = api_helpers.credential_from_keyfile(
f, fake_key_file.FAKE_REQUIRED_SCOPES, test_delegate)
self.assertEqual(credentials._kwargs['sub'], test_delegate)
def test_upload_text_file(self):
"""Test upload text file."""
http_mocks.mock_http_response(u'{}')
with unittest_utils.create_temp_file(b'12345') as temp_file:
result = self.gcs_api_client.put_text_file(
temp_file, 'gs://{}/{}'.format(fake_storage.FAKE_BUCKET_NAME,
fake_storage.FAKE_OBJECT_NAME))
self.assertEqual({}, result)
def test_upload_text_file_raises(self):
"""Test upload text access forbidden."""
http_mocks.mock_http_response(fake_storage.ACCESS_FORBIDDEN, '403')
with self.assertRaises(storage.errors.HttpError):
with unittest_utils.create_temp_file(b'12345') as temp_file:
self.gcs_api_client.put_text_file(
temp_file,
'gs://{}/{}'.format(fake_storage.FAKE_BUCKET_NAME,
fake_storage.FAKE_OBJECT_NAME))
def setUpClass(cls, mock_default_credential, signer_factory):
"""Set up."""
with unittest_utils.create_temp_file(
fake_key_file.FAKE_KEYFILE) as key_file:
fake_global_configs = {
'groups_service_account_key_file': key_file,
'domain_super_admin_email': '*****@*****.**',
'max_admin_api_calls_per_100_seconds': 1500
}
cls.ad_api_client = admin.AdminDirectoryClient(fake_global_configs)
mock_default_credential.assert_not_called()
# Override _use_cached_http so we can use mock http response objects
cls.ad_api_client.repository._use_cached_http = True
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig('fake_engine'))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
filtered_org = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_org_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_org.append(line)
filtered_org = ''.join(filtered_org)
filtered_access = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_access_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_access.append(line)
filtered_access = ''.join(filtered_access)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
with unittest_utils.create_temp_file(
filtered_org) as org_policies:
with unittest_utils.create_temp_file(
filtered_access) as access_policies:
# Mock download to return correct test data file
def _fake_download(full_bucket_path, output_file):
if 'resource' in full_bucket_path:
fake_file = resources
elif 'iam_policy' in full_bucket_path:
fake_file = iam_policies
elif 'org_policy' in full_bucket_path:
fake_file = org_policies
elif 'access_policy' in full_bucket_path:
fake_file = access_policies
with open(fake_file, 'rb') as f:
output_file.write(f.read())
with MemoryStorage() as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
gcp_mocks.mock_storage.download.side_effect = (
_fake_download)
run_crawler(storage, progresser,
inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ORG_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
output_config=mock.ANY,
content_type='ACCESS_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls,
any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_access_level': {
'resource': 3
},
'crm_access_policy': {
'resource': 1
},
'crm_org_policy': {
'resource': 3
},
'crm_service_perimeter': {
'resource': 1
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)
def test_crawl_cai_data_with_asset_types(self):
"""Validate including asset_types in the CAI inventory config works."""
asset_types = [
'cloudresourcemanager.googleapis.com/Folder',
'cloudresourcemanager.googleapis.com/Organization',
'cloudresourcemanager.googleapis.com/Project'
]
inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID, '',
{}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket',
'asset_types': asset_types
})
inventory_config.set_service_config(FakeServerConfig(self.engine))
# Create subsets of the mock resource dumps that only contain the
# filtered asset types
filtered_assets = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_assets.append(line)
filtered_assets = ''.join(filtered_assets)
filtered_iam = []
with open(
os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump'), 'r') as f:
for line in f:
if any('"%s"' % asset_type in line
for asset_type in asset_types):
filtered_iam.append(line)
filtered_iam = ''.join(filtered_iam)
with unittest_utils.create_temp_file(filtered_assets) as resources:
with unittest_utils.create_temp_file(filtered_iam) as iam_policies:
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return resources
elif 'iam_policy' in file_path:
return iam_policies
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
with MemoryStorage(session=self.session) as storage:
progresser = NullProgresser()
with gcp_api_mocks.mock_gcp() as gcp_mocks:
run_crawler(storage, progresser, inventory_config)
# Validate export_assets called with asset_types
expected_calls = [
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='RESOURCE',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY),
mock.call(gcp_api_mocks.ORGANIZATION_ID,
mock.ANY,
content_type='IAM_POLICY',
asset_types=asset_types,
blocking=mock.ANY,
timeout=mock.ANY)
]
(gcp_mocks.mock_cloudasset.export_assets.
assert_has_calls(expected_calls, any_order=True))
self.assertEqual(0, progresser.errors,
'No errors should have occurred')
result_counts = self._get_resource_counts_from_storage(
storage)
expected_counts = {
'crm_org_policy': {
'resource': 5
},
'folder': {
'iam_policy': 3,
'resource': 3
},
'gsuite_group': {
'resource': 4
},
'gsuite_group_member': {
'resource': 1
},
'gsuite_groups_settings': {
'resource': 4
},
'gsuite_user': {
'resource': 4
},
'gsuite_user_member': {
'resource': 3
},
'kubernetes_cluster': {
'resource': 1,
'service_config': 1
},
'lien': {
'resource': 1
},
'organization': {
'iam_policy': 1,
'resource': 1
},
'project': {
'billing_info': 4,
'enabled_apis': 4,
'iam_policy': 4,
'resource': 4
},
'role': {
'resource': 18
},
'sink': {
'resource': 6
},
}
self.assertEqual(expected_counts, result_counts)