def validate_data_in_table(self):
"""Validate there is actual data in the CAI table."""
cai_name = '//cloudresourcemanager.googleapis.com/organizations/111222333'
cai_type = 'cloudresourcemanager.googleapis.com/Organization'
resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
cai_temporary_storage.ContentTypes.resource, cai_type, cai_name,
self.engine)
expected_resource = ({
'creationTime': '2015-09-09T19:34:18.591Z',
'displayName': 'forseti.test',
'lifecycleState': 'ACTIVE',
'name': 'organizations/111222333',
'owner': {
'directoryCustomerId': 'ABC123DEF'
}
}, AssetMetadata(cai_name=cai_name, cai_type=cai_type))
self.assertEqual(expected_resource, resource)
cai_name = '//cloudresourcemanager.googleapis.com/folders/1033'
cai_type = 'cloudresourcemanager.googleapis.com/Folder'
iam_policy = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
cai_temporary_storage.ContentTypes.iam_policy, cai_type, cai_name,
self.engine)
expected_iam_policy = ({
'bindings': [{
'members': ['user:[email protected]'],
'role': 'roles/resourcemanager.folderAdmin'
}]
}, AssetMetadata(cai_name=cai_name, cai_type=cai_type))
self.assertEqual(expected_iam_policy, iam_policy)
def test_fetch_cai_asset(self):
"""Validate querying single CAI asset."""
self._add_iam_policies()
cai_type = 'cloudresourcemanager.googleapis.com/Organization'
cai_name = '//cloudresourcemanager.googleapis.com/organizations/1234567890'
results = CaiDataAccess.fetch_cai_asset(ContentTypes.iam_policy,
cai_type, cai_name,
self.session)
expected_iam_policy = {
'etag':
'BwVvLqcT+M4=',
'bindings': [{
'role': 'roles/Owner',
'members': ['user:[email protected]']
}, {
'role':
'roles/Viewer',
'members': [('serviceAccount:forseti-server-gcp-d9fffac'
'@forseti-test-project.iam.gserviceaccount.com'),
'user:[email protected]']
}]
}
self.assertEqual((expected_iam_policy,
AssetMetadata(cai_type=cai_type, cai_name=cai_name)),
results)
def test_iter_cai_assets(self):
"""Validate querying CAI asset data."""
self._add_resources()
cai_type = 'cloudresourcemanager.googleapis.com/Folder'
results = cai_temporary_storage.CaiDataAccess.iter_cai_assets(
cai_temporary_storage.ContentTypes.resource, cai_type,
'//cloudresourcemanager.googleapis.com/organizations/1234567890',
self.engine)
expected_results = [
('folders/11111',
AssetMetadata(
cai_type=cai_type,
cai_name='//cloudresourcemanager.googleapis.com/folders/11111'
))
]
self.assertEqual(expected_results, [(asset['name'], metadata)
for asset, metadata in results])
cai_type = 'appengine.googleapis.com/Service'
results = cai_temporary_storage.CaiDataAccess.iter_cai_assets(
cai_temporary_storage.ContentTypes.resource, cai_type,
'//appengine.googleapis.com/apps/forseti-test-project',
self.engine)
expected_results = [
('apps/forseti-test-project/services/default',
AssetMetadata(cai_name=(
'//appengine.googleapis.com/apps/forseti-test-project/'
'services/default'),
cai_type=cai_type))
]
self.assertEqual(expected_results, [(asset['name'], metadata)
for asset, metadata in results])
def _extract_asset_data(row):
"""Extracts the data from the database row.
Args:
row (dict): Database row from select query.
Returns:
Tuple[dict, AssetMetadata]: The dict representation of the asset
data and an Asset metadata along with it.
"""
asset = json.loads(row['asset_data'])
asset_metadata = AssetMetadata(cai_name=row['name'],
cai_type=row['asset_type'])
return asset, asset_metadata
def test_long_resource_name(self):
"""Validate load_cloudasset_data handles resources with long names."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock download to return correct test data file
def _fake_download(self, full_bucket_path, output_file):
"""Fake copy_file_from_gcs."""
if 'resource' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_long_resource_name.dump')
elif 'iam_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_iam_policies.dump')
elif 'org_policy' in full_bucket_path:
fake_file = os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_org_policies.dump')
elif 'access_policy' in full_bucket_path:
fake_file = os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_access_policies.dump')
with open(fake_file, 'rb') as f:
output_file.write(f.read())
self.mock_download.side_effect = _fake_download
results = cloudasset.load_cloudasset_data(self.engine,
self.inventory_config,
self.inventory_index_id)
# Expect both resources got imported.
expected_results = 2
self.assertEqual(results, expected_results)
cai_type = 'spanner.googleapis.com/Instance'
cai_name = '//spanner.googleapis.com/projects/project2/instances/test123'
# Validate resource with short name is in database.
resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset(
cai_temporary_storage.ContentTypes.resource, cai_type, cai_name,
self.engine)
expected_resource = ({
'config': 'projects/project2/instanceConfigs/regional-us-east1',
'displayName': 'Test123',
'name': 'projects/project2/instances/test123',
'nodeCount': 1,
'state': 'READY'
}, AssetMetadata(cai_type=cai_type, cai_name=cai_name))
self.assertEqual(expected_resource, resource)
def extract_asset_data(self, content_type):
"""Extracts the data from the asset protobuf based on the content type.
Args:
content_type (ContentTypes): The content type data to extract.
Returns:
Tuple[dict, AssetMetadata]: The dict representation of the asset
data and an Asset metadata along with it.
"""
asset = json.loads(self.asset_data)
if content_type == ContentTypes.resource:
asset = asset['resource']['data']
elif content_type == ContentTypes.iam_policy:
asset = asset['iam_policy']
asset_metadata = AssetMetadata(cai_name=self.name,
cai_type=self.asset_type)
return asset, asset_metadata
def test_long_resource_name(self):
"""Validate load_cloudasset_data handles resources with long names."""
# Ignore call to export_assets for this test.
self.mock_export_assets.return_value = {'done': True}
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
if 'resource' in file_path:
return os.path.join(
TEST_RESOURCE_DIR_PATH,
'mock_cai_long_resource_name.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_empty_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
results = cloudasset.load_cloudasset_data(self.session,
self.inventory_config)
# Expect only the resource with the short name got imported.
expected_results = 1
self.assertEqual(results, expected_results)
cai_type = 'spanner.googleapis.com/Instance'
cai_name = '//spanner.googleapis.com/projects/project2/instances/test123'
# Validate resource with short name is in database.
resource = storage.CaiDataAccess.fetch_cai_asset(
storage.ContentTypes.resource,
cai_type,
cai_name,
self.session)
expected_resource = ({
'config': 'projects/project2/instanceConfigs/regional-us-east1',
'displayName': 'Test123',
'name': 'projects/project2/instances/test123',
'nodeCount': 1,
'state': 'READY'}, AssetMetadata(cai_type=cai_type, cai_name=cai_name))
self.assertEqual(expected_resource, resource)
from google.cloud.forseti.services.inventory.storage import (
Categories, DataAccess, initialize, InventoryIndex, Storage)
from sqlalchemy.orm import sessionmaker
from tests.services.util.db import create_test_engine_with_file
from tests.services.util.mock import ResourceMock
from tests.unittest_utils import ForsetiTestCase
MOCK_ACCESS_POLICY = [
(
{
'name': 'accessPolicies/678657630408',
'parent': 'organizations/92932930834',
'title': 'default policy'
},
AssetMetadata(cai_name='accessPolicies/678657630408',
cai_type='cloudresourcemanager.googleapis.com/Organization')
)]
MOCK_ORG_POLICY = [
(
{
'boolean_policy': {'enforced': True},
'constraint': 'constraints/appengine.disableCodeDownload',
'update_time': {'nanos': 712000000, 'seconds': 1579031330}
},
AssetMetadata(cai_name='constraints/appengine.disableCodeDownload',
cai_type='cloudresourcemanager.googleapis.com/Organization')
)]
class StorageTest(ForsetiTestCase):