def test_clear_cai_data(self):
"""Validate CAI data delete."""
self._add_resources()
rows = CaiDataAccess.clear_cai_data(self.session)
expected_rows = len(CAI_RESOURCE_DATA.split('\n'))
self.assertEqual(expected_rows, rows)
results = CaiDataAccess.iter_cai_assets(
ContentTypes.resource,
'cloudresourcemanager.googleapis.com/Folder',
'//cloudresourcemanager.googleapis.com/organizations/1234567890',
self.session)
self.assertEqual(0, len(list(results)))
def test_fetch_cai_asset(self):
"""Validate querying single CAI asset."""
self._add_iam_policies()
cai_type = 'cloudresourcemanager.googleapis.com/Organization'
cai_name = '//cloudresourcemanager.googleapis.com/organizations/1234567890'
results = CaiDataAccess.fetch_cai_asset(ContentTypes.iam_policy,
cai_type, cai_name,
self.session)
expected_iam_policy = {
'etag':
'BwVvLqcT+M4=',
'bindings': [{
'role': 'roles/Owner',
'members': ['user:[email protected]']
}, {
'role':
'roles/Viewer',
'members': [('serviceAccount:forseti-server-gcp-d9fffac'
'@forseti-test-project.iam.gserviceaccount.com'),
'user:[email protected]']
}]
}
self.assertEqual((expected_iam_policy,
AssetMetadata(cai_type=cai_type, cai_name=cai_name)),
results)
def _clear_cai_data(session):
"""Clear CAI data from storage.
Args:
session (object): Database session.
"""
LOGGER.debug('Deleting Cloud Asset data from database.')
count = CaiDataAccess.clear_cai_data(session)
LOGGER.debug('%s assets deleted from database.', count)
return None
def test_iter_cai_assets(self):
"""Validate querying CAI asset data."""
self._add_resources()
results = CaiDataAccess.iter_cai_assets(
ContentTypes.resource,
'cloudresourcemanager.googleapis.com/Folder',
'//cloudresourcemanager.googleapis.com/organizations/1234567890',
self.session)
expected_names = ['folders/11111']
self.assertEqual(expected_names, [asset['name'] for asset in results])
results = CaiDataAccess.iter_cai_assets(
ContentTypes.resource, 'appengine.googleapis.com/Service',
'//appengine.googleapis.com/apps/forseti-test-project',
self.session)
expected_names = ['apps/forseti-test-project/services/default']
self.assertEqual(expected_names, [asset['name'] for asset in results])
def load_cloudasset_data(session, config):
"""Export asset data from Cloud Asset API and load into storage.
Args:
session (object): Database session.
config (object): Inventory configuration on server.
Returns:
int: The count of assets imported into the database, or None if there
is an error.
"""
# Start by ensuring that there is no existing CAI data in storage.
_clear_cai_data(session)
cloudasset_client = cloudasset.CloudAssetClient(
config.get_api_quota_configs())
imported_assets = 0
root_resources = []
if config.use_composite_root():
root_resources.extend(config.get_composite_root_resources())
else:
root_resources.append(config.get_root_resource_id())
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
futures = []
for root_id in root_resources:
for content_type in CONTENT_TYPES:
futures.append(executor.submit(_export_assets,
cloudasset_client,
config,
root_id,
content_type))
for future in concurrent.futures.as_completed(futures):
temporary_file = ''
try:
temporary_file = future.result()
if not temporary_file:
return _clear_cai_data(session)
LOGGER.debug('Importing Cloud Asset data from %s to database.',
temporary_file)
with open(temporary_file, 'r') as cai_data:
rows = CaiDataAccess.populate_cai_data(cai_data, session)
imported_assets += rows
LOGGER.info('%s assets imported to database.', rows)
finally:
if temporary_file:
os.unlink(temporary_file)
return imported_assets
def test_iter_cai_assets(self):
"""Validate querying CAI asset data."""
self._add_resources()
cai_type = 'cloudresourcemanager.googleapis.com/Folder'
results = CaiDataAccess.iter_cai_assets(
ContentTypes.resource, cai_type,
'//cloudresourcemanager.googleapis.com/organizations/1234567890',
self.session)
expected_results = [
('folders/11111',
AssetMetadata(
cai_type=cai_type,
cai_name='//cloudresourcemanager.googleapis.com/folders/11111'
))
]
self.assertEqual(expected_results, [(asset['name'], metadata)
for asset, metadata in results])
cai_type = 'appengine.googleapis.com/Service'
results = CaiDataAccess.iter_cai_assets(
ContentTypes.resource, cai_type,
'//appengine.googleapis.com/apps/forseti-test-project',
self.session)
expected_results = [(
'apps/forseti-test-project/services/default',
AssetMetadata(
cai_name=
'//appengine.googleapis.com/apps/forseti-test-project/services/default',
cai_type=cai_type))]
self.assertEqual(expected_results, [(asset['name'], metadata)
for asset, metadata in results])
def _add_iam_policies(self):
"""Add CAI IAM Policies to temporary table."""
iam_policy_data = StringIO(CAI_IAM_POLICY_DATA)
rows = CaiDataAccess.populate_cai_data(iam_policy_data, self.session)
expected_rows = len(CAI_IAM_POLICY_DATA.split('\n'))
self.assertEqual(expected_rows, rows)
def _add_resources(self):
"""Add CAI resources to temporary table."""
resource_data = StringIO(CAI_RESOURCE_DATA)
rows = CaiDataAccess.populate_cai_data(resource_data, self.session)
expected_rows = len(CAI_RESOURCE_DATA.split('\n'))
self.assertEqual(expected_rows, rows)