def test_clear_cai_data(self): """Validate CAI data delete.""" self._add_resources() rows = CaiDataAccess.clear_cai_data(self.session) expected_rows = len(CAI_RESOURCE_DATA.split('\n')) self.assertEqual(expected_rows, rows) results = CaiDataAccess.iter_cai_assets( ContentTypes.resource, 'cloudresourcemanager.googleapis.com/Folder', '//cloudresourcemanager.googleapis.com/organizations/1234567890', self.session) self.assertEqual(0, len(list(results)))
def test_fetch_cai_asset(self): """Validate querying single CAI asset.""" self._add_iam_policies() cai_type = 'cloudresourcemanager.googleapis.com/Organization' cai_name = '//cloudresourcemanager.googleapis.com/organizations/1234567890' results = CaiDataAccess.fetch_cai_asset(ContentTypes.iam_policy, cai_type, cai_name, self.session) expected_iam_policy = { 'etag': 'BwVvLqcT+M4=', 'bindings': [{ 'role': 'roles/Owner', 'members': ['user:[email protected]'] }, { 'role': 'roles/Viewer', 'members': [('serviceAccount:forseti-server-gcp-d9fffac' '@forseti-test-project.iam.gserviceaccount.com'), 'user:[email protected]'] }] } self.assertEqual((expected_iam_policy, AssetMetadata(cai_type=cai_type, cai_name=cai_name)), results)
def _clear_cai_data(session): """Clear CAI data from storage. Args: session (object): Database session. """ LOGGER.debug('Deleting Cloud Asset data from database.') count = CaiDataAccess.clear_cai_data(session) LOGGER.debug('%s assets deleted from database.', count) return None
def test_iter_cai_assets(self): """Validate querying CAI asset data.""" self._add_resources() results = CaiDataAccess.iter_cai_assets( ContentTypes.resource, 'cloudresourcemanager.googleapis.com/Folder', '//cloudresourcemanager.googleapis.com/organizations/1234567890', self.session) expected_names = ['folders/11111'] self.assertEqual(expected_names, [asset['name'] for asset in results]) results = CaiDataAccess.iter_cai_assets( ContentTypes.resource, 'appengine.googleapis.com/Service', '//appengine.googleapis.com/apps/forseti-test-project', self.session) expected_names = ['apps/forseti-test-project/services/default'] self.assertEqual(expected_names, [asset['name'] for asset in results])
def load_cloudasset_data(session, config): """Export asset data from Cloud Asset API and load into storage. Args: session (object): Database session. config (object): Inventory configuration on server. Returns: int: The count of assets imported into the database, or None if there is an error. """ # Start by ensuring that there is no existing CAI data in storage. _clear_cai_data(session) cloudasset_client = cloudasset.CloudAssetClient( config.get_api_quota_configs()) imported_assets = 0 root_resources = [] if config.use_composite_root(): root_resources.extend(config.get_composite_root_resources()) else: root_resources.append(config.get_root_resource_id()) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: futures = [] for root_id in root_resources: for content_type in CONTENT_TYPES: futures.append(executor.submit(_export_assets, cloudasset_client, config, root_id, content_type)) for future in concurrent.futures.as_completed(futures): temporary_file = '' try: temporary_file = future.result() if not temporary_file: return _clear_cai_data(session) LOGGER.debug('Importing Cloud Asset data from %s to database.', temporary_file) with open(temporary_file, 'r') as cai_data: rows = CaiDataAccess.populate_cai_data(cai_data, session) imported_assets += rows LOGGER.info('%s assets imported to database.', rows) finally: if temporary_file: os.unlink(temporary_file) return imported_assets
def test_iter_cai_assets(self): """Validate querying CAI asset data.""" self._add_resources() cai_type = 'cloudresourcemanager.googleapis.com/Folder' results = CaiDataAccess.iter_cai_assets( ContentTypes.resource, cai_type, '//cloudresourcemanager.googleapis.com/organizations/1234567890', self.session) expected_results = [ ('folders/11111', AssetMetadata( cai_type=cai_type, cai_name='//cloudresourcemanager.googleapis.com/folders/11111' )) ] self.assertEqual(expected_results, [(asset['name'], metadata) for asset, metadata in results]) cai_type = 'appengine.googleapis.com/Service' results = CaiDataAccess.iter_cai_assets( ContentTypes.resource, cai_type, '//appengine.googleapis.com/apps/forseti-test-project', self.session) expected_results = [( 'apps/forseti-test-project/services/default', AssetMetadata( cai_name= '//appengine.googleapis.com/apps/forseti-test-project/services/default', cai_type=cai_type))] self.assertEqual(expected_results, [(asset['name'], metadata) for asset, metadata in results])
def _add_iam_policies(self): """Add CAI IAM Policies to temporary table.""" iam_policy_data = StringIO(CAI_IAM_POLICY_DATA) rows = CaiDataAccess.populate_cai_data(iam_policy_data, self.session) expected_rows = len(CAI_IAM_POLICY_DATA.split('\n')) self.assertEqual(expected_rows, rows)
def _add_resources(self): """Add CAI resources to temporary table.""" resource_data = StringIO(CAI_RESOURCE_DATA) rows = CaiDataAccess.populate_cai_data(resource_data, self.session) expected_rows = len(CAI_RESOURCE_DATA.split('\n')) self.assertEqual(expected_rows, rows)