def test_load_cloudasset_data_cai_valueerror(self): """Validate load_cloud_asset raises exception on bad root resource.""" inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0, { 'enabled': True, 'gcs_path': 'gs://test-bucket' }) self.mock_export_assets.side_effect = (ValueError( 'parent must start with folders/, projects/, or ' 'organizations/')) with self.assertRaises(ValueError): cloudasset.load_cloudasset_data(self.engine, inventory_config) self.assertFalse(self.mock_download.called) self.validate_no_data_in_table()
def test_load_cloudasset_data(self): """Validate load_cloudasset_data correctly dumps and imports data.""" # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} # Mock download to return correct test data file def _fake_download(self, full_bucket_path, output_file): """Fake copy_file_from_gcs.""" if 'resource' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_resources.dump') elif 'iam_policy' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_iam_policies.dump') elif 'org_policy' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_org_policies.dump') elif 'access_policy' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_access_policies.dump') with open(fake_file, 'rb') as f: output_file.write(f.read()) self.mock_download.side_effect = _fake_download results = cloudasset.load_cloudasset_data(self.engine, self.inventory_config, self.inventory_index_id) self.assertTrue(results) self.validate_data_in_table()
def _api_client_factory(storage, config, parallel): """Creates the proper initialized API client based on the configuration. Args: storage (object): Storage implementation to use. config (object): Inventory configuration on server. parallel (bool): If true, use the parallel crawler implementation. Returns: Union[gcp.ApiClientImpl, cai_gcp_client.CaiApiClientImpl]: The initialized api client implementation class. """ client_config = config.get_api_quota_configs() client_config['domain_super_admin_email'] = config.get_gsuite_admin_email() client_config['excluded_resources'] = config.get_excluded_resources() asset_count = 0 if config.get_cai_enabled(): # TODO: When CAI supports resource exclusion, update the following # method to handle resource exclusion during export time. asset_count = cloudasset.load_cloudasset_data(storage.session, config) LOGGER.info('%s total assets loaded from Cloud Asset data.', asset_count) if asset_count: engine = config.get_service_config().get_engine() return cai_gcp_client.CaiApiClientImpl(client_config, engine, parallel, storage.session) # Default to the non-CAI implementation return gcp.ApiClientImpl(client_config)
def _api_client_factory(config, threads): """Creates the proper initialized API client based on the configuration. Args: config (object): Inventory configuration on server. threads (int): how many threads to use. Returns: Union[gcp.ApiClientImpl, cai_gcp_client.CaiApiClientImpl]: The initialized api client implementation class. """ client_config = config.get_api_quota_configs() client_config['domain_super_admin_email'] = config.get_gsuite_admin_email() client_config['excluded_resources'] = config.get_excluded_resources() if config.get_cai_enabled(): # TODO: When CAI supports resource exclusion, update the following # method to handle resource exclusion during export time. engine, tmpfile = cai_temporary_storage.create_sqlite_db(threads) asset_count = cloudasset.load_cloudasset_data(engine, config) LOGGER.info('%s total assets loaded from Cloud Asset data.', asset_count) if asset_count: return cai_gcp_client.CaiApiClientImpl(client_config, engine, tmpfile) # Default to the non-CAI implementation return gcp.ApiClientImpl(client_config)
def test_load_cloudasset_data_composite_root(self): """Validate load_cloudasset_data correctly works with composite root.""" composite_root_resources = ['projects/1043', 'projects/1044'] inventory_config = InventoryConfig(None, '', {}, 0, { 'enabled': True, 'gcs_path': 'gs://test-bucket' }, composite_root_resources) # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} # Mock download to return correct test data file def _fake_download(self, full_bucket_path, output_file): """Fake copy_file_from_gcs.""" if 'resource' in full_bucket_path: if 'projects-1043' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_project3_resources.dump') if 'projects-1044' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_project4_resources.dump') elif 'iam_policy' in full_bucket_path: if 'projects-1043' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_project3_iam_policies.dump') if 'projects-1044' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_project4_iam_policies.dump') with open(fake_file, 'rb') as f: output_file.write(f.read()) self.mock_download.side_effect = _fake_download results = cloudasset.load_cloudasset_data(self.engine, inventory_config, self.inventory_index_id) expected_results = 12 # Total of resources and IAM policies in dumps. self.assertEqual(expected_results, results) # Validate data from both projects in database. for root_id in composite_root_resources: for content_type in [ cai_temporary_storage.ContentTypes.resource, cai_temporary_storage.ContentTypes.iam_policy ]: expected_resource_name = ( '//cloudresourcemanager.googleapis.com/%s' % root_id) resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset( content_type, 'cloudresourcemanager.googleapis.com/Project', expected_resource_name, self.engine) self.assertTrue(resource, msg=('Resource %s type %s is missing' % (root_id, content_type)))
def test_load_cloudasset_data_cai_error_response(self): """Validate load_cloud_asset handles an error result from CAI.""" self.mock_export_assets.return_value = EXPORT_ASSETS_ERROR results = cloudasset.load_cloudasset_data(self.engine, self.inventory_config) self.assertIsNone(results) self.assertFalse(self.mock_download.called) self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_timeout(self): """Validate load_cloud_asset handles a timeout error.""" self.mock_export_assets.side_effect = ( api_errors.OperationTimeoutError('organizations/987654321', {})) results = cloudasset.load_cloudasset_data(self.engine, self.inventory_config) self.assertIsNone(results) self.assertFalse(self.mock_download.called) self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_valueerror(self): """Validate load_cloud_asset handles a bad root resource id.""" inventory_config = InventoryConfig('bad_resource/987654321', '', {}, 0, { 'enabled': True, 'gcs_path': 'gs://test-bucket' }) self.mock_export_assets.side_effect = (ValueError( 'parent must start with folders/, projects/, or ' 'organizations/')) results = cloudasset.load_cloudasset_data(self.session, inventory_config) self.assertIsNone(results) self.assertFalse(self.mock_copy_file_from_gcs.called) self.validate_no_data_in_table()
def test_load_cloudasset_data_download_error(self): """Validate load_cloud_asset handles an error downloading from GCS.""" # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} response = httplib2.Response( {'status': '403', 'content-type': 'application/json'}) content = PERMISSION_DENIED.encode() error_403 = errors.HttpError(response, content) self.mock_copy_file_from_gcs.side_effect = error_403 results = cloudasset.load_cloudasset_data(self.session, self.inventory_config) self.assertIsNone(results) self.validate_no_data_in_table()
def test_load_cloudasset_data_cai_apierror(self): """Validate load_cloud_asset handles an API error from CAI.""" response = httplib2.Response( {'status': '403', 'content-type': 'application/json'}) content = PERMISSION_DENIED.encode() error_403 = errors.HttpError(response, content) self.mock_export_assets.side_effect = ( api_errors.ApiExecutionError('organizations/987654321', error_403) ) results = cloudasset.load_cloudasset_data(self.session, self.inventory_config) self.assertIsNone(results) self.assertFalse(self.mock_copy_file_from_gcs.called) self.validate_no_data_in_table()
def test_long_resource_name(self): """Validate load_cloudasset_data handles resources with long names.""" # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} # Mock download to return correct test data file def _fake_download(self, full_bucket_path, output_file): """Fake copy_file_from_gcs.""" if 'resource' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_long_resource_name.dump') elif 'iam_policy' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_iam_policies.dump') elif 'org_policy' in full_bucket_path: fake_file = os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_org_policies.dump') elif 'access_policy' in full_bucket_path: fake_file = os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_access_policies.dump') with open(fake_file, 'rb') as f: output_file.write(f.read()) self.mock_download.side_effect = _fake_download results = cloudasset.load_cloudasset_data(self.engine, self.inventory_config, self.inventory_index_id) # Expect both resources got imported. expected_results = 2 self.assertEqual(results, expected_results) cai_type = 'spanner.googleapis.com/Instance' cai_name = '//spanner.googleapis.com/projects/project2/instances/test123' # Validate resource with short name is in database. resource = cai_temporary_storage.CaiDataAccess.fetch_cai_asset( cai_temporary_storage.ContentTypes.resource, cai_type, cai_name, self.engine) expected_resource = ({ 'config': 'projects/project2/instanceConfigs/regional-us-east1', 'displayName': 'Test123', 'name': 'projects/project2/instances/test123', 'nodeCount': 1, 'state': 'READY' }, AssetMetadata(cai_type=cai_type, cai_name=cai_name)) self.assertEqual(expected_resource, resource)
def test_load_cloudasset_data(self): """Validate load_cloudasset_data correctly dumps and imports data.""" # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} # Mock copy_file_from_gcs to return correct test data file def _copy_file_from_gcs(file_path, *args, **kwargs): """Fake copy_file_from_gcs.""" if 'resource' in file_path: return os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_resources.dump') elif 'iam_policy' in file_path: return os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_iam_policies.dump') self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs results = cloudasset.load_cloudasset_data(self.session, self.inventory_config) self.assertTrue(results) self.validate_data_in_table()
def test_long_resource_name(self): """Validate load_cloudasset_data handles resources with long names.""" # Ignore call to export_assets for this test. self.mock_export_assets.return_value = {'done': True} # Mock copy_file_from_gcs to return correct test data file def _copy_file_from_gcs(file_path, *args, **kwargs): """Fake copy_file_from_gcs.""" if 'resource' in file_path: return os.path.join( TEST_RESOURCE_DIR_PATH, 'mock_cai_long_resource_name.dump') elif 'iam_policy' in file_path: return os.path.join(TEST_RESOURCE_DIR_PATH, 'mock_cai_empty_iam_policies.dump') self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs results = cloudasset.load_cloudasset_data(self.session, self.inventory_config) # Expect only the resource with the short name got imported. expected_results = 1 self.assertEqual(results, expected_results) cai_type = 'spanner.googleapis.com/Instance' cai_name = '//spanner.googleapis.com/projects/project2/instances/test123' # Validate resource with short name is in database. resource = storage.CaiDataAccess.fetch_cai_asset( storage.ContentTypes.resource, cai_type, cai_name, self.session) expected_resource = ({ 'config': 'projects/project2/instanceConfigs/regional-us-east1', 'displayName': 'Test123', 'name': 'projects/project2/instances/test123', 'nodeCount': 1, 'state': 'READY'}, AssetMetadata(cai_type=cai_type, cai_name=cai_name)) self.assertEqual(expected_resource, resource)