def test_api_version(self): """Verify we retrieve the correct API version for a resource type""" s = Session() client = s.client('azure.mgmt.resource.ResourceManagementClient') resource = next(client.resources.list()) self.assertTrue(re.match('\\d{4}-\\d{2}-\\d{2}', s.resource_api_version(resource.id)) is not None)
def test_get_client_non_default_base_url(self): s = Session(cloud_endpoints=AZURE_CHINA_CLOUD) client = s.client('azure.mgmt.resource.ResourceManagementClient') self.assertEqual(AZURE_CHINA_CLOUD.endpoints.resource_manager, client._client._base_url) self.assertEqual(AZURE_CHINA_CLOUD.endpoints.management + ".default", client._client._config.credential_scopes[0])
def _enhance_policies(self, access_policies): if not access_policies: return access_policies if self.graph_client is None: s = Session(resource='https://graph.windows.net') self.graph_client = GraphRbacManagementClient( s.get_credentials(), s.get_tenant_id()) # Retrieve graph objects for all object_id object_ids = [p['objectId'] for p in access_policies] # GraphHelper.get_principal_dictionary returns empty AADObject if not found with graph # or if graph is not available. principal_dics = GraphHelper.get_principal_dictionary( self.graph_client, object_ids, True) for policy in access_policies: aad_object = principal_dics[policy['objectId']] if aad_object.object_id: policy['displayName'] = aad_object.display_name policy['aadType'] = aad_object.object_type policy['principalName'] = GraphHelper.get_principal_name( aad_object) return access_policies
def test_get_functions_auth_string_overrides(self): with patch('azure.common.credentials.ServicePrincipalCredentials.__init__', autospec=True, return_value=None): with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: 'ea42f556-5106-4743-99b0-c129bfa71a47', constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret', constants.ENV_FUNCTION_TENANT_ID: 'functiontenant', constants.ENV_FUNCTION_SUB_ID: '000000-5106-4743-99b0-c129bfa71a47', constants.ENV_FUNCTION_CLIENT_ID: 'functionclient', constants.ENV_FUNCTION_CLIENT_SECRET: 'functionsecret' }, clear=True): s = Session() auth = s.get_functions_auth_string('000000-5106-4743-99b0-c129bfa71a47') expected = """{"client_id": "functionclient", "client_secret": "functionsecret", "tenant_id": "functiontenant", "subscription_id": "000000-5106-4743-99b0-c129bfa71a47" }""" self.assertEqual(json.loads(auth), json.loads(expected))
def augment(self, resources): s = Session(resource='https://graph.windows.net') graph_client = GraphRbacManagementClient(s.get_credentials(), s.get_tenant_id()) object_ids = list(set( resource['properties']['principalId'] for resource in resources if resource['properties']['principalId'])) object_params = GetObjectsParameters( include_directory_object_references=True, object_ids=object_ids) aad_objects = graph_client.objects.get_objects_by_object_ids(object_params) try: principal_dics = {aad_object.object_id: aad_object for aad_object in aad_objects} for resource in resources: if resource['properties']['principalId'] in principal_dics.keys(): graph_resource = principal_dics[resource['properties']['principalId']] resource['principalName'] = self.get_principal_name(graph_resource) resource['displayName'] = graph_resource.display_name resource['aadType'] = graph_resource.object_type except CloudError: log.warning('Credentials not authorized for access to read from Microsoft Graph. \n ' 'Can not query on principalName, displayName, or aadType. \n' ) return resources
def test_auto_tag_add_creator_tag(self, utcnow_mock): """Adds CreatorEmail to a resource group """ p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.resourcegroup', 'filters': [ {'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'test_vm'} ], 'actions': [ {'type': 'auto-tag-user', 'tag': 'CreatorEmail'}, ], }) p.run() # verify CreatorEmail tag set s = Session() client = s.client('azure.mgmt.resource.ResourceManagementClient') rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0] self.assertTrue(re.match(self.EMAIL_REGEX, rg.tags['CreatorEmail']))
def setup_account(): # Find actual name of storage account provisioned in our test environment s = Session() client = s.client('azure.mgmt.storage.StorageManagementClient') accounts = list(client.storage_accounts.list()) matching_account = [a for a in accounts if a.name.startswith("cctstorage")] return matching_account[0]
def test_initialize_session_auth_file(self): s = Session(authorization_file=self.authorization_file) self.assertIs(type(s.get_credentials()._credential), ClientSecretCredential) self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID) self.assertEqual(s.get_tenant_id(), 'tenant')
def __init__(self, event_queue_id, event_queue_name, policy_storage, log_group=None, metrics=None, output_dir=None): logging.basicConfig(level=logging.INFO, format='%(message)s') log.info("Running Azure Cloud Custodian Self-Host") load_resources() self.session = local_session(Session) self.storage_session = self.session storage_subscription_id = ResourceIdParser.get_subscription_id( event_queue_id) if storage_subscription_id != self.session.subscription_id: self.storage_session = Session( subscription_id=storage_subscription_id) # Load configuration self.options = Host.build_options(output_dir, log_group, metrics) self.policy_storage_uri = policy_storage self.event_queue_name = event_queue_name self.event_queue_id = event_queue_id # Prepare storage bits self.policy_blob_client = None self.blob_cache = {} self.queue_storage_account = self.prepare_queue_storage( self.event_queue_id, self.event_queue_name) self.queue_service = None # Register event subscription self.update_event_subscription() # Policy cache and dictionary self.policy_cache = tempfile.mkdtemp() self.policies = {} # Configure scheduler self.scheduler = BlockingScheduler() logging.getLogger('apscheduler.executors.default').setLevel( logging.ERROR) # Schedule recurring policy updates self.scheduler.add_job(self.update_policies, 'interval', seconds=policy_update_seconds, id="update_policies", next_run_time=datetime.now()) # Schedule recurring queue polling self.scheduler.add_job(self.poll_queue, 'interval', seconds=queue_poll_seconds, id="poll_queue") self.scheduler.start()
def test_add_or_update_single_tag(self): """Verifies we can add a new tag to a VM and not modify an existing tag on that resource """ p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.vm', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'cctestvm' }], 'actions': [{ 'type': 'tag', 'tag': 'tag1', 'value': 'value1' }], }) p.run() # verify that the a new tag is added without modifying existing tags s = Session() client = s.client('azure.mgmt.compute.ComputeManagementClient') vm = client.virtual_machines.get('test_vm', 'cctestvm') self.assertEqual(vm.tags, {'tag1': 'value1', 'testtag': 'testvalue'})
def test_get_functions_auth_string(self): with patch( 'azure.common.credentials.ServicePrincipalCredentials.__init__', autospec=True, return_value=None): with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID, constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret' }, clear=True): s = Session() auth = s.get_functions_auth_string(CUSTOM_SUBSCRIPTION_ID) expected = { "credentials": { "client_id": "client", "secret": "secret", "tenant": "tenant" }, "subscription": CUSTOM_SUBSCRIPTION_ID } self.assertEqual(json.loads(auth), expected)
def test_tag_trim_removes_tags_for_space(self): """Verifies tag trim removes tags when the space value and number of tags on the resource are greater than the max tag value (15) """ # Add tags to trim p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.vm', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'cctestvm' }], 'actions': [{ 'type': 'tag', 'tags': { 'tag-to-trim1': 'value1', 'tag-to-trim2': 'value2' } }], }) p.run() # verify more than 1 tag on resource s = Session() client = s.client('azure.mgmt.compute.ComputeManagementClient') vm = client.virtual_machines.get('test_vm', 'cctestvm') self.assertTrue(len(vm.tags) > 1) p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.vm', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'cctestvm' }], 'actions': [{ 'type': 'tag-trim', 'space': 14, 'preserve': ['testtag'] }], }) p.run() # verify that tags were trimmed to # have 14 spaces and 1 preserved vm = client.virtual_machines.get('test_vm', 'cctestvm') self.assertEqual(len(vm.tags), 1)
def test_get_client_us_gov(self): """Verify we are setting the correct credential scope for us government""" s = Session(cloud_endpoints=AZURE_US_GOV_CLOUD) client = s.client('azure.mgmt.resource.ResourceManagementClient') self.assertEqual(AZURE_US_GOV_CLOUD.endpoints.resource_manager, client._client._base_url) self.assertEqual(AZURE_US_GOV_CLOUD.endpoints.management + ".default", client._client._config.credential_scopes[0])
def test_initialize_session_cli(self, mock_run): mock_run.return_value = \ f'{{"id":"{DEFAULT_SUBSCRIPTION_ID}", "tenantId":"{DEFAULT_TENANT_ID}"}}' with patch.dict(os.environ, {}, clear=True): s = Session() self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID) self.assertEqual(s.get_tenant_id(), DEFAULT_TENANT_ID)
def test_initialize_session_auth_file(self): with patch('azure.common.credentials.ServicePrincipalCredentials.__init__', autospec=True, return_value=None): s = Session(authorization_file=self.authorization_file) self.assertIs(type(s.get_credentials()), ServicePrincipalCredentials) self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID) self.assertEqual(s.get_tenant_id(), 'tenant')
class FunctionalFiltersTagsTest(BaseTest): rg_name = 'test_vm' vm_name = 'cctestvm' DAYS = 10 initial_tags = {} client = Session().client('azure.mgmt.compute.ComputeManagementClient') @classmethod def setUpClass(cls, *args, **kwargs): super(FunctionalFiltersTagsTest, cls).setUpClass(*args, **kwargs) cls.client = Session().client('azure.mgmt.compute.ComputeManagementClient') try: cls.initial_tags = tools.get_tags(cls.client, cls.rg_name, cls.vm_name) # Using some date in the past for marked-for-op to avoid patching utc_now tools.set_tags(cls.client, cls.rg_name, cls.vm_name, {'test_filters_tag': 'test_value', 'custodian_status': 'TTL: delete@2018-01-01'}) except Exception: # Can fail without real auth pass @classmethod def tearDownClass(cls, *args, **kwargs): super(FunctionalFiltersTagsTest, cls).tearDownClass(*args, **kwargs) try: tools.set_tags(cls.client, cls.rg_name, cls.vm_name, cls.initial_tags) except Exception: # Can fail without real auth pass @arm_template('vm.json') def test_tag(self): resources = self._run_policy([{'tag:test_filters_tag': 'test_value'}]) self.assertEqual(len(resources), 1) @arm_template('vm.json') def test_marked_for_op(self): resources = self._run_policy([{'type': 'marked-for-op', 'op': 'delete'}]) self.assertEqual(len(resources), 1) def _run_policy(self, filters): return self.load_policy({ 'name': 'test-tag', 'resource': 'azure.vm', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': self.vm_name }] + filters }).run()
def test_get_client_overrides(self, mock): # Reload the module to re-import patched function reload(sys.modules['c7n_azure.session']) s = Session() client = s.client('azure.mgmt.resource.ResourceManagementClient') self.assertFalse(client._client.config.retry_policy.policy.respect_retry_after_header) self.assertIsNotNone(client._client.orig_send) client._client.send() self.assertTrue(mock.called)
def test_get_blob_client_by_uri_china_cloud(self, mock_create): url = CHINA_STORAGE_ACCOUNT + "/testcontainer/extrafolder" blob_service, container_name, key_prefix = \ StorageUtilities.get_blob_client_by_uri(url, Session(cloud_endpoints=AZURE_CHINA_CLOUD)) self.assertIsNotNone(blob_service) self.assertEqual(container_name, "testcontainer") self.assertEqual(key_prefix, "extrafolder") self.assertTrue(CHINA_STORAGE_ENDPOINT in blob_service.primary_endpoint) self.assertTrue(mock_create.called_once())
def test_remove_tags(self): """Verifies we can delete multiple tags from a resource group without modifying existing tags. """ p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.resourcegroup', 'filters': [ {'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'test_vm'} ], 'actions': [ {'type': 'tag', 'tags': {'pre-existing-1': 'to-keep', 'pre-existing-2': 'to-keep', 'added-1': 'to-delete', 'added-2': 'to-delete'}}, ], }) p.run() # verify initial tag set s = Session() client = s.client('azure.mgmt.resource.ResourceManagementClient') rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0] start_tags = rg.tags self.assertTrue('pre-existing-1' in start_tags) self.assertTrue('pre-existing-2' in start_tags) self.assertTrue('added-1' in start_tags) self.assertTrue('added-2' in start_tags) p = self.load_policy({ 'name': 'test-azure-remove-tag', 'resource': 'azure.resourcegroup', 'filters': [ {'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'test_vm'} ], 'actions': [ {'type': 'untag', 'tags': ['added-1', 'added-2']} ], }) p.run() # verify tags removed and pre-existing tags not removed rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0] # NOQA end_tags = rg.tags self.assertTrue('pre-existing-1' in end_tags) self.assertTrue('pre-existing-2' in end_tags) self.assertTrue('added-1' not in end_tags) self.assertTrue('added-2' not in end_tags)
def test_initialize_session_token(self, _1): with patch.dict(os.environ, { constants.ENV_ACCESS_TOKEN: 'token', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID }, clear=True): s = Session() self.assertIs(type(s.get_credentials()), BasicTokenAuthentication) self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
def test_get_queue_client_by_uri_china_cloud(self, mock_create): url = CHINA_STORAGE_ACCOUNT + "/queuename" queue_service, queue_name =\ StorageUtilities.get_queue_client_by_uri(url, Session(cloud_endpoints=AZURE_CHINA_CLOUD)) self.assertIsNotNone(queue_service) self.assertEqual(queue_name, "queuename") self.assertTrue( CHINA_STORAGE_ENDPOINT in queue_service.primary_endpoint) self.assertTrue(mock_create.called_once())
def test_get_function_target_subscription(self): with patch('azure.common.credentials.ServicePrincipalCredentials.__init__', autospec=True, return_value=None): with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID, constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret' }, clear=True): s = Session() self.assertEqual(s.get_function_target_subscription_name(), DEFAULT_SUBSCRIPTION_ID) self.assertEqual(s.get_function_target_subscription_ids(), [DEFAULT_SUBSCRIPTION_ID]) with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID, constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret', constants.ENV_FUNCTION_SUB_ID: CUSTOM_SUBSCRIPTION_ID }, clear=True): s = Session() self.assertEqual(s.get_function_target_subscription_name(), CUSTOM_SUBSCRIPTION_ID) self.assertEqual(s.get_function_target_subscription_ids(), [CUSTOM_SUBSCRIPTION_ID]) with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID, constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret', constants.ENV_FUNCTION_MANAGEMENT_GROUP_NAME: 'test' }, clear=True): with patch('c7n_azure.utils.ManagedGroupHelper.get_subscriptions_list', return_value=[]): s = Session() self.assertEqual(s.get_function_target_subscription_name(), 'test') self.assertEqual(s.get_function_target_subscription_ids(), [])
def setUpClass(cls, *args, **kwargs): super(FunctionalActionsTagsTest, cls).setUpClass(*args, **kwargs) cls.client = Session().client('azure.mgmt.compute.ComputeManagementClient') try: cls.initial_tags = tools.get_tags(cls.client, cls.rg_name, cls.vm_name) tools.set_tags(cls.client, cls.rg_name, cls.vm_name, {}) except Exception: # Can fail without real auth pass
def test_initialize_msi_auth_system(self): with patch.dict(os.environ, { constants.ENV_USE_MSI: 'true', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID }, clear=True): s = Session() self.assertIsInstance(s.get_credentials()._credential, ManagedIdentityCredential) self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
def test_initialize_session_auth_file_no_sub(self): s = Session(subscription_id=CUSTOM_SUBSCRIPTION_ID, authorization_file=self.authorization_file_no_sub) self.assertIs(type(s.get_credentials()._credential), ClientSecretCredential) self.assertEqual(s.get_subscription_id(), CUSTOM_SUBSCRIPTION_ID) # will vary between recorded/live auth options but useful to ensure # we ended up with one of the valid values self.assertTrue(s.get_tenant_id() in [DEFAULT_TENANT_ID, 'tenant'])
def test_auto_tag_update_false_noop_for_existing_tag(self, utcnow_mock): """Adds CreatorEmail to a resource group """ # setup by adding an existing CreatorEmail tag p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.resourcegroup', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'test_vm' }], 'actions': [ { 'type': 'tag', 'tag': 'CreatorEmail', 'value': 'do-not-modify' }, ], }) p.run() p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.resourcegroup', 'filters': [{ 'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'test_vm' }], 'actions': [{ 'type': 'auto-tag-user', 'tag': 'CreatorEmail', 'update': False, 'days': 10 }], }) p.run() # verify CreatorEmail tag was not modified s = Session() client = s.client('azure.mgmt.resource.ResourceManagementClient') rg = [ rg for rg in client.resource_groups.list() if rg.name == 'test_vm' ][0] self.assertEqual(rg.tags['CreatorEmail'], 'do-not-modify')
def test_tag_trim_space_0_removes_all_tags_but_preserve(self): """Verifies tag trim removes all other tags but tags listed in preserve """ # Add tags to trim p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.vm', 'filters': [ {'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'cctestvm'} ], 'actions': [ {'type': 'tag', 'tags': {'tag-to-trim1': 'value1', 'tag-to-trim2': 'value2', 'tag-to-trim3': 'value3'}} ], }) p.run() # verify initial tags contain more than testtag s = Session() client = s.client('azure.mgmt.compute.ComputeManagementClient') vm = client.virtual_machines.get('test_vm', 'cctestvm') self.assertTrue('tag-to-trim1' in vm.tags) self.assertTrue('tag-to-trim2' in vm.tags) self.assertTrue('tag-to-trim3' in vm.tags) self.assertTrue('testtag' in vm.tags) p = self.load_policy({ 'name': 'test-azure-tag', 'resource': 'azure.vm', 'filters': [ {'type': 'value', 'key': 'name', 'op': 'eq', 'value_type': 'normalize', 'value': 'cctestvm'} ], 'actions': [ {'type': 'tag-trim', 'space': 0, 'preserve': ['testtag'] } ], }) p.run() # verify all tags trimmed but testtag vm = client.virtual_machines.get('test_vm', 'cctestvm') self.assertEqual(vm.tags, {'testtag': 'testvalue'})
def test_compare_auth_params(self, _1, _2): reload(sys.modules['c7n_azure.session']) with patch.dict(os.environ, { constants.ENV_TENANT_ID: 'tenant', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID, constants.ENV_CLIENT_ID: 'client', constants.ENV_CLIENT_SECRET: 'secret', constants.ENV_USE_MSI: 'true', constants.ENV_ACCESS_TOKEN: 'access_token', constants.ENV_KEYVAULT_CLIENT_ID: 'kv_client', constants.ENV_KEYVAULT_SECRET_ID: 'kv_secret' }, clear=True): env_params = Session().auth_params file_params = Session(authorization_file=self.authorization_file_full).auth_params self.assertTrue(env_params.pop('enable_cli_auth')) self.assertFalse(file_params.pop('enable_cli_auth', None)) self.assertEqual(env_params, file_params)
def __init__(self, config, logger, session=None, max_num_processes=16): if StorageUtilities is None: raise Exception("Using Azure queue requires package c7n_azure to be installed.") self.max_num_processes = max_num_processes self.config = config self.logger = logger self.receive_queue = self.config['queue_url'] self.batch_size = 16 self.max_message_retry = 3 self.session = session or Session()
def test_initialize_session_token(self): with patch.dict(os.environ, { constants.ENV_ACCESS_TOKEN: 'token', constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID }, clear=True): s = Session() self.assertIsNone(s.get_credentials()._credential) self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID) self.assertEqual(s.get_credentials().get_token(), AccessToken('token', 0))