def test_api_version(self):
"""Verify we retrieve the correct API version for a resource type"""
s = Session()
client = s.client('azure.mgmt.resource.ResourceManagementClient')
resource = next(client.resources.list())
self.assertTrue(re.match('\\d{4}-\\d{2}-\\d{2}',
s.resource_api_version(resource.id)) is not None)
def test_get_client_non_default_base_url(self):
s = Session(cloud_endpoints=AZURE_CHINA_CLOUD)
client = s.client('azure.mgmt.resource.ResourceManagementClient')
self.assertEqual(AZURE_CHINA_CLOUD.endpoints.resource_manager,
client._client._base_url)
self.assertEqual(AZURE_CHINA_CLOUD.endpoints.management + ".default",
client._client._config.credential_scopes[0])
def _enhance_policies(self, access_policies):
if not access_policies:
return access_policies
if self.graph_client is None:
s = Session(resource='https://graph.windows.net')
self.graph_client = GraphRbacManagementClient(
s.get_credentials(), s.get_tenant_id())
# Retrieve graph objects for all object_id
object_ids = [p['objectId'] for p in access_policies]
# GraphHelper.get_principal_dictionary returns empty AADObject if not found with graph
# or if graph is not available.
principal_dics = GraphHelper.get_principal_dictionary(
self.graph_client, object_ids, True)
for policy in access_policies:
aad_object = principal_dics[policy['objectId']]
if aad_object.object_id:
policy['displayName'] = aad_object.display_name
policy['aadType'] = aad_object.object_type
policy['principalName'] = GraphHelper.get_principal_name(
aad_object)
return access_policies
def test_get_functions_auth_string_overrides(self):
with patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
autospec=True, return_value=None):
with patch.dict(os.environ,
{
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: 'ea42f556-5106-4743-99b0-c129bfa71a47',
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret',
constants.ENV_FUNCTION_TENANT_ID: 'functiontenant',
constants.ENV_FUNCTION_SUB_ID: '000000-5106-4743-99b0-c129bfa71a47',
constants.ENV_FUNCTION_CLIENT_ID: 'functionclient',
constants.ENV_FUNCTION_CLIENT_SECRET: 'functionsecret'
}, clear=True):
s = Session()
auth = s.get_functions_auth_string('000000-5106-4743-99b0-c129bfa71a47')
expected = """{"client_id": "functionclient",
"client_secret": "functionsecret",
"tenant_id": "functiontenant",
"subscription_id": "000000-5106-4743-99b0-c129bfa71a47"
}"""
self.assertEqual(json.loads(auth), json.loads(expected))
def augment(self, resources):
s = Session(resource='https://graph.windows.net')
graph_client = GraphRbacManagementClient(s.get_credentials(), s.get_tenant_id())
object_ids = list(set(
resource['properties']['principalId'] for resource in resources
if resource['properties']['principalId']))
object_params = GetObjectsParameters(
include_directory_object_references=True,
object_ids=object_ids)
aad_objects = graph_client.objects.get_objects_by_object_ids(object_params)
try:
principal_dics = {aad_object.object_id: aad_object for aad_object in aad_objects}
for resource in resources:
if resource['properties']['principalId'] in principal_dics.keys():
graph_resource = principal_dics[resource['properties']['principalId']]
resource['principalName'] = self.get_principal_name(graph_resource)
resource['displayName'] = graph_resource.display_name
resource['aadType'] = graph_resource.object_type
except CloudError:
log.warning('Credentials not authorized for access to read from Microsoft Graph. \n '
'Can not query on principalName, displayName, or aadType. \n'
)
return resources
def test_auto_tag_add_creator_tag(self, utcnow_mock):
"""Adds CreatorEmail to a resource group
"""
p = self.load_policy({
'name': 'test-azure-tag',
'resource': 'azure.resourcegroup',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'test_vm'}
],
'actions': [
{'type': 'auto-tag-user',
'tag': 'CreatorEmail'},
],
})
p.run()
# verify CreatorEmail tag set
s = Session()
client = s.client('azure.mgmt.resource.ResourceManagementClient')
rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0]
self.assertTrue(re.match(self.EMAIL_REGEX, rg.tags['CreatorEmail']))
def setup_account():
# Find actual name of storage account provisioned in our test environment
s = Session()
client = s.client('azure.mgmt.storage.StorageManagementClient')
accounts = list(client.storage_accounts.list())
matching_account = [a for a in accounts if a.name.startswith("cctstorage")]
return matching_account[0]
def test_initialize_session_auth_file(self):
s = Session(authorization_file=self.authorization_file)
self.assertIs(type(s.get_credentials()._credential),
ClientSecretCredential)
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
self.assertEqual(s.get_tenant_id(), 'tenant')
def __init__(self,
event_queue_id,
event_queue_name,
policy_storage,
log_group=None,
metrics=None,
output_dir=None):
logging.basicConfig(level=logging.INFO, format='%(message)s')
log.info("Running Azure Cloud Custodian Self-Host")
load_resources()
self.session = local_session(Session)
self.storage_session = self.session
storage_subscription_id = ResourceIdParser.get_subscription_id(
event_queue_id)
if storage_subscription_id != self.session.subscription_id:
self.storage_session = Session(
subscription_id=storage_subscription_id)
# Load configuration
self.options = Host.build_options(output_dir, log_group, metrics)
self.policy_storage_uri = policy_storage
self.event_queue_name = event_queue_name
self.event_queue_id = event_queue_id
# Prepare storage bits
self.policy_blob_client = None
self.blob_cache = {}
self.queue_storage_account = self.prepare_queue_storage(
self.event_queue_id, self.event_queue_name)
self.queue_service = None
# Register event subscription
self.update_event_subscription()
# Policy cache and dictionary
self.policy_cache = tempfile.mkdtemp()
self.policies = {}
# Configure scheduler
self.scheduler = BlockingScheduler()
logging.getLogger('apscheduler.executors.default').setLevel(
logging.ERROR)
# Schedule recurring policy updates
self.scheduler.add_job(self.update_policies,
'interval',
seconds=policy_update_seconds,
id="update_policies",
next_run_time=datetime.now())
# Schedule recurring queue polling
self.scheduler.add_job(self.poll_queue,
'interval',
seconds=queue_poll_seconds,
id="poll_queue")
self.scheduler.start()
def test_add_or_update_single_tag(self):
"""Verifies we can add a new tag to a VM and not modify
an existing tag on that resource
"""
p = self.load_policy({
'name':
'test-azure-tag',
'resource':
'azure.vm',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'cctestvm'
}],
'actions': [{
'type': 'tag',
'tag': 'tag1',
'value': 'value1'
}],
})
p.run()
# verify that the a new tag is added without modifying existing tags
s = Session()
client = s.client('azure.mgmt.compute.ComputeManagementClient')
vm = client.virtual_machines.get('test_vm', 'cctestvm')
self.assertEqual(vm.tags, {'tag1': 'value1', 'testtag': 'testvalue'})
def test_get_functions_auth_string(self):
with patch(
'azure.common.credentials.ServicePrincipalCredentials.__init__',
autospec=True,
return_value=None):
with patch.dict(os.environ, {
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret'
},
clear=True):
s = Session()
auth = s.get_functions_auth_string(CUSTOM_SUBSCRIPTION_ID)
expected = {
"credentials": {
"client_id": "client",
"secret": "secret",
"tenant": "tenant"
},
"subscription": CUSTOM_SUBSCRIPTION_ID
}
self.assertEqual(json.loads(auth), expected)
def test_tag_trim_removes_tags_for_space(self):
"""Verifies tag trim removes tags when the space value
and number of tags on the resource are greater than the max
tag value (15)
"""
# Add tags to trim
p = self.load_policy({
'name':
'test-azure-tag',
'resource':
'azure.vm',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'cctestvm'
}],
'actions': [{
'type': 'tag',
'tags': {
'tag-to-trim1': 'value1',
'tag-to-trim2': 'value2'
}
}],
})
p.run()
# verify more than 1 tag on resource
s = Session()
client = s.client('azure.mgmt.compute.ComputeManagementClient')
vm = client.virtual_machines.get('test_vm', 'cctestvm')
self.assertTrue(len(vm.tags) > 1)
p = self.load_policy({
'name':
'test-azure-tag',
'resource':
'azure.vm',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'cctestvm'
}],
'actions': [{
'type': 'tag-trim',
'space': 14,
'preserve': ['testtag']
}],
})
p.run()
# verify that tags were trimmed to
# have 14 spaces and 1 preserved
vm = client.virtual_machines.get('test_vm', 'cctestvm')
self.assertEqual(len(vm.tags), 1)
def test_get_client_us_gov(self):
"""Verify we are setting the correct credential scope for us government"""
s = Session(cloud_endpoints=AZURE_US_GOV_CLOUD)
client = s.client('azure.mgmt.resource.ResourceManagementClient')
self.assertEqual(AZURE_US_GOV_CLOUD.endpoints.resource_manager,
client._client._base_url)
self.assertEqual(AZURE_US_GOV_CLOUD.endpoints.management + ".default",
client._client._config.credential_scopes[0])
def test_initialize_session_cli(self, mock_run):
mock_run.return_value = \
f'{{"id":"{DEFAULT_SUBSCRIPTION_ID}", "tenantId":"{DEFAULT_TENANT_ID}"}}'
with patch.dict(os.environ, {}, clear=True):
s = Session()
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
self.assertEqual(s.get_tenant_id(), DEFAULT_TENANT_ID)
def test_initialize_session_auth_file(self):
with patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
autospec=True, return_value=None):
s = Session(authorization_file=self.authorization_file)
self.assertIs(type(s.get_credentials()), ServicePrincipalCredentials)
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
self.assertEqual(s.get_tenant_id(), 'tenant')
class FunctionalFiltersTagsTest(BaseTest):
rg_name = 'test_vm'
vm_name = 'cctestvm'
DAYS = 10
initial_tags = {}
client = Session().client('azure.mgmt.compute.ComputeManagementClient')
@classmethod
def setUpClass(cls, *args, **kwargs):
super(FunctionalFiltersTagsTest, cls).setUpClass(*args, **kwargs)
cls.client = Session().client('azure.mgmt.compute.ComputeManagementClient')
try:
cls.initial_tags = tools.get_tags(cls.client, cls.rg_name, cls.vm_name)
# Using some date in the past for marked-for-op to avoid patching utc_now
tools.set_tags(cls.client, cls.rg_name, cls.vm_name,
{'test_filters_tag': 'test_value',
'custodian_status': 'TTL: delete@2018-01-01'})
except Exception:
# Can fail without real auth
pass
@classmethod
def tearDownClass(cls, *args, **kwargs):
super(FunctionalFiltersTagsTest, cls).tearDownClass(*args, **kwargs)
try:
tools.set_tags(cls.client, cls.rg_name, cls.vm_name, cls.initial_tags)
except Exception:
# Can fail without real auth
pass
@arm_template('vm.json')
def test_tag(self):
resources = self._run_policy([{'tag:test_filters_tag': 'test_value'}])
self.assertEqual(len(resources), 1)
@arm_template('vm.json')
def test_marked_for_op(self):
resources = self._run_policy([{'type': 'marked-for-op', 'op': 'delete'}])
self.assertEqual(len(resources), 1)
def _run_policy(self, filters):
return self.load_policy({
'name': 'test-tag',
'resource': 'azure.vm',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': self.vm_name
}] + filters
}).run()
def test_get_client_overrides(self, mock):
# Reload the module to re-import patched function
reload(sys.modules['c7n_azure.session'])
s = Session()
client = s.client('azure.mgmt.resource.ResourceManagementClient')
self.assertFalse(client._client.config.retry_policy.policy.respect_retry_after_header)
self.assertIsNotNone(client._client.orig_send)
client._client.send()
self.assertTrue(mock.called)
def test_get_blob_client_by_uri_china_cloud(self, mock_create):
url = CHINA_STORAGE_ACCOUNT + "/testcontainer/extrafolder"
blob_service, container_name, key_prefix = \
StorageUtilities.get_blob_client_by_uri(url, Session(cloud_endpoints=AZURE_CHINA_CLOUD))
self.assertIsNotNone(blob_service)
self.assertEqual(container_name, "testcontainer")
self.assertEqual(key_prefix, "extrafolder")
self.assertTrue(CHINA_STORAGE_ENDPOINT in blob_service.primary_endpoint)
self.assertTrue(mock_create.called_once())
def test_remove_tags(self):
"""Verifies we can delete multiple tags from a resource
group without modifying existing tags.
"""
p = self.load_policy({
'name': 'test-azure-tag',
'resource': 'azure.resourcegroup',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'test_vm'}
],
'actions': [
{'type': 'tag',
'tags': {'pre-existing-1': 'to-keep', 'pre-existing-2': 'to-keep',
'added-1': 'to-delete', 'added-2': 'to-delete'}},
],
})
p.run()
# verify initial tag set
s = Session()
client = s.client('azure.mgmt.resource.ResourceManagementClient')
rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0]
start_tags = rg.tags
self.assertTrue('pre-existing-1' in start_tags)
self.assertTrue('pre-existing-2' in start_tags)
self.assertTrue('added-1' in start_tags)
self.assertTrue('added-2' in start_tags)
p = self.load_policy({
'name': 'test-azure-remove-tag',
'resource': 'azure.resourcegroup',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'test_vm'}
],
'actions': [
{'type': 'untag',
'tags': ['added-1', 'added-2']}
],
})
p.run()
# verify tags removed and pre-existing tags not removed
rg = [rg for rg in client.resource_groups.list() if rg.name == 'test_vm'][0] # NOQA
end_tags = rg.tags
self.assertTrue('pre-existing-1' in end_tags)
self.assertTrue('pre-existing-2' in end_tags)
self.assertTrue('added-1' not in end_tags)
self.assertTrue('added-2' not in end_tags)
def test_initialize_session_token(self, _1):
with patch.dict(os.environ, {
constants.ENV_ACCESS_TOKEN: 'token',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID
},
clear=True):
s = Session()
self.assertIs(type(s.get_credentials()), BasicTokenAuthentication)
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
def test_get_queue_client_by_uri_china_cloud(self, mock_create):
url = CHINA_STORAGE_ACCOUNT + "/queuename"
queue_service, queue_name =\
StorageUtilities.get_queue_client_by_uri(url,
Session(cloud_endpoints=AZURE_CHINA_CLOUD))
self.assertIsNotNone(queue_service)
self.assertEqual(queue_name, "queuename")
self.assertTrue(
CHINA_STORAGE_ENDPOINT in queue_service.primary_endpoint)
self.assertTrue(mock_create.called_once())
def test_get_function_target_subscription(self):
with patch('azure.common.credentials.ServicePrincipalCredentials.__init__',
autospec=True, return_value=None):
with patch.dict(os.environ,
{
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret'
}, clear=True):
s = Session()
self.assertEqual(s.get_function_target_subscription_name(),
DEFAULT_SUBSCRIPTION_ID)
self.assertEqual(s.get_function_target_subscription_ids(),
[DEFAULT_SUBSCRIPTION_ID])
with patch.dict(os.environ,
{
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret',
constants.ENV_FUNCTION_SUB_ID: CUSTOM_SUBSCRIPTION_ID
}, clear=True):
s = Session()
self.assertEqual(s.get_function_target_subscription_name(),
CUSTOM_SUBSCRIPTION_ID)
self.assertEqual(s.get_function_target_subscription_ids(),
[CUSTOM_SUBSCRIPTION_ID])
with patch.dict(os.environ,
{
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret',
constants.ENV_FUNCTION_MANAGEMENT_GROUP_NAME: 'test'
}, clear=True):
with patch('c7n_azure.utils.ManagedGroupHelper.get_subscriptions_list',
return_value=[]):
s = Session()
self.assertEqual(s.get_function_target_subscription_name(), 'test')
self.assertEqual(s.get_function_target_subscription_ids(), [])
def setUpClass(cls, *args, **kwargs):
super(FunctionalActionsTagsTest, cls).setUpClass(*args, **kwargs)
cls.client = Session().client('azure.mgmt.compute.ComputeManagementClient')
try:
cls.initial_tags = tools.get_tags(cls.client, cls.rg_name, cls.vm_name)
tools.set_tags(cls.client, cls.rg_name, cls.vm_name, {})
except Exception:
# Can fail without real auth
pass
def test_initialize_msi_auth_system(self):
with patch.dict(os.environ,
{
constants.ENV_USE_MSI: 'true',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID
}, clear=True):
s = Session()
self.assertIsInstance(s.get_credentials()._credential, ManagedIdentityCredential)
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
def test_initialize_session_auth_file_no_sub(self):
s = Session(subscription_id=CUSTOM_SUBSCRIPTION_ID,
authorization_file=self.authorization_file_no_sub)
self.assertIs(type(s.get_credentials()._credential), ClientSecretCredential)
self.assertEqual(s.get_subscription_id(), CUSTOM_SUBSCRIPTION_ID)
# will vary between recorded/live auth options but useful to ensure
# we ended up with one of the valid values
self.assertTrue(s.get_tenant_id() in [DEFAULT_TENANT_ID, 'tenant'])
def test_auto_tag_update_false_noop_for_existing_tag(self, utcnow_mock):
"""Adds CreatorEmail to a resource group
"""
# setup by adding an existing CreatorEmail tag
p = self.load_policy({
'name':
'test-azure-tag',
'resource':
'azure.resourcegroup',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'test_vm'
}],
'actions': [
{
'type': 'tag',
'tag': 'CreatorEmail',
'value': 'do-not-modify'
},
],
})
p.run()
p = self.load_policy({
'name':
'test-azure-tag',
'resource':
'azure.resourcegroup',
'filters': [{
'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'test_vm'
}],
'actions': [{
'type': 'auto-tag-user',
'tag': 'CreatorEmail',
'update': False,
'days': 10
}],
})
p.run()
# verify CreatorEmail tag was not modified
s = Session()
client = s.client('azure.mgmt.resource.ResourceManagementClient')
rg = [
rg for rg in client.resource_groups.list() if rg.name == 'test_vm'
][0]
self.assertEqual(rg.tags['CreatorEmail'], 'do-not-modify')
def test_tag_trim_space_0_removes_all_tags_but_preserve(self):
"""Verifies tag trim removes all other tags but tags
listed in preserve
"""
# Add tags to trim
p = self.load_policy({
'name': 'test-azure-tag',
'resource': 'azure.vm',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'cctestvm'}
],
'actions': [
{'type': 'tag',
'tags': {'tag-to-trim1': 'value1', 'tag-to-trim2': 'value2',
'tag-to-trim3': 'value3'}}
],
})
p.run()
# verify initial tags contain more than testtag
s = Session()
client = s.client('azure.mgmt.compute.ComputeManagementClient')
vm = client.virtual_machines.get('test_vm', 'cctestvm')
self.assertTrue('tag-to-trim1' in vm.tags)
self.assertTrue('tag-to-trim2' in vm.tags)
self.assertTrue('tag-to-trim3' in vm.tags)
self.assertTrue('testtag' in vm.tags)
p = self.load_policy({
'name': 'test-azure-tag',
'resource': 'azure.vm',
'filters': [
{'type': 'value',
'key': 'name',
'op': 'eq',
'value_type': 'normalize',
'value': 'cctestvm'}
],
'actions': [
{'type': 'tag-trim',
'space': 0,
'preserve': ['testtag']
}
],
})
p.run()
# verify all tags trimmed but testtag
vm = client.virtual_machines.get('test_vm', 'cctestvm')
self.assertEqual(vm.tags, {'testtag': 'testvalue'})
def test_compare_auth_params(self, _1, _2):
reload(sys.modules['c7n_azure.session'])
with patch.dict(os.environ,
{
constants.ENV_TENANT_ID: 'tenant',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
constants.ENV_CLIENT_ID: 'client',
constants.ENV_CLIENT_SECRET: 'secret',
constants.ENV_USE_MSI: 'true',
constants.ENV_ACCESS_TOKEN: 'access_token',
constants.ENV_KEYVAULT_CLIENT_ID: 'kv_client',
constants.ENV_KEYVAULT_SECRET_ID: 'kv_secret'
}, clear=True):
env_params = Session().auth_params
file_params = Session(authorization_file=self.authorization_file_full).auth_params
self.assertTrue(env_params.pop('enable_cli_auth'))
self.assertFalse(file_params.pop('enable_cli_auth', None))
self.assertEqual(env_params, file_params)
def __init__(self, config, logger, session=None, max_num_processes=16):
if StorageUtilities is None:
raise Exception("Using Azure queue requires package c7n_azure to be installed.")
self.max_num_processes = max_num_processes
self.config = config
self.logger = logger
self.receive_queue = self.config['queue_url']
self.batch_size = 16
self.max_message_retry = 3
self.session = session or Session()
def test_initialize_session_token(self):
with patch.dict(os.environ,
{
constants.ENV_ACCESS_TOKEN: 'token',
constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID
}, clear=True):
s = Session()
self.assertIsNone(s.get_credentials()._credential)
self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
self.assertEqual(s.get_credentials().get_token(), AccessToken('token', 0))
