def provision(self):
"""Provision any resources needed for the policy."""
parameters = self.get_parameters()
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
existing_webapp = self.template_util.resource_exist(
group_name, webapp_name)
if not existing_webapp:
self.template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
self.template_util.deploy_resource_template(
group_name, 'dedicated_functionapp.json', parameters).wait()
else:
self.log.info("Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
self.log.info("Building function package for %s" % webapp_name)
archive = FunctionPackage(self.policy.data)
archive.build()
if archive.status(webapp_name):
archive.publish(webapp_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def test_event_package_files(self, session_mock):
p = self.load_policy({
'name': 'test-azure-package',
'resource': 'azure.resourcegroup',
'mode': {
'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['VmWrite']
},
})
packer = FunctionPackage(p.data['name'])
packer._add_functions_required_files(p.data, 'test-queue')
files = packer.pkg._zip_file.filelist
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/function.py'))
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/__init__.py'))
self.assertTrue(
FunctionPackageTest._file_exists(
files, 'test-azure-package/function.json'))
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/config.json'))
self.assertTrue(FunctionPackageTest._file_exists(files, 'host.json'))
self.assertTrue(
FunctionPackageTest._file_exists(files, 'extensions.csproj'))
self.assertTrue(
FunctionPackageTest._file_exists(files, 'bin/extensions.dll'))
def test_package_build_cache(self, _1, rmtree_mock, add_files_mock):
cache_zip = os.path.join(test_files_folder, 'cache.zip')
self._create_patch(
'c7n_azure.dependency_manager.DependencyManager.check_cache',
return_value=True)
functions = [('prepare_non_binary_wheels', None),
('download_wheels', None), ('install_wheels', None),
('create_cache_metadata', None)]
mocks = []
for f in functions:
mocks.append(
self._create_patch(
'c7n_azure.dependency_manager.DependencyManager.' + f[0],
return_value=f[1]))
add_modules_mock = self._create_patch(
'c7n_azure.function_package.AzurePythonPackageArchive.add_modules')
self._create_patch(
'c7n_azure.function_package.AzurePythonPackageArchive.__init__')
packer = FunctionPackage('test')
packer.build({}, [], [], [], 'queue')
for m in mocks:
self.assertEqual(m.call_count, 0)
add_files_mock.assert_called_once()
self.assertEqual(rmtree_mock.call_count, 0)
self.assertEqual(add_modules_mock.call_count, 1)
self.assertFalse(os.path.exists(cache_zip))
def test_event_package_files(self, session_mock):
p = self.load_policy({
'name': 'test-azure-package',
'resource': 'azure.resourcegroup',
'mode': {
'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['ResourceGroupWrite']
},
})
packer = FunctionPackage(p.data['name'])
packer.pkg = AzurePythonPackageArchive()
packer._add_functions_required_files(p.data, 'c7n-azure==1.0',
'test-queue')
files = packer.pkg._zip_file.filelist
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/function.py'))
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/__init__.py'))
self.assertTrue(
FunctionPackageTest._file_exists(
files, 'test-azure-package/function.json'))
self.assertTrue(
FunctionPackageTest._file_exists(files,
'test-azure-package/config.json'))
self.assertTrue(FunctionPackageTest._file_exists(files, 'host.json'))
self.assertTrue(
FunctionPackageTest._file_exists(files, 'requirements.txt'))
def test_auth_file_user_assigned_identity(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode': {
'type': FUNCTION_EVENT_TRIGGER_MODE,
'provision-options': {
'identity': {
'type': 'SystemAssigned'
}
},
'events': ['PublicIpWrite']
}
})
packer = FunctionPackage(p.data['name'])
packer.pkg = AzurePythonPackageArchive()
packer._add_functions_required_files(p.data, 'c7n-azure==1.0',
'test-queue')
packer.pkg.close()
with zipfile.ZipFile(packer.pkg.path) as zf:
content = json.loads(zf.read('test-azure-public-ip/auth.json'))
self.assertEqual(content, {
'subscription_id': None,
'use_msi': True
})
def test_publish_functions_package_consumption(self, _1):
function_app_name = 'cloud-custodian-test-consumption%s' % self.subscription_id[
-12:]
parameters = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test'
},
storage_account={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': self.storage_name
},
service_plan={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test',
'sku_tier': 'dynamic'
},
function_app_resource_group_name=CONST_GROUP_NAME,
function_app_name=function_app_name)
package = FunctionPackage("TestPolicy")
package.pkg = AzurePythonPackageArchive()
package.close()
FunctionAppUtilities.publish_functions_package(parameters, package)
# verify app setting updated
wc = self.session.client('azure.mgmt.web.WebSiteManagementClient')
app_settings = wc.web_apps.list_application_settings(
CONST_GROUP_NAME, function_app_name)
self.assertIsNotNone(
app_settings.properties['WEBSITE_RUN_FROM_PACKAGE'])
def test_publish_functions_package_dedicated(self, _1):
parameters = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test'
},
storage_account={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': self.storage_name
},
service_plan={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test',
'sku_tier': 'Basic'
},
function_app={
'resource_group_name': CONST_GROUP_NAME,
'name': self.dedicated_function_name})
package = FunctionPackage("TestPolicy")
package.pkg = AzurePythonPackageArchive()
package.close()
FunctionAppUtilities.publish_functions_package(parameters, package)
def test_package_build_no_cache(self, rmtree_mock, add_files_mock):
functions = [('check_cache', False),
('prepare_non_binary_wheels', None),
('download_wheels', None), ('install_wheels', None),
('create_cache_metadata', None)]
mocks = []
for f in functions:
mocks.append(
self._create_patch(
'c7n_azure.dependency_manager.DependencyManager.' + f[0],
return_value=f[1]))
add_modules_mock = self._create_patch(
'c7n.mu.PythonPackageArchive.add_modules')
mocks.append(
self._create_patch('c7n.mu.PythonPackageArchive.add_file'))
cache_zip = os.path.join(test_files_folder, 'cache.zip')
self.addCleanup(os.remove, cache_zip)
packer = FunctionPackage('test', cache_override_path=test_files_folder)
packer.build({}, [], [], [], 'queue')
for m in mocks:
m.assert_called_once()
add_files_mock.assert_called_once()
self.assertEqual(rmtree_mock.call_count, 3)
self.assertEqual(add_modules_mock.call_count, 3)
self.assertTrue(os.path.exists(cache_zip))
def provision(self):
"""Provision any resources needed for the policy."""
existing_service_plan = self.client.app_service_plans.get(
self.group_name, self.parameters['servicePlanName']['value'])
if not existing_service_plan:
self.template_util.create_resource_group(
self.group_name, {'location': self.parameters['location']['value']})
self.template_util.deploy_resource_template(
self.group_name, 'dedicated_functionapp.json', self.parameters).wait()
else:
existing_webapp = self.client.web_apps.get(self.group_name, self.webapp_name)
if not existing_webapp:
functionapp_util = FunctionAppUtilities()
functionapp_util.deploy_webapp(self.webapp_name,
self.group_name, existing_service_plan,
self.parameters['storageName']['value'])
else:
self.log.info("Found existing App %s (%s) in group %s" %
(self.webapp_name, existing_webapp.location, self.group_name))
self.log.info("Building function package for %s" % self.webapp_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data)
archive.close()
self.log.info("Function package built, size is %dMB" % (archive.pkg.size / (1024 * 1024)))
if archive.wait_for_status(self.webapp_name):
archive.publish(self.webapp_name)
else:
self.log.error("Aborted deployment, ensure Application Service is healthy.")
def test_publish_functions_package_dedicated(
self, mock_function_package_publish):
parameters = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test'
},
storage_account={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': self.storage_name
},
service_plan={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test',
'sku_tier': 'Basic'
},
function_app_resource_group_name=CONST_GROUP_NAME,
function_app_name=self.dedicated_function_name)
FunctionAppUtilities.publish_functions_package(
parameters, FunctionPackage("TestPolicy"))
mock_function_package_publish.assert_called_once()
def _build_functions_package(self, queue_name):
package = FunctionPackage(self.policy_name, )
package.build(
self.policy.data,
modules=['c7n', 'c7n-azure'],
non_binary_packages=['pyyaml', 'pycparser', 'tabulate'],
excluded_packages=['azure-cli-core', 'distlib', 'futures'],
queue_name=queue_name)
package.close()
return package
def test_no_policy_add_required_files(self, session_mock):
""" Tools such as mailer will package with no policy """
packer = FunctionPackage('name')
packer.pkg = AzurePythonPackageArchive()
packer._add_functions_required_files(None)
files = packer.pkg._zip_file.filelist
self.assertTrue(FunctionPackageTest._file_exists(files, 'host.json'))
def def_cert_validation_on_by_default(self):
p = self.load_policy({
'name': 'test-azure-package',
'resource': 'azure.resourcegroup',
'mode':
{'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['VmWrite']},
})
packer = FunctionPackage(p.data['name'])
self.assertTrue(packer.enable_ssl_cert)
def build_functions_package(self, queue_name=None, target_subscription_ids=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
package = FunctionPackage(self.policy_name, target_sub_ids=target_subscription_ids)
package.build(self.policy.data,
modules=['c7n', 'c7n-azure'],
non_binary_packages=['pyyaml', 'pycparser', 'tabulate', 'pyrsistent'],
excluded_packages=['azure-cli-core', 'distlib', 'future', 'futures'],
queue_name=queue_name)
package.close()
self.log.info("Function package built, size is %dMB" % (package.pkg.size / (1024 * 1024)))
return package
def build_function_package(config, function_name, sub_id):
schedule = config.get('function_schedule', '0 */10 * * * *')
cache_override_path = cache_path()
function_path = function_name + "_" + sub_id
# Build package
package = FunctionPackage(function_name,
os.path.join(os.path.dirname(__file__),
'function.py'),
target_sub_ids=[sub_id],
cache_override_path=cache_override_path)
package.build(
None,
modules=['c7n', 'c7n-azure', 'c7n-mailer'],
non_binary_packages=[
'pyyaml', 'pycparser', 'tabulate', 'jmespath', 'datadog',
'MarkupSafe', 'simplejson', 'pyrsistent'
],
excluded_packages=['azure-cli-core', 'distlib', 'future', 'futures'])
package.pkg.add_contents(function_path + '/function.json',
contents=package.get_function_config({
'mode': {
'type': 'azure-periodic',
'schedule': schedule
}
}))
# Add mail templates
for d in set(config['templates_folders']):
if not os.path.exists(d):
continue
for t in [f for f in os.listdir(d) if os.path.splitext(f)[1] == '.j2']:
with open(os.path.join(d, t)) as fh:
package.pkg.add_contents(
function_path + '/msg-templates/%s' % t, fh.read())
function_config = copy.deepcopy(config)
functions_full_template_path = '/home/site/wwwroot/' + function_path + '/msg-templates/'
function_config['templates_folders'] = [functions_full_template_path]
package.pkg.add_contents(function_path + '/config.json',
contents=json.dumps(function_config))
package.close()
return package
def test_add_host_config(self):
packer = FunctionPackage('test')
packer.pkg = AzurePythonPackageArchive()
with patch('c7n_azure.function_package.AzurePythonPackageArchive.add_contents') as mock:
packer._add_host_config(FUNCTION_EVENT_TRIGGER_MODE)
mock.assert_called_once()
self.assertEqual(mock.call_args[1]['dest'], 'host.json')
self.assertTrue('extensionBundle' in json.loads(mock.call_args[1]['contents']))
with patch('c7n_azure.function_package.AzurePythonPackageArchive.add_contents') as mock:
packer._add_host_config(FUNCTION_TIME_TRIGGER_MODE)
mock.assert_called_once()
self.assertEqual(mock.call_args[1]['dest'], 'host.json')
self.assertFalse('extensionBundle' in json.loads(mock.call_args[1]['contents']))
def build_functions_package(self, queue_name=None, target_subscription_ids=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
requirements = generate_requirements('c7n-azure',
ignore=['boto3', 'botocore', 'pywin32'],
exclude='c7n')
package = FunctionPackage(self.policy_name, target_sub_ids=target_subscription_ids)
package.build(self.policy.data,
modules=['c7n', 'c7n-azure'],
requirements=requirements,
queue_name=queue_name)
package.close()
self.log.info("Function package built, size is %dKB" % (package.pkg.size / 1024))
return package
def test_env_var_disables_cert_validation(self):
p = self.load_policy({
'name': 'test-azure-package',
'resource': 'azure.resourcegroup',
'mode': {
'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['VmWrite']
},
})
with patch.dict(os.environ,
{ENV_CUSTODIAN_DISABLE_SSL_CERT_VERIFICATION: 'YES'},
clear=True):
packer = FunctionPackage(p.data['name'])
self.assertFalse(packer.enable_ssl_cert)
def test_add_function_config_events(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode': {
'type': 'azure-stream'
}
})
packer = FunctionPackage(p.data['name'])
config = packer.get_function_config(p.data)
binding = json.loads(config)
self.assertEqual(binding['bindings'][0]['type'], 'httpTrigger')
def _publish_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" %
self.function_app_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data, queue_name=queue_name)
archive.close()
self.log.info("Function package built, size is %dMB" %
(archive.pkg.size / (1024 * 1024)))
if archive.wait_for_status(self.function_app_name):
archive.publish(self.function_app_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def build_function_package(config, function_name, sub_id):
schedule = config.get('function_schedule', '0 */10 * * * *')
cache_override_path = cache_path()
function_path = function_name + "_" + sub_id
# Build package
package = FunctionPackage(function_name,
os.path.join(os.path.dirname(__file__),
'function.py'),
target_sub_ids=[sub_id],
cache_override_path=cache_override_path)
identity = jmespath.search('function_properties.identity', config)
package.build(None,
modules=['c7n', 'c7n_azure', 'c7n_mailer'],
requirements=get_mailer_requirements(),
identity=identity)
package.pkg.add_contents(function_path + '/function.json',
contents=package.get_function_config({
'mode': {
'type': 'azure-periodic',
'schedule': schedule
}
}))
# Add mail templates
for d in set(config['templates_folders']):
if not os.path.exists(d):
continue
for t in [f for f in os.listdir(d) if os.path.splitext(f)[1] == '.j2']:
with open(os.path.join(d, t)) as fh:
package.pkg.add_contents(
function_path + '/msg-templates/%s' % t, fh.read())
function_config = copy.deepcopy(config)
functions_full_template_path = '/home/site/wwwroot/' + function_path + '/msg-templates/'
function_config['templates_folders'] = [functions_full_template_path]
package.pkg.add_contents(function_path + '/config.json',
contents=json.dumps(function_config))
package.close()
return package
def test_add_function_config_events(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode': {
'type': CONST_AZURE_EVENT_TRIGGER_MODE,
'events': ['VmWrite']
},
})
packer = FunctionPackage(p.data['name'])
config = packer.get_function_config(p.data)
binding = json.loads(config)
self.assertEqual(binding['bindings'][0]['type'], 'httpTrigger')
def test_add_function_config_events(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode':
{'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['PublicIpWrite']},
})
packer = FunctionPackage(p.data['name'])
config = packer.get_function_config(p.data)
binding = json.loads(config)
self.assertEqual(binding['bindings'][0]['type'], 'queueTrigger')
self.assertEqual(binding['bindings'][0]['connection'], 'AzureWebJobsStorage')
def test_add_function_config_eventhub(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode': {
'type': 'azure-stream'
}
})
packer = FunctionPackage(p.data)
packer.pkg = mock.MagicMock()
packer._add_function_config()
binding = json.loads(packer.pkg.add_contents.call_args[1]['contents'])
self.assertEqual(binding['bindings'][0]['type'], 'eventHubTrigger')
def test_add_function_config_periodic(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode':
{'type': FUNCTION_TIME_TRIGGER_MODE,
'schedule': '0 1 0 1 1 1'}
})
packer = FunctionPackage(p.data['name'])
config = packer.get_function_config(p.data)
binding = json.loads(config)
self.assertEqual(binding['bindings'][0]['type'], 'timerTrigger')
self.assertEqual(binding['bindings'][0]['name'], 'input')
self.assertEqual(binding['bindings'][0]['schedule'], '0 1 0 1 1 1')
def test_publish(self, post_mock):
status_mock = MagicMock()
post_mock.return_value = status_mock
packer = FunctionPackage('test')
packer.pkg = AzurePythonPackageArchive()
creds = User(publishing_user_name='user',
publishing_password='******',
scm_uri='https://uri')
packer.publish(creds)
post_mock.assert_called_once()
status_mock.raise_for_status.assert_called_once()
self.assertEqual(post_mock.call_args[0][0],
'https://uri/api/zipdeploy?isAsync=true&synctriggers=true')
self.assertEqual(post_mock.call_args[1]['headers']['content-type'],
'application/octet-stream')
def test_add_policy(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode':
{'type': FUNCTION_EVENT_TRIGGER_MODE,
'events': ['PublicIpWrite']},
})
packer = FunctionPackage(p.data['name'])
policy = json.loads(packer._get_policy(p.data))
self.assertEqual(policy['policies'][0],
{u'resource': u'azure.publicip',
u'name': u'test-azure-public-ip',
u'mode': {u'type': u'azure-event-grid',
u'events': [u'PublicIpWrite']}})
def _publish_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data, queue_name=queue_name)
archive.close()
self.log.info("Function package built, size is %dMB" % (archive.pkg.size / (1024 * 1024)))
client = local_session(self.policy.session_factory)\
.client('azure.mgmt.web.WebSiteManagementClient')
publish_creds = client.web_apps.list_publishing_credentials(
self.function_params.function_app_resource_group_name,
self.function_params.function_app_name).result()
if archive.wait_for_status(publish_creds):
archive.publish(publish_creds)
else:
self.log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(self):
"""Provision any resources needed for the policy."""
template_util = TemplateUtilities()
parameters = self._get_parameters(template_util)
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
policy_name = self.policy.data['name'].replace(' ', '-').lower()
existing_service_plan = self.client.app_service_plans.get(
group_name, parameters['servicePlanName']['value'])
if not existing_service_plan:
template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
template_util.deploy_resource_template(
group_name, 'dedicated_functionapp.json', parameters).wait()
else:
existing_webapp = self.client.web_apps.get(group_name, webapp_name)
if not existing_webapp:
functionapp_util = FunctionAppUtilities()
functionapp_util.deploy_webapp(
webapp_name, group_name, existing_service_plan,
parameters['storageName']['value'])
else:
self.log.info(
"Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
self.log.info("Building function package for %s" % webapp_name)
archive = FunctionPackage(policy_name)
archive.build(self.policy.data)
archive.close()
if archive.wait_for_status(webapp_name):
archive.publish(webapp_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def test_add_function_config_periodic(self):
p = self.load_policy({
'name': 'test-azure-public-ip',
'resource': 'azure.publicip',
'mode': {
'type': 'azure-periodic',
'schedule': '0 1 0 0 0'
}
})
packer = FunctionPackage(p.data)
packer.pkg = mock.MagicMock()
packer._add_function_config()
binding = json.loads(packer.pkg.add_contents.call_args[1]['contents'])
self.assertEqual(binding['bindings'][0]['type'], 'timerTrigger')
self.assertEqual(binding['bindings'][0]['name'], 'input')
self.assertEqual(binding['bindings'][0]['schedule'], '0 1 0 0 0')
