def test_publish_functions_package_consumption(self):
parameters = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test'
},
storage_account={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloudcustodiantest'
},
service_plan={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test',
'sku_tier': 'dynamic'
},
function_app_resource_group_name=CONST_GROUP_NAME,
function_app_name='cloud-custodian-test')
package = FunctionPackage("TestPolicy")
package.close()
FunctionAppUtilities.publish_functions_package(
parameters, package)
# verify app setting updated
wc = self.session.client('azure.mgmt.web.WebSiteManagementClient')
app_settings = wc.web_apps.list_application_settings(
CONST_GROUP_NAME, 'cloud-custodian-test')
self.assertIsNotNone(app_settings.properties['WEBSITE_RUN_FROM_PACKAGE'])
def build_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
package = FunctionPackage(self.policy_name)
package.build(self.policy.data,
modules=['c7n', 'c7n-azure', 'applicationinsights'],
non_binary_packages=['pyyaml', 'pycparser', 'tabulate'],
excluded_packages=['azure-cli-core', 'distlib', 'futures'],
queue_name=queue_name)
package.close()
self.log.info("Function package built, size is %dMB" % (package.pkg.size / (1024 * 1024)))
return package
def build_function_package(config, function_name, sub_id):
schedule = config.get('function_schedule', '0 */10 * * * *')
cache_override_path = cache_path()
function_path = function_name + "_" + sub_id
# Build package
package = FunctionPackage(
function_name,
os.path.join(os.path.dirname(__file__), 'function.py'),
target_sub_ids=[sub_id],
cache_override_path=cache_override_path)
identity = jmespath.search('function_properties.identity', config)
package.build(None,
modules=['c7n', 'c7n_azure', 'c7n_mailer'],
requirements=get_mailer_requirements(),
identity=identity)
package.pkg.add_contents(
function_path + '/function.json',
contents=package.get_function_config({'mode':
{'type': 'azure-periodic',
'schedule': schedule}}))
# Add mail templates
for d in set(config['templates_folders']):
if not os.path.exists(d):
continue
for t in [f for f in os.listdir(d) if os.path.splitext(f)[1] == '.j2']:
with open(os.path.join(d, t)) as fh:
package.pkg.add_contents(function_path + '/msg-templates/%s' % t, fh.read())
function_config = copy.deepcopy(config)
functions_full_template_path = '/home/site/wwwroot/' + function_path + '/msg-templates/'
function_config['templates_folders'] = [functions_full_template_path]
package.pkg.add_contents(
function_path + '/config.json',
contents=json.dumps(function_config))
package.close()
return package
def build_function_package(config, function_name):
schedule = config.get('function_schedule', '0 */10 * * * *')
cache_override_path = cache_path()
# Build package
package = FunctionPackage(function_name,
os.path.join(os.path.dirname(__file__),
'function.py'),
cache_override_path=cache_override_path)
package.build(
None,
modules=['c7n', 'c7n-azure', 'c7n-mailer'],
non_binary_packages=[
'pyyaml', 'pycparser', 'tabulate', 'jmespath', 'datadog',
'MarkupSafe', 'simplejson', 'pyrsistent'
],
excluded_packages=['azure-cli-core', 'distlib', 'future', 'futures'])
package.pkg.add_contents(function_name + '/function.json',
contents=package.get_function_config({
'mode': {
'type': 'azure-periodic',
'schedule': schedule
}
}))
# Add mail templates
for d in set(config['templates_folders']):
if not os.path.exists(d):
continue
for t in [f for f in os.listdir(d) if os.path.splitext(f)[1] == '.j2']:
with open(os.path.join(d, t)) as fh:
package.pkg.add_contents(
function_name + '/msg-templates/%s' % t, fh.read())
function_config = copy.deepcopy(config)
function_config['templates_folders'] = [function_name + '/msg-templates/']
package.pkg.add_contents(function_name + '/config.json',
contents=json.dumps(function_config))
package.close()
return package
def provision(self):
"""Provision any resources needed for the policy."""
session = local_session(self.policy.session_factory)
client = session.client('azure.mgmt.web.WebSiteManagementClient')
existing_service_plan = client.app_service_plans.get(
self.group_name, self.parameters['servicePlanName']['value'])
if not existing_service_plan:
self.template_util.create_resource_group(
self.group_name,
{'location': self.parameters['location']['value']})
self.template_util.deploy_resource_template(
self.group_name, 'dedicated_functionapp.json',
self.parameters).wait()
else:
existing_webapp = client.web_apps.get(self.group_name,
self.webapp_name)
if not existing_webapp:
functionapp_util = FunctionAppUtilities()
functionapp_util.deploy_webapp(
self.webapp_name, self.group_name, existing_service_plan,
self.parameters['storageName']['value'])
else:
self.log.info("Found existing App %s (%s) in group %s" %
(self.webapp_name, existing_webapp.location,
self.group_name))
self.log.info("Building function package for %s" % self.webapp_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data)
archive.close()
self.log.info("Function package built, size is %dMB" %
(archive.pkg.size / (1024 * 1024)))
if archive.wait_for_status(self.webapp_name):
archive.publish(self.webapp_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def _publish_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data, queue_name=queue_name)
archive.close()
self.log.info("Function package built, size is %dMB" % (archive.pkg.size / (1024 * 1024)))
client = local_session(self.policy.session_factory)\
.client('azure.mgmt.web.WebSiteManagementClient')
publish_creds = client.web_apps.list_publishing_credentials(
self.function_params.function_app_resource_group_name,
self.function_params.function_app_name).result()
if archive.wait_for_status(publish_creds):
archive.publish(publish_creds)
else:
self.log.error("Aborted deployment, ensure Application Service is healthy.")
def build_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" %
self.function_params.function_app_name)
package = FunctionPackage(self.policy_name)
package.build(self.policy.data,
modules=['c7n', 'c7n-azure', 'applicationinsights'],
non_binary_packages=[
'pyyaml', 'pycparser', 'tabulate', 'pyrsistent'
],
excluded_packages=[
'azure-cli-core', 'distlib', 'future', 'futures'
],
queue_name=queue_name)
package.close()
self.log.info("Function package built, size is %dMB" %
(package.pkg.size / (1024 * 1024)))
return package
def _publish_functions_package(self, queue_name=None):
self.log.info("Building function package for %s" % self.function_params.function_app_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data, queue_name=queue_name)
archive.close()
self.log.info("Function package built, size is %dMB" % (archive.pkg.size / (1024 * 1024)))
client = local_session(self.policy.session_factory)\
.client('azure.mgmt.web.WebSiteManagementClient')
publish_creds = client.web_apps.list_publishing_credentials(
self.function_params.function_app_resource_group_name,
self.function_params.function_app_name).result()
if archive.wait_for_status(publish_creds):
archive.publish(publish_creds)
else:
self.log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(self):
"""Provision any resources needed for the policy."""
template_util = TemplateUtilities()
parameters = self._get_parameters(template_util)
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
policy_name = self.policy.data['name'].replace(' ', '-').lower()
existing_service_plan = self.client.app_service_plans.get(
group_name, parameters['servicePlanName']['value'])
if not existing_service_plan:
template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
template_util.deploy_resource_template(
group_name, 'dedicated_functionapp.json', parameters).wait()
else:
existing_webapp = self.client.web_apps.get(group_name, webapp_name)
if not existing_webapp:
functionapp_util = FunctionAppUtilities()
functionapp_util.deploy_webapp(
webapp_name, group_name, existing_service_plan,
parameters['storageName']['value'])
else:
self.log.info(
"Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
self.log.info("Building function package for %s" % webapp_name)
archive = FunctionPackage(policy_name)
archive.build(self.policy.data)
archive.close()
if archive.wait_for_status(webapp_name):
archive.publish(webapp_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def build_functions_package(self,
queue_name=None,
target_subscription_ids=None):
self.log.info("Building function package for %s",
self.function_params.function_app['name'])
requirements = generate_requirements(
'c7n-azure',
ignore=['boto3', 'botocore', 'pywin32'],
exclude='c7n')
package = FunctionPackage(self.policy_name,
target_sub_ids=target_subscription_ids)
package.build(self.policy.data,
modules=['c7n', 'c7n-azure'],
requirements=requirements,
queue_name=queue_name)
package.close()
self.log.info("Function package built, size is %dKB" %
(package.pkg.size / 1024))
return package
def provision(self):
"""Provision any resources needed for the policy."""
template_util = TemplateUtilities()
parameters = self._get_parameters(template_util)
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
policy_name = self.policy.data['name'].replace(' ', '-').lower()
existing_service_plan = self.client.app_service_plans.get(
group_name, parameters['servicePlanName']['value'])
if not existing_service_plan:
template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
template_util.deploy_resource_template(
group_name, 'dedicated_functionapp.json', parameters).wait()
else:
existing_webapp = self.client.web_apps.get(group_name, webapp_name)
if not existing_webapp:
functionapp_util = FunctionAppUtilities()
functionapp_util.deploy_webapp(webapp_name, group_name, existing_service_plan,
parameters['storageName']['value'])
else:
self.log.info("Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
self.log.info("Building function package for %s" % webapp_name)
archive = FunctionPackage(policy_name)
archive.build(self.policy.data)
archive.close()
if archive.wait_for_status(webapp_name):
archive.publish(webapp_name)
else:
self.log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(self):
# If storage account name is not provided, we'll try to make it unique using
# resource group name & subscription id values.
# Can't be a part of constructor because local_session is not working with
# custodian validate.
if self.storage_account['name'] == self.default_storage_name:
rg_name = self.storage_account['resource_group_name']
sub_id = local_session(
self.policy.session_factory).get_subscription_id()
suffix = hashlib.sha256(bytes(rg_name + sub_id,
'utf-8')).hexdigest().lower()[:8]
self.storage_account['name'] = self.default_storage_name + suffix
params = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights=self.app_insights,
service_plan=self.service_plan,
storage_account=self.storage_account,
functionapp_name=self.functionapp_name)
FunctionAppUtilities().deploy_dedicated_function_app(params)
self.log.info("Building function package for %s" %
self.functionapp_name)
archive = FunctionPackage(self.policy_name)
archive.build(self.policy.data)
archive.close()
self.log.info("Function package built, size is %dMB" %
(archive.pkg.size / (1024 * 1024)))
if archive.wait_for_status(self.functionapp_name):
archive.publish(self.functionapp_name)
else:
self.log.error(
"Aborted deployment, ensure Application Service is healthy.")
def test_publish_functions_package_consumption(self, _1):
function_app_name = 'cloud-custodian-test-consumption%s' % self.subscription_id[
-12:]
parameters = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test'
},
storage_account={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': self.storage_name
},
service_plan={
'id': '',
'resource_group_name': CONST_GROUP_NAME,
'name': 'cloud-custodian-test',
'sku_tier': 'dynamic'
},
function_app={
'resource_group_name': CONST_GROUP_NAME,
'name': function_app_name
})
package = FunctionPackage("TestPolicy")
package.pkg = AzurePythonPackageArchive()
package.close()
FunctionAppUtilities.publish_functions_package(parameters, package)
# verify app setting updated
wc = self.session.client('azure.mgmt.web.WebSiteManagementClient')
app_settings = wc.web_apps.list_application_settings(
CONST_GROUP_NAME, function_app_name)
self.assertNotIn('WEBSITE_RUN_FROM_PACKAGE', app_settings.properties)
def build_function_package(config, function_name):
schedule = config.get('function_schedule', '0 */10 * * * *')
# Build package
package = FunctionPackage(
function_name,
os.path.join(os.path.dirname(__file__), 'function.py'))
package.build(None,
modules=['c7n', 'c7n-azure', 'c7n-mailer', 'applicationinsights'],
non_binary_packages=['pyyaml', 'pycparser', 'tabulate', 'jmespath',
'datadog', 'MarkupSafe', 'simplejson', 'pyrsistent'],
excluded_packages=['azure-cli-core', 'distlib', 'future', 'futures'])
package.pkg.add_contents(
function_name + '/function.json',
contents=package.get_function_config({'mode':
{'type': 'azure-periodic',
'schedule': schedule}}))
# Add mail templates
for d in set(config['templates_folders']):
if not os.path.exists(d):
continue
for t in [f for f in os.listdir(d) if os.path.splitext(f)[1] == '.j2']:
with open(os.path.join(d, t)) as fh:
package.pkg.add_contents(function_name + '/msg-templates/%s' % t, fh.read())
function_config = copy.deepcopy(config)
function_config['templates_folders'] = [function_name + '/msg-templates/']
package.pkg.add_contents(
function_name + '/config.json',
contents=json.dumps(function_config))
package.close()
return package
def provision(config):
log = logging.getLogger('c7n_mailer.azure.deploy')
function_name = config.get('function_name', 'mailer')
func_config = dict(
name=function_name,
servicePlanName=config.get('function_servicePlanName', 'cloudcustodian'),
location=config.get('function_location'),
appInsightsLocation=config.get('function_appInsightsLocation'),
schedule=config.get('function_schedule', '0 */10 * * * *'),
skuCode=config.get('function_skuCode'),
sku=config.get('function_sku'))
template_util = TemplateUtilities()
parameters = _get_parameters(template_util, func_config)
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
# Check if already existing
existing_webapp = template_util.resource_exist(group_name, webapp_name)
# Deploy
if not existing_webapp:
template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
template_util.deploy_resource_template(
group_name, 'dedicated_functionapp.json', parameters).wait()
else:
log.info("Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
log.info("Building function package for %s" % webapp_name)
# Build package
packager = FunctionPackage(
function_name,
os.path.join(os.path.dirname(__file__), 'function.py'))
packager.build(None,
entry_point=os.path.join(os.path.dirname(__file__), 'handle.py'),
extra_modules={'c7n_mailer', 'ruamel'})
packager.pkg.add_contents(
function_name + '/config.json',
contents=json.dumps(config))
packager.pkg.add_contents(
function_name + '/function.json',
contents=packager.get_function_config({'mode':
{'type': 'azure-periodic',
'schedule': func_config['schedule']}}))
# Add mail templates
template_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..', 'msg-templates'))
for t in os.listdir(template_dir):
with open(os.path.join(template_dir, t)) as fh:
packager.pkg.add_contents('msg-templates/%s' % t, fh.read())
packager.close()
if packager.wait_for_status(webapp_name):
packager.publish(webapp_name)
else:
log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(config):
log = logging.getLogger('c7n_mailer.azure.deploy')
function_name = config.get('function_name', 'mailer')
schedule = config.get('function_schedule', '0 */10 * * * *')
function_properties = config.get('function_properties', {})
# service plan is parse first, because its location might be shared with storage & insights
service_plan = AzureFunctionMode.extract_properties(function_properties,
'servicePlan',
{'name': 'cloud-custodian',
'location': 'westus2',
'resource_group_name': 'cloud-custodian',
'sku_name': 'B1',
'sku_tier': 'Basic'})
location = service_plan.get('location', 'westus2')
rg_name = service_plan['resource_group_name']
sub_id = local_session(Session).get_subscription_id()
suffix = StringUtils.naming_hash(rg_name + sub_id)
storage_account = AzureFunctionMode.extract_properties(function_properties,
'storageAccount',
{'name': 'mailerstorage' + suffix,
'location': location,
'resource_group_name': rg_name})
app_insights = AzureFunctionMode.extract_properties(function_properties,
'appInsights',
{'name': service_plan['name'],
'location': location,
'resource_group_name': rg_name})
function_app_name = \
'-'.join([service_plan['name'], function_name, suffix]) \
.replace(' ', '-').lower()
params = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights=app_insights,
service_plan=service_plan,
storage_account=storage_account,
function_app_resource_group_name=service_plan['resource_group_name'],
function_app_name=function_app_name)
function_app = FunctionAppUtilities().deploy_dedicated_function_app(params)
log.info("Building function package for %s" % function_app_name)
# Build package
packager = FunctionPackage(
function_name,
os.path.join(os.path.dirname(__file__), 'function.py'))
packager.build(None,
entry_point=os.path.join(os.path.dirname(__file__), 'handle.py'),
extra_modules={'c7n_mailer', 'ruamel'})
packager.pkg.add_contents(
function_name + '/config.json',
contents=json.dumps(config))
packager.pkg.add_contents(
function_name + '/function.json',
contents=packager.get_function_config({'mode':
{'type': 'azure-periodic',
'schedule': schedule}}))
# Add mail templates
template_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..', 'msg-templates'))
for t in os.listdir(template_dir):
with open(os.path.join(template_dir, t)) as fh:
packager.pkg.add_contents('msg-templates/%s' % t, fh.read())
packager.close()
if packager.wait_for_status(function_app):
packager.publish(function_app)
else:
log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(config):
log = logging.getLogger('c7n_mailer.azure.deploy')
function_name = config.get('function_name', 'mailer')
schedule = config.get('function_schedule', '0 */10 * * * *')
function_properties = config.get('function_properties', {})
# service plan is parse first, because its location might be shared with storage & insights
service_plan = AzureFunctionMode.extract_properties(
function_properties, 'servicePlan', {
'name': 'cloud-custodian',
'location': 'westus2',
'resource_group_name': 'cloud-custodian',
'sku_name': 'B1',
'sku_tier': 'Basic'
})
location = service_plan.get('location', 'westus2')
rg_name = service_plan['resource_group_name']
sub_id = local_session(Session).get_subscription_id()
suffix = StringUtils.naming_hash(rg_name + sub_id)
storage_account = AzureFunctionMode.extract_properties(
function_properties, 'storageAccount', {
'name': 'mailerstorage' + suffix,
'location': location,
'resource_group_name': rg_name
})
app_insights = AzureFunctionMode.extract_properties(
function_properties, 'appInsights', {
'name': service_plan['name'],
'location': location,
'resource_group_name': rg_name
})
function_app_name = \
'-'.join([service_plan['name'], function_name, suffix]) \
.replace(' ', '-').lower()
params = FunctionAppUtilities.FunctionAppInfrastructureParameters(
app_insights=app_insights,
service_plan=service_plan,
storage_account=storage_account,
function_app_resource_group_name=service_plan['resource_group_name'],
function_app_name=function_app_name)
function_app = FunctionAppUtilities().deploy_dedicated_function_app(params)
log.info("Building function package for %s" % function_app_name)
# Build package
packager = FunctionPackage(
function_name, os.path.join(os.path.dirname(__file__), 'function.py'))
packager.build(None,
entry_point=os.path.join(os.path.dirname(__file__),
'handle.py'),
extra_modules={'c7n_mailer', 'ruamel'})
packager.pkg.add_contents(function_name + '/config.json',
contents=json.dumps(config))
packager.pkg.add_contents(function_name + '/function.json',
contents=packager.get_function_config({
'mode': {
'type': 'azure-periodic',
'schedule': schedule
}
}))
# Add mail templates
template_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..', 'msg-templates'))
for t in os.listdir(template_dir):
with open(os.path.join(template_dir, t)) as fh:
packager.pkg.add_contents('msg-templates/%s' % t, fh.read())
packager.close()
if packager.wait_for_status(function_app):
packager.publish(function_app)
else:
log.error("Aborted deployment, ensure Application Service is healthy.")
def provision(config):
log = logging.getLogger('c7n_mailer.azure.deploy')
function_name = config.get('function_name', 'mailer')
func_config = dict(
name=function_name,
servicePlanName=config.get('function_servicePlanName',
'cloudcustodian'),
location=config.get('function_location'),
appInsightsLocation=config.get('function_appInsightsLocation'),
schedule=config.get('function_schedule', '0 */10 * * * *'),
skuCode=config.get('function_skuCode'),
sku=config.get('function_sku'))
template_util = TemplateUtilities()
parameters = _get_parameters(template_util, func_config)
group_name = parameters['servicePlanName']['value']
webapp_name = parameters['name']['value']
# Check if already existing
existing_webapp = template_util.resource_exist(group_name, webapp_name)
# Deploy
if not existing_webapp:
template_util.create_resource_group(
group_name, {'location': parameters['location']['value']})
template_util.deploy_resource_template(group_name,
'dedicated_functionapp.json',
parameters).wait()
else:
log.info("Found existing App %s (%s) in group %s" %
(webapp_name, existing_webapp.location, group_name))
log.info("Building function package for %s" % webapp_name)
# Build package
packager = FunctionPackage(
function_name, os.path.join(os.path.dirname(__file__), 'function.py'))
packager.build(None,
entry_point=os.path.join(os.path.dirname(__file__),
'handle.py'),
extra_modules={'c7n_mailer', 'ruamel'})
packager.pkg.add_contents(function_name + '/config.json',
contents=json.dumps(config))
packager.pkg.add_contents(function_name + '/function.json',
contents=packager.get_function_config({
'mode': {
'type': 'azure-periodic',
'schedule': func_config['schedule']
}
}))
# Add mail templates
template_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../..', 'msg-templates'))
for t in os.listdir(template_dir):
with open(os.path.join(template_dir, t)) as fh:
packager.pkg.add_contents('msg-templates/%s' % t, fh.read())
packager.close()
if packager.wait_for_status(webapp_name):
packager.publish(webapp_name)
else:
log.error("Aborted deployment, ensure Application Service is healthy.")
