def test_get_j2_excludes_file(self, get_obj_client_mock):
mock_ctx = mock.MagicMock()
swift = mock.MagicMock()
get_obj_client_mock.return_value = swift
def return_multiple_files(*args):
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
return ['', J2_EXCLUDES]
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
# Test - J2 exclude file with valid templates
action = templates.ProcessTemplatesAction()
self.assertTrue({'name': ['puppet/controller-role.yaml']} ==
action._get_j2_excludes_file(mock_ctx))
def return_multiple_files(*args):
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
return ['', J2_EXCLUDES_EMPTY_LIST]
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
# Test - J2 exclude file with no template to exlude
action = templates.ProcessTemplatesAction()
self.assertTrue({'name': []} == action._get_j2_excludes_file(mock_ctx))
def return_multiple_files(*args):
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
return ['', J2_EXCLUDES_EMPTY_FILE]
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
# Test - J2 exclude file empty
action = templates.ProcessTemplatesAction()
self.assertTrue({'name': []} == action._get_j2_excludes_file(mock_ctx))
def test_legacy_api_network_exists(self, get_obj_client_mock, j2_mock,
resource_exists_mock):
resource_exists_mock.return_value = True
swift = self._custom_roles_mock_objclient('role-networks.role.j2.yaml',
JINJA_SNIPPET_ROLE_NETWORKS,
ROLE_DATA_ENABLE_NETWORKS)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
mock_ctx = mock.MagicMock()
action._process_custom_roles(mock_ctx)
expected_j2_template = get_obj_client_mock.get_object(
action.container, 'foo.j2.yaml')[1]
expected_j2_data = {
'roles': [{
'name': 'CustomRole'
}],
'networks': [{
'name': 'InternalApi',
'compat_name': 'Internal'
}]
}
assert j2_mock.called_with(expected_j2_template, expected_j2_data,
'foo.yaml', mock_ctx)
def test_j2_render_and_put_include_relative(
self,
get_obj_client_mock):
def return_multiple_files(*args):
if args[1] == 'bar/foo.yaml':
return ['', JINJA_SNIPPET_CONFIG]
def return_container_files(*args):
return ('headers', [{'name': 'bar/foo.yaml'}])
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
swift.get_container = mock.MagicMock(
side_effect=return_container_files)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._j2_render_and_put(r"{% include 'foo.yaml' %}",
{'role': 'CustomRole'},
'bar/customrole-config.yaml')
action_result = swift.put_object._mock_mock_calls[0]
self.assertTrue("CustomRole" in str(action_result))
def _process_custom_roles_disable_constraints(
self, snippet, get_obj_client_mock, resource_exists_mock):
resource_exists_mock.return_value = False
swift = self._custom_roles_mock_objclient(
'disable-constraints.role.j2.yaml', snippet,
ROLE_DATA_DISABLE_CONSTRAINTS_YAML)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
mock_ctx = mock.MagicMock()
action._process_custom_roles(mock_ctx)
expected = EXPECTED_JINJA_RESULT.replace(
'CustomRole', 'RoleWithDisableConstraints')
put_object_mock_call = mock.call(
constants.DEFAULT_CONTAINER_NAME,
'overcloud.yaml',
expected)
self.assertEqual(swift.put_object.call_args_list[0],
put_object_mock_call)
put_object_mock_call = mock.call(
constants.DEFAULT_CONTAINER_NAME,
"rolewithdisableconstraints-disable-constraints.yaml",
EXPECTED_JINJA_RESULT_DISABLE_CONSTRAINTS)
self.assertEqual(put_object_mock_call,
swift.put_object.call_args_list[1])
def test_custom_roles_networks(self, get_obj_client_mock,
resource_exists_mock):
resource_exists_mock.return_value = False
swift = self._custom_roles_mock_objclient(
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
ROLE_DATA_ENABLE_NETWORKS)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
mock_ctx = mock.MagicMock()
action._process_custom_roles(mock_ctx)
expected = EXPECTED_JINJA_RESULT.replace(
'CustomRole', 'RoleWithNetworks')
put_object_mock_call = mock.call(
constants.DEFAULT_CONTAINER_NAME,
'overcloud.yaml',
expected)
self.assertEqual(swift.put_object.call_args_list[0],
put_object_mock_call)
put_object_mock_call = mock.call(
constants.DEFAULT_CONTAINER_NAME,
"rolewithnetworks-role-networks.yaml",
EXPECTED_JINJA_RESULT_ROLE_NETWORKS)
self.assertEqual(put_object_mock_call,
swift.put_object.call_args_list[1])
def test_run(self, mock_get_object_client,
mock_get_template_contents,
mock_process_multiple_environments_and_files):
mock_ctx = mock.MagicMock()
swift = mock.MagicMock(url="http://test.com")
mock_env = yaml.safe_dump({
'temp_environment': 'temp_environment',
'template': 'template',
'environments': [{u'path': u'environments/test.yaml'}]
}, default_flow_style=False)
swift.get_object.side_effect = (
({}, mock_env),
swiftexceptions.ClientException('atest2')
)
mock_get_object_client.return_value = swift
mock_get_template_contents.return_value = ({}, {
'heat_template_version': '2016-04-30'
})
mock_process_multiple_environments_and_files.return_value = ({}, {})
# Test
action = templates.ProcessTemplatesAction()
result = action.run(mock_ctx)
# Verify the values we get out
self.assertEqual(result, {
'environment': {},
'files': {},
'stack_name': constants.DEFAULT_CONTAINER_NAME,
'template': {
'heat_template_version': '2016-04-30'
}
})
def test_process_custom_roles(self, get_obj_client_mock,
resource_exists_mock):
resource_exists_mock.return_value = False
swift = self._custom_roles_mock_objclient(
'foo.j2.yaml', JINJA_SNIPPET)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
mock_ctx = mock.MagicMock()
action._process_custom_roles(mock_ctx)
get_object_mock_calls = [
mock.call('overcloud', constants.OVERCLOUD_J2_NAME),
mock.call('overcloud', constants.OVERCLOUD_J2_ROLES_NAME),
mock.call('overcloud', 'foo.j2.yaml'),
]
swift.get_object.assert_has_calls(
get_object_mock_calls, any_order=True)
put_object_mock_calls = [
mock.call(constants.DEFAULT_CONTAINER_NAME,
constants.OVERCLOUD_YAML_NAME,
EXPECTED_JINJA_RESULT),
mock.call(constants.DEFAULT_CONTAINER_NAME,
'foo.yaml',
EXPECTED_JINJA_RESULT),
]
swift.put_object.assert_has_calls(
put_object_mock_calls, any_order=True)
def _update_stack(self,
parameters={},
timeout_mins=constants.STACK_TIMEOUT_DEFAULT,
context=None):
# TODO(rbrady): migrate _update_stack to it's own action and update
# the workflow for scale down
# update the plan parameters with the scaled down parameters
update_params_action = parameters_actions.UpdateParametersAction(
parameters, self.container)
updated_plan = update_params_action.run(context)
if isinstance(updated_plan, actions.Result):
return updated_plan
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
if isinstance(processed_data, actions.Result):
return processed_data
update.add_breakpoints_cleanup_into_env(processed_data['environment'])
fields = processed_data.copy()
fields['timeout_mins'] = timeout_mins
fields['existing'] = True
# As we do a PATCH update when deleting nodes, parameters set for a
# stack before upgrade to newton (ex. ComputeRemovalPolicies),
# would still take precedence over the ones set in parameter_defaults
# after upgrade. Clear these parameters for backward compatibility.
fields['clear_parameters'] = list(parameters.keys())
LOG.debug('stack update params: %s', fields)
self.get_orchestration_client(context).stacks.update(
self.container, **fields)
def test_process_custom_roles(self, ctx_mock, get_obj_client_mock):
def return_multiple_files(*args):
if args[1] == constants.OVERCLOUD_J2_NAME:
return ['', JINJA_SNIPPET]
if args[1] == 'foo.j2.yaml':
return ['', JINJA_SNIPPET]
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
return ['', J2_EXCLUDES]
elif args[1] == constants.OVERCLOUD_J2_ROLES_NAME:
return ['', ROLE_DATA_YAML]
elif args[1] == constants.OVERCLOUD_J2_NETWORKS_NAME:
return ['', NETWORK_DATA_YAML]
def return_container_files(*args):
return ('headers', [{
'name': constants.OVERCLOUD_J2_NAME
}, {
'name': 'foo.j2.yaml'
}, {
'name': constants.OVERCLOUD_J2_ROLES_NAME
}, {
'name': constants.OVERCLOUD_J2_NETWORKS_NAME
}])
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
swift.get_container = mock.MagicMock(
side_effect=return_container_files)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._process_custom_roles()
get_object_mock_calls = [
mock.call('overcloud', constants.OVERCLOUD_J2_NAME),
mock.call('overcloud', constants.OVERCLOUD_J2_ROLES_NAME),
mock.call('overcloud', 'foo.j2.yaml'),
]
swift.get_object.assert_has_calls(get_object_mock_calls,
any_order=True)
put_object_mock_calls = [
mock.call(constants.DEFAULT_CONTAINER_NAME,
constants.OVERCLOUD_YAML_NAME, EXPECTED_JINJA_RESULT),
mock.call(constants.DEFAULT_CONTAINER_NAME, 'foo.yaml',
EXPECTED_JINJA_RESULT),
]
swift.put_object.assert_has_calls(put_object_mock_calls,
any_order=True)
def test_no_heat_resource_exists(self, client_mock):
mock_ctx = mock.MagicMock()
heat_client = mock.MagicMock()
stack = mock.MagicMock(stack_name='overcloud')
def return_not_found(*args):
raise heat_exc.HTTPNotFound()
heat_client.resources.get.side_effect = return_not_found
client_mock.return_value = heat_client
action = templates.ProcessTemplatesAction()
self.assertFalse(
action._heat_resource_exists(
heat_client, stack, 'Networks', 'InternalNetwork', mock_ctx))
def test_j2_render_and_put(self, get_obj_client_mock):
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock()
swift.get_container = mock.MagicMock()
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._j2_render_and_put(JINJA_SNIPPET_CONFIG, {'role': 'CustomRole'},
'customrole-config.yaml')
action_result = swift.put_object._mock_mock_calls[0]
self.assertTrue("CustomRole" in str(action_result))
def run(self, context):
cached = self.cache_get(context, self.container,
"tripleo.parameters.get")
if cached is not None:
return cached
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
# If we receive a 'Result' instance it is because the parent action
# had an error.
if isinstance(processed_data, actions.Result):
return processed_data
processed_data['show_nested'] = True
# respect previously user set param values
swift = self.get_object_client(context)
heat = self.get_orchestration_client(context)
try:
env = plan_utils.get_env(swift, self.container)
except swiftexceptions.ClientException as err:
err_msg = ("Error retrieving environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
params = env.get('parameter_defaults')
fields = {
'template': processed_data['template'],
'files': processed_data['files'],
'environment': processed_data['environment'],
'show_nested': True
}
result = {
'heat_resource_tree': heat.stacks.validate(**fields),
'environment_parameters': params,
}
self.cache_set(context, self.container, "tripleo.parameters.get",
result)
return result
def test_heat_resource_exists(self, client_mock):
mock_ctx = mock.MagicMock()
heat_client = mock.MagicMock()
stack = mock.MagicMock(stack_name='overcloud')
heat_client.resources.get.return_value = mock.MagicMock(
links=[{'rel': 'stack',
'href': 'http://192.0.2.1:8004/v1/'
'a959ac7d6a4a475daf2428df315c41ef/'
'stacks/overcloud/123'}],
logical_resource_id='logical_id',
physical_resource_id='resource_id',
resource_type='OS::Heat::ResourceGroup',
resource_name='InternalApiNetwork'
)
client_mock.return_value = heat_client
action = templates.ProcessTemplatesAction()
self.assertTrue(
action._heat_resource_exists(
heat_client, stack, 'Networks', 'InternalNetwork', mock_ctx))
def test_run(self, mock_ctx, mock_get_object_client,
mock_get_workflow_client, mock_get_template_contents,
mock_process_multiple_environments_and_files):
mock_ctx.return_value = mock.MagicMock()
swift = mock.MagicMock(url="http://test.com")
swift.get_object.side_effect = swiftexceptions.ClientException(
'atest2')
mock_get_object_client.return_value = swift
mock_mistral = mock.MagicMock()
mock_env = mock.MagicMock()
mock_env.variables = {
'temp_environment': 'temp_environment',
'template': 'template',
'environments': [{
u'path': u'environments/test.yaml'
}],
}
mock_mistral.environments.get.return_value = mock_env
mock_get_workflow_client.return_value = mock_mistral
mock_get_template_contents.return_value = ({}, {
'heat_template_version':
'2016-04-30'
})
mock_process_multiple_environments_and_files.return_value = ({}, {})
# Test
action = templates.ProcessTemplatesAction()
result = action.run()
# Verify the values we get out
self.assertEqual(
result, {
'environment': {},
'files': {},
'stack_name': constants.DEFAULT_CONTAINER_NAME,
'template': {
'heat_template_version': '2016-04-30'
}
})
def test_process_custom_roles_disable_constraints(self, ctx_mock,
get_obj_client_mock):
def return_multiple_files(*args):
if args[1] == constants.OVERCLOUD_J2_NAME:
return ['', JINJA_SNIPPET_DISABLE_CONSTRAINTS]
if args[1] == 'disable-constraints.role.j2.yaml':
return ['', JINJA_SNIPPET_DISABLE_CONSTRAINTS]
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
return ['', J2_EXCLUDES]
elif args[1] == constants.OVERCLOUD_J2_ROLES_NAME:
return ['', ROLE_DATA_DISABLE_CONSTRAINTS_YAML]
elif args[1] == constants.OVERCLOUD_J2_NETWORKS_NAME:
return ['', NETWORK_DATA_YAML]
def return_container_files(*args):
return ('headers', [{
'name': constants.OVERCLOUD_J2_NAME
}, {
'name': 'disable-constraints.role.j2.yaml'
}, {
'name': constants.OVERCLOUD_J2_ROLES_NAME
}, {
'name': constants.OVERCLOUD_J2_NETWORKS_NAME
}])
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
swift.get_container = mock.MagicMock(
side_effect=return_container_files)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._process_custom_roles()
put_object_mock_call = mock.call(
constants.DEFAULT_CONTAINER_NAME,
"rolewithdisableconstraints-disable-constraints.yaml",
EXPECTED_JINJA_RESULT_DISABLE_CONSTRAINTS)
self.assertEqual(swift.put_object.call_args_list[1],
put_object_mock_call)
def run(self, context):
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
# If we receive a 'Result' instance it is because the parent action
# had an error.
if isinstance(processed_data, actions.Result):
return processed_data
# Default temporary value is used when no user input for any
# interface routes for the role networks to find network config.
role_networks = processed_data['template'].get('resources', {}).get(
self.role_name + 'GroupVars',
{}).get('properties', {}).get('value',
{}).get('role_networks', [])
for nw in role_networks:
rt = nw + 'InterfaceRoutes'
if rt not in processed_data['environment']['parameter_defaults']:
processed_data['environment']['parameter_defaults'][rt] = [[]]
# stacks.preview method raises validation message if stack is
# already deployed. here renaming container to get preview data.
container_temp = self.container + "-TEMP"
fields = {
'template': processed_data['template'],
'files': processed_data['files'],
'environment': processed_data['environment'],
'stack_name': container_temp,
}
orc = self.get_orchestration_client(context)
preview_data = orc.stacks.preview(**fields)
try:
result = self.get_network_config(preview_data, container_temp,
self.role_name)
return result
except exception.DeriveParamsError as err:
LOG.exception('Derive Params Error: %s' % err)
return actions.Result(error=str(err))
def run(self, context):
# get the stack. Error if doesn't exist
heat = self.get_orchestration_client(context)
try:
stack = heat.stacks.get(self.container)
except heat_exc.HTTPNotFound:
msg = "Error retrieving stack: %s" % self.container
LOG.exception(msg)
return actions.Result(error=msg)
swift = self.get_object_client(context)
try:
env = plan_utils.get_env(swift, self.container)
except swiftexceptions.ClientException as err:
err_msg = ("Error retrieving environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
update_env = {
'parameter_defaults': {
'DeployIdentifier': int(time.time()),
},
}
noop_env = {
'resource_registry': {
'OS::TripleO::DeploymentSteps': 'OS::Heat::None',
},
}
for output in stack.to_dict().get('outputs', {}):
if output['output_key'] == 'RoleData':
for role in output['output_value']:
role_env = {
"OS::TripleO::Tasks::%sPreConfig" % role:
'OS::Heat::None',
"OS::TripleO::Tasks::%sPostConfig" % role:
'OS::Heat::None',
}
noop_env['resource_registry'].update(role_env)
update_env.update(noop_env)
template_utils.deep_update(env, update_env)
try:
plan_utils.put_env(swift, env)
except swiftexceptions.ClientException as err:
err_msg = ("Error updating environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
# process all plan files and create or update a stack
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
# If we receive a 'Result' instance it is because the parent action
# had an error.
if isinstance(processed_data, actions.Result):
return processed_data
stack_args = processed_data.copy()
LOG.info("Performing Heat stack update")
LOG.info('updating stack: %s', stack.stack_name)
return heat.stacks.update(stack.id, **stack_args)
def run(self, context):
# check to see if the stack exists
heat = self.get_orchestration_client(context)
try:
stack = heat.stacks.get(self.container, resolve_outputs=False)
except heat_exc.HTTPNotFound:
stack = None
stack_is_new = stack is None
# update StackAction, DeployIdentifier and UpdateIdentifier
swift = self.get_object_client(context)
parameters = dict()
if not self.skip_deploy_identifier:
parameters['DeployIdentifier'] = int(time.time())
parameters['UpdateIdentifier'] = ''
parameters['StackAction'] = 'CREATE' if stack_is_new else 'UPDATE'
try:
env = plan_utils.get_env(swift, self.container)
except swiftexceptions.ClientException as err:
err_msg = ("Error retrieving environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
self.set_tls_parameters(parameters, env)
try:
plan_utils.update_in_env(swift, env, 'parameter_defaults',
parameters)
except swiftexceptions.ClientException as err:
err_msg = ("Error updating environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
if not stack_is_new:
try:
LOG.debug('Checking for compatible neutron mechanism drivers')
msg = update.check_neutron_mechanism_drivers(
env, stack, swift, self.container)
if msg:
return actions.Result(error=msg)
except swiftexceptions.ClientException as err:
err_msg = ("Error getting template %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
# process all plan files and create or update a stack
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
# If we receive a 'Result' instance it is because the parent action
# had an error.
if isinstance(processed_data, actions.Result):
return processed_data
stack_args = processed_data.copy()
stack_args['timeout_mins'] = self.timeout_mins
if stack_is_new:
try:
swift.copy_object(
"%s-swift-rings" % self.container, "swift-rings.tar.gz",
"%s-swift-rings/%s-%d" %
(self.container, "swift-rings.tar.gz", time.time()))
swift.delete_object("%s-swift-rings" % self.container,
"swift-rings.tar.gz")
except swiftexceptions.ClientException:
pass
LOG.info("Perfoming Heat stack create")
try:
return heat.stacks.create(**stack_args)
except heat_exc.HTTPException as err:
err_msg = "Error during stack creation: %s" % (err, )
LOG.exception(err_msg)
return actions.Result(error=err_msg)
LOG.info("Performing Heat stack update")
stack_args['existing'] = 'true'
try:
return heat.stacks.update(stack.id, **stack_args)
except heat_exc.HTTPException as err:
err_msg = "Error during stack update: %s" % (err, )
LOG.exception(err_msg)
return actions.Result(error=err_msg)
def run(self, context):
swift = self.get_object_client(context)
heat = self.get_orchestration_client(context)
try:
env = plan_utils.get_env(swift, self.container)
except swiftexceptions.ClientException as err:
err_msg = ("Error retrieving environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
saved_env = copy.deepcopy(env)
try:
plan_utils.update_in_env(swift, env, self.key, self.parameters)
except swiftexceptions.ClientException as err:
err_msg = ("Error updating environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
process_templates_action = templates.ProcessTemplatesAction(
container=self.container)
processed_data = process_templates_action.run(context)
# If we receive a 'Result' instance it is because the parent action
# had an error.
if isinstance(processed_data, actions.Result):
return processed_data
env = plan_utils.get_env(swift, self.container)
if not self.validate:
return env
params = env.get('parameter_defaults')
fields = {
'template': processed_data['template'],
'files': processed_data['files'],
'environment': processed_data['environment'],
'show_nested': True
}
try:
result = {
'heat_resource_tree': heat.stacks.validate(**fields),
'environment_parameters': params,
}
# Validation passes so the old cache gets replaced.
self.cache_set(context, self.container, "tripleo.parameters.get",
result)
if result['heat_resource_tree']:
flattened = {'resources': {}, 'parameters': {}}
_flat_it(flattened, 'Root', result['heat_resource_tree'])
result['heat_resource_tree'] = flattened
except heat_exc.HTTPException as err:
LOG.debug("Validation failed rebuilding saved env")
# There has been an error validating we must reprocess the
# templates with the saved working env
plan_utils.put_env(swift, saved_env)
process_templates_action._process_custom_roles(context)
err_msg = ("Error validating environment for plan %s: %s" %
(self.container, err))
LOG.exception(err_msg)
return actions.Result(error=err_msg)
LOG.debug("Validation worked new env is saved")
return result