def test_wait_for_stack_in_progress(self, sleep_mock, mock_el):
mock_el.side_effect = [[
self.mock_event('stack', 'aaa', 'Stack CREATE started',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
self.mock_event('thing', 'bbb', 'state changed',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
], [
self.mock_event('thing', 'ccc', 'state changed',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
self.mock_event('stack', 'ddd',
'Stack CREATE completed successfully',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
], [], []]
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = 'CREATE_IN_PROGRESS'
complete_stack = mock.Mock()
complete_stack.stack_name = 'stack'
complete_stack.stack_status = 'CREATE_COMPLETE'
self.mock_orchestration.stacks.get.side_effect = [
stack, stack, stack, complete_stack]
utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertEqual(2, sleep_mock.call_count)
def test_wait_for_stack_in_progress(self, sleep_mock, mock_el):
mock_el.side_effect = [
[
self.mock_event('stack', 'aaa', 'Stack CREATE started',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
self.mock_event('thing', 'bbb', 'state changed',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
],
[
self.mock_event('thing', 'ccc', 'state changed',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
self.mock_event('stack', 'ddd',
'Stack CREATE completed successfully',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
], [], []
]
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = 'CREATE_IN_PROGRESS'
complete_stack = mock.Mock()
complete_stack.stack_name = 'stack'
complete_stack.stack_status = 'CREATE_COMPLETE'
self.mock_orchestration.stacks.get.side_effect = [
stack, stack, stack, complete_stack
]
utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertEqual(2, sleep_mock.call_count)
def _heat_deploy(self, stack_name, template_path, parameters,
environments, timeout):
"""Verify the Baremetal nodes are available and do a stack update"""
clients = self.app.client_manager
orchestration_client = clients.tripleoclient.orchestration
deploy_manager = deploy.FileDeployManager(
orchestration_client, stack_name)
deploy_args = {
'template_path': template_path,
'environment_files': environments,
'parameters': parameters,
}
if timeout:
deploy_args['timeout_mins'] = timeout
deploy_manager.deploy(**deploy_args)
create_result = utils.wait_for_stack_ready(
orchestration_client, stack_name)
if not create_result:
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def update(clients, **workflow_input):
workflow_client = clients.workflow_engine
tripleoclients = clients.tripleoclient
plan_name = workflow_input['container']
with tripleoclients.messaging_websocket() as ws:
execution = base.start_workflow(
workflow_client,
'tripleo.package_update.v1.package_update_plan',
workflow_input=workflow_input)
for payload in base.wait_for_messages(workflow_client, ws, execution):
assert payload['status'] == "SUCCESS", pprint.pformat(payload)
orchestration_client = clients.orchestration
events = event_utils.get_events(orchestration_client,
stack_id=plan_name,
event_args={
'sort_dir': 'desc',
'limit': 1
})
marker = events[0].id if events else None
time.sleep(10)
create_result = utils.wait_for_stack_ready(orchestration_client, plan_name,
marker, 'UPDATE', 1)
if not create_result:
shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
raise exceptions.DeploymentError("Heat Stack update failed.")
def deploy_and_wait(log,
clients,
stack,
plan_name,
verbose_level,
timeout=None,
run_validations=False,
skip_deploy_identifier=False,
deployment_options={}):
"""Start the deploy and wait for it to finish"""
orchestration_client = clients.orchestration
if stack is None:
log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
else:
log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=plan_name,
event_args={
'sort_dir': 'desc',
'limit': 1
})
marker = events[0].id if events else None
action = 'UPDATE'
set_deployment_status(clients=clients, plan=plan_name, status='DEPLOYING')
try:
deploy(container=plan_name,
run_validations=run_validations,
skip_deploy_identifier=skip_deploy_identifier,
timeout=timeout,
verbosity=verbose_level)
except Exception:
set_deployment_status(clients=clients,
plan=plan_name,
status='DEPLOY_FAILED')
raise
# we always want the heat stack output while it's going.
verbose_events = True
# TODO(rabi) Simplify call to get events as we don't need to wait
# for stack to be ready anymore i.e just get the events.
create_result = utils.wait_for_stack_ready(orchestration_client, plan_name,
marker, action, verbose_events)
if not create_result:
shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
set_deployment_status(clients=clients,
plan=plan_name,
status='DEPLOY_FAILED')
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def test_wait_for_stack_ready(self, sleep_mock, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
self.mock_orchestration.stacks.get.return_value = stack
mock_el.side_effect = [
[
self.mock_event('stack', 'aaa', 'Stack CREATE started',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
self.mock_event('thing', 'bbb', 'state changed',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
],
[
self.mock_event('thing', 'ccc', 'state changed',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
self.mock_event('stack', 'ddd',
'Stack CREATE completed successfully',
'CREATE_COMPLETE', '2015-10-14T02:25:43Z'),
]
]
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertTrue(complete)
sleep_mock.assert_called_once_with(mock.ANY)
def test_wait_for_stack_ready(self, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = "CREATE_COMPLETE"
self.mock_orchestration.stacks.get.return_value = stack
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertTrue(complete)
def test_wait_for_stack_ready(self, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = "CREATE_COMPLETE"
self.mock_orchestration.stacks.get.return_value = stack
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertTrue(complete)
def test_wait_for_stack_ready_failed(self, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = "CREATE_FAILED"
self.mock_orchestration.stacks.get.return_value = stack
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertFalse(complete)
def _stack_delete(self, orchestration_client, stack_name):
print("Deleting stack {s}...".format(s=stack_name))
stack = utils.get_stack(orchestration_client, stack_name)
if stack is None:
self.log.warning("No stack found ('{s}'), skipping delete".
format(s=stack_name))
else:
try:
utils.wait_for_stack_ready(
orchestration_client=orchestration_client,
stack_name=stack_name,
action='DELETE')
except Exception as e:
self.log.error("Exception while waiting for stack to delete "
"{}".format(e))
raise oscexc.CommandError(
"Error occurred while waiting for stack to delete {}".
format(e))
def test_wait_for_stack_ready_failed(self, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = "CREATE_FAILED"
self.mock_orchestration.stacks.get.return_value = stack
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertFalse(complete)
def test_wait_for_stack_ready_no_stack(self):
self.mock_orchestration.reset_mock()
self.mock_orchestration.stacks.get.return_value = None
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.mock_orchestration.stacks.get.return_value = self.mock_stacks
self.assertFalse(complete)
def test_wait_for_stack_in_progress(self, mock_poll_for_events):
mock_poll_for_events.return_value = ("CREATE_IN_PROGRESS", "MESSAGE")
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = 'CREATE_IN_PROGRESS'
self.mock_orchestration.stacks.get.return_value = stack
result = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertEqual(False, result)
def test_wait_for_stack_in_progress(self, mock_poll_for_events):
mock_poll_for_events.return_value = ("CREATE_IN_PROGRESS", "MESSAGE")
stack = mock.Mock()
stack.stack_name = 'stack'
stack.stack_status = 'CREATE_IN_PROGRESS'
self.mock_orchestration.stacks.get.return_value = stack
result = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertEqual(False, result)
def test_wait_for_stack_ready_failed(self):
self.mock_orchestration.reset_mock()
self.mock_stacks.reset_mock()
return_values = ['CREATE_FAILED']
self.stack_status.side_effect = return_values
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertFalse(complete)
def deploy_and_wait(log,
clients,
stack,
plan_name,
verbose_level,
timeout=None,
run_validations=False,
skip_deploy_identifier=False,
deployment_options={}):
"""Start the deploy and wait for it to finish"""
workflow_input = {
"container": plan_name,
"run_validations": run_validations,
"skip_deploy_identifier": skip_deploy_identifier,
"deployment_options": deployment_options,
}
if timeout is not None:
workflow_input['timeout'] = timeout
deploy(log, clients, **workflow_input)
orchestration_client = clients.orchestration
if stack is None:
log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
else:
log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=plan_name,
event_args={
'sort_dir': 'desc',
'limit': 1
})
marker = events[0].id if events else None
action = 'UPDATE'
time.sleep(10)
verbose_events = verbose_level >= 1
create_result = utils.wait_for_stack_ready(orchestration_client, plan_name,
marker, action, verbose_events)
if not create_result:
shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
set_deployment_status(clients, 'failed', plan=plan_name)
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def _heat_deploy(self, stack, stack_name, template_path, parameters,
environments, timeout):
"""Verify the Baremetal nodes are available and do a stack update"""
self.log.debug("Processing environment files")
env_files, env = (
template_utils.process_multiple_environments_and_files(
environments))
if stack:
update.add_breakpoints_cleanup_into_env(env)
self.log.debug("Getting template contents")
template_files, template = template_utils.get_template_contents(
template_path)
files = dict(list(template_files.items()) + list(env_files.items()))
clients = self.app.client_manager
orchestration_client = clients.tripleoclient.orchestration()
self.log.debug("Deploying stack: %s", stack_name)
self.log.debug("Deploying template: %s", template)
self.log.debug("Deploying parameters: %s", parameters)
self.log.debug("Deploying environment: %s", env)
self.log.debug("Deploying files: %s", files)
stack_args = {
'stack_name': stack_name,
'template': template,
'environment': env,
'files': files
}
if timeout:
stack_args['timeout_mins'] = timeout
if stack is None:
self.log.info("Performing Heat stack create")
orchestration_client.stacks.create(**stack_args)
else:
self.log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
stack_args['existing'] = 'true'
orchestration_client.stacks.update(stack.id, **stack_args)
create_result = utils.wait_for_stack_ready(
orchestration_client, stack_name)
if not create_result:
if stack is None:
raise Exception("Heat Stack create failed.")
else:
raise Exception("Heat Stack update failed.")
def test_wait_for_stack_ready(self, sleep_mock):
self.mock_orchestration.reset_mock()
self.mock_stacks.reset_mock()
return_values = ['CREATE_IN_PROGRESS', 'CREATE_COMPLETE']
self.stack_status.side_effect = return_values
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertTrue(complete)
sleep_mock.assert_called_once_with(mock.ANY)
def deploy_and_wait(log,
clients,
stack,
plan_name,
verbose_level,
timeout=None):
"""Start the deploy and wait for it to finish"""
workflow_input = {
"container": plan_name,
"queue_name": str(uuid.uuid4()),
}
if timeout is not None:
workflow_input['timeout'] = timeout
deploy(clients, **workflow_input)
orchestration_client = clients.orchestration
if stack is None:
log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
else:
log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=plan_name,
event_args={
'sort_dir': 'desc',
'limit': 1
})
marker = events[0].id if events else None
action = 'UPDATE'
time.sleep(10)
verbose_events = verbose_level > 0
create_result = utils.wait_for_stack_ready(orchestration_client, plan_name,
marker, action, verbose_events)
if not create_result:
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def update(clients, container):
"""Update the heat stack outputs for purposes of update/upgrade.
This workflow assumes that previously the
plan_management.update_deployment_plan workflow has already been
run to process the templates and environments (the same way as
'deploy' command processes them).
:param clients: Application client object.
:type clients: Object
:param container: Container name to pull from.
:type container: String.
"""
tripleoclients = clients.tripleoclient
orchestration_client = clients.orchestration
object_client = tripleoclients.object_store
plan.update_plan_environment_with_image_parameters(
object_client, container)
stack.stack_update(object_client, orchestration_client,
_STACK_TIMEOUT, container)
events = event_utils.get_events(orchestration_client,
stack_id=container,
event_args={'sort_dir': 'desc',
'limit': 1})
marker = events[0].id if events else None
time.sleep(10)
create_result = utils.wait_for_stack_ready(
clients.orchestration,
container,
marker,
'UPDATE',
verbose=True
)
if not create_result:
raise exceptions.DeploymentError(
'Heat Stack update failed, run the following command'
' `openstack --os-cloud undercloud stack failures list {}`'
' to investigate these failures further.'.format(container)
)
def test_wait_for_stack_ready_failed(self, sleep_mock, mock_el):
stack = mock.Mock()
stack.stack_name = "stack"
self.mock_orchestration.stacks.get.return_value = stack
mock_el.side_effect = [
[
self.mock_event("stack", "aaa", "Stack CREATE started", "CREATE_IN_PROGRESS", "2015-10-14T02:25:21Z"),
self.mock_event("thing", "bbb", "state changed", "CREATE_IN_PROGRESS", "2015-10-14T02:25:21Z"),
],
[
self.mock_event("thing", "ccc", "ouch", "CREATE_FAILED", "2015-10-14T02:25:43Z"),
self.mock_event("stack", "ddd", "ouch", "CREATE_FAILED", "2015-10-14T02:25:43Z"),
],
]
complete = utils.wait_for_stack_ready(self.mock_orchestration, "stack", verbose=True)
self.assertFalse(complete)
sleep_mock.assert_called_once_with(mock.ANY)
def test_wait_for_stack_ready_failed(self, sleep_mock, mock_el):
stack = mock.Mock()
stack.stack_name = 'stack'
self.mock_orchestration.stacks.get.return_value = stack
mock_el.side_effect = [[
self.mock_event('stack', 'aaa', 'Stack CREATE started',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
self.mock_event('thing', 'bbb', 'state changed',
'CREATE_IN_PROGRESS', '2015-10-14T02:25:21Z'),
], [
self.mock_event('thing', 'ccc', 'ouch',
'CREATE_FAILED', '2015-10-14T02:25:43Z'),
self.mock_event('stack', 'ddd', 'ouch',
'CREATE_FAILED', '2015-10-14T02:25:43Z'),
]]
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertFalse(complete)
sleep_mock.assert_called_once_with(mock.ANY)
def deploy_and_wait(log, clients, stack, plan_name, verbose_level,
timeout=None):
"""Start the deploy and wait for it to finish"""
workflow_input = {
"container": plan_name,
"queue_name": str(uuid.uuid4()),
}
if timeout is not None:
workflow_input['timeout'] = timeout
deploy(clients, **workflow_input)
orchestration_client = clients.orchestration
if stack is None:
log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
else:
log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=plan_name,
event_args={'sort_dir': 'desc',
'limit': 1})
marker = events[0].id if events else None
action = 'UPDATE'
time.sleep(10)
verbose_events = verbose_level > 0
create_result = utils.wait_for_stack_ready(
orchestration_client, plan_name, marker, action, verbose_events)
if not create_result:
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def test_wait_for_stack_ready_no_stack(self):
self.mock_orchestration.stacks.get.return_value = None
complete = utils.wait_for_stack_ready(self.mock_orchestration, 'stack')
self.assertFalse(complete)
def _heat_deploy(self, stack, stack_name, template_path, parameters,
environments, timeout):
"""Verify the Baremetal nodes are available and do a stack update"""
self.log.debug("Processing environment files")
env_files, env = (
template_utils.process_multiple_environments_and_files(
environments))
if stack:
update.add_breakpoints_cleanup_into_env(env)
self.log.debug("Getting template contents")
template_files, template = template_utils.get_template_contents(
template_path)
files = dict(list(template_files.items()) + list(env_files.items()))
clients = self.app.client_manager
orchestration_client = clients.tripleoclient.orchestration
self.log.debug("Deploying stack: %s", stack_name)
self.log.debug("Deploying template: %s", template)
self.log.debug("Deploying parameters: %s", parameters)
self.log.debug("Deploying environment: %s", env)
self.log.debug("Deploying files: %s", files)
stack_args = {
'stack_name': stack_name,
'template': template,
'environment': env,
'files': files,
'clear_parameters': env['parameter_defaults'].keys(),
}
if timeout:
stack_args['timeout_mins'] = timeout
if stack is None:
self.log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
orchestration_client.stacks.create(**stack_args)
else:
self.log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
stack_args['existing'] = 'true'
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=stack_name,
event_args={'sort_dir': 'desc',
'limit': 1})
marker = events[0].id if events else None
action = 'UPDATE'
orchestration_client.stacks.update(stack.id, **stack_args)
verbose_events = self.app_args.verbose_level > 0
create_result = utils.wait_for_stack_ready(
orchestration_client, stack_name, marker, action, verbose_events)
if not create_result:
if stack is None:
raise Exception("Heat Stack create failed.")
else:
raise Exception("Heat Stack update failed.")
def _heat_deploy(self, stack, stack_name, template_path, parameters,
environments, timeout):
"""Verify the Baremetal nodes are available and do a stack update"""
self.log.debug("Processing environment files")
env_files, env = (
template_utils.process_multiple_environments_and_files(
environments))
if stack:
update.add_breakpoints_cleanup_into_env(env)
self.log.debug("Getting template contents")
template_files, template = template_utils.get_template_contents(
template_path)
files = dict(list(template_files.items()) + list(env_files.items()))
clients = self.app.client_manager
orchestration_client = clients.tripleoclient.orchestration
self.log.debug("Deploying stack: %s", stack_name)
self.log.debug("Deploying template: %s", template)
self.log.debug("Deploying parameters: %s", parameters)
self.log.debug("Deploying environment: %s", env)
self.log.debug("Deploying files: %s", files)
stack_args = {
'stack_name': stack_name,
'template': template,
'environment': env,
'files': files,
'clear_parameters': env['parameter_defaults'].keys(),
}
if timeout:
stack_args['timeout_mins'] = timeout
if stack is None:
self.log.info("Performing Heat stack create")
action = 'CREATE'
marker = None
orchestration_client.stacks.create(**stack_args)
else:
self.log.info("Performing Heat stack update")
# Make sure existing parameters for stack are reused
stack_args['existing'] = 'true'
# Find the last top-level event to use for the first marker
events = event_utils.get_events(orchestration_client,
stack_id=stack_name,
event_args={
'sort_dir': 'desc',
'limit': 1
})
marker = events[0].id if events else None
action = 'UPDATE'
orchestration_client.stacks.update(stack.id, **stack_args)
verbose_events = self.app_args.verbose_level > 0
create_result = utils.wait_for_stack_ready(orchestration_client,
stack_name, marker, action,
verbose_events)
if not create_result:
if stack is None:
raise exceptions.DeploymentError("Heat Stack create failed.")
else:
raise exceptions.DeploymentError("Heat Stack update failed.")
def scale_down(log,
clients,
stack,
nodes,
timeout=None,
verbosity=0,
connection_timeout=None):
"""Unprovision and deletes overcloud nodes from a heat stack.
:param log: Logging object
:type log: Object
:param clients: Application client object.
:type clients: Object
:param stack: Heat Stack object
:type stack: Object
:param nodes: List of nodes to delete. If the node UUID is used the
UUID will be used to lookup the node name before being
passed through to the cleanup playbook.
:type nodes: List
:param timeout: Timeout to use when deleting nodes. If timeout is None
it will be set to 240 minutes.
:type timeout: Integer
:param verbosity: Verbosity level
:type verbosity: Integer
:param connection_timeout: Ansible connection timeout in seconds.
:type connection_timeout: Integer
"""
if not timeout:
timeout = 240
limit_list = list()
for node in nodes:
try:
_node = clients.compute.servers.get(node)
limit_list.append(_node.name)
except Exception:
limit_list.append(node)
deployment.config_download(log=log,
clients=clients,
stack=stack,
timeout=connection_timeout,
ansible_playbook_name='scale_playbook.yaml',
limit_list=limit_list,
verbosity=verbosity,
deployment_timeout=timeout)
print('Running scale down')
context = clients.tripleoclient.create_mistral_context()
scale_down_action = scale.ScaleDownAction(nodes=nodes,
timeout=timeout,
container=stack.stack_name)
scale_down_action.run(context=context)
utils.wait_for_stack_ready(orchestration_client=clients.orchestration,
stack_name=stack.stack_name,
action='UPDATE')