def __call__(self):
"""Return a co-routine that updates the stack."""
existing_deps = self.existing_stack.dependencies
new_deps = self.new_stack.dependencies
cleanup_prev = scheduler.DependencyTaskGroup(
self.previous_stack.dependencies,
self._remove_backup_resource,
reverse=True)
cleanup = scheduler.DependencyTaskGroup(existing_deps,
self._remove_old_resource,
reverse=True)
create_new = scheduler.DependencyTaskGroup(new_deps,
self._create_new_resource)
update = scheduler.DependencyTaskGroup(new_deps,
self._update_resource)
if not self.rollback:
yield cleanup_prev()
yield create_new()
try:
yield update()
finally:
prev_deps = self.previous_stack._get_dependencies(
self.previous_stack.resources.itervalues())
self.previous_stack.dependencies = prev_deps
yield cleanup()
def stack_task(self, action, reverse=False, post_func=None):
'''
A task to perform an action on the stack and all of the resources
in forward or reverse dependency order as specfifed by reverse
'''
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action.lower()
res = None
def resource_action(r):
# Find e.g resource.create and call it
action_l = action.lower()
handle = getattr(r, '%s' % action_l)
return handle()
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource_action, reverse)
try:
yield action_task()
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action.lower(), str(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
self.state_set(action, stack_status, reason)
if callable(post_func):
post_func()
def create_task(self):
'''
A task to create the stack and all of the resources.
'''
self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started')
stack_status = self.CREATE_COMPLETE
reason = 'Stack successfully created'
res = None
def resource_create(r):
return r.create
create_task = scheduler.DependencyTaskGroup(self.dependencies,
resource_create)
try:
yield create_task()
except exception.ResourceFailure as ex:
stack_status = self.CREATE_FAILED
reason = 'Resource failed: %s' % str(ex)
except scheduler.Timeout:
stack_status = self.CREATE_FAILED
reason = 'Timed out'
self.state_set(stack_status, reason)
if stack_status == self.CREATE_FAILED and not self.disable_rollback:
self.delete(action=self.ROLLBACK)
def __call__(self):
"""Return a co-routine that updates the stack."""
existing_deps = self.existing_stack.dependencies
new_deps = self.new_stack.dependencies
cleanup = scheduler.DependencyTaskGroup(existing_deps,
self._remove_old_resource,
reverse=True)
create_new = scheduler.DependencyTaskGroup(new_deps,
self._create_new_resource)
update = scheduler.DependencyTaskGroup(new_deps, self._update_resource)
yield cleanup()
yield create_new()
yield update()
def delete(self, action=DELETE):
'''
Delete all of the resources, and then the stack itself.
The action parameter is used to differentiate between a user
initiated delete and an automatic stack rollback after a failed
create, which amount to the same thing, but the states are recorded
differently.
'''
if action not in (self.DELETE, self.ROLLBACK):
logger.error(_("Unexpected action %s passed to delete!") % action)
self.state_set(self.DELETE, self.FAILED,
"Invalid action %s" % action)
return
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action.lower()
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
backup_stack = self._backup_stack(False)
if backup_stack is not None:
backup_stack.delete()
if backup_stack.status != backup_stack.COMPLETE:
errs = backup_stack.status_reason
failure = 'Error deleting backup resources: %s' % errs
self.state_set(action, self.FAILED,
'Failed to %s : %s' % (action, failure))
return
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource.Resource.destroy,
reverse=True)
try:
scheduler.TaskRunner(action_task)(timeout=self.timeout_secs())
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action.lower(), str(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
if stack_status != self.FAILED:
# If we created a trust, delete it
stack = db_api.stack_get(self.context, self.id)
user_creds = db_api.user_creds_get(stack.user_creds_id)
trust_id = user_creds.get('trust_id')
if trust_id:
try:
self.clients.keystone().delete_trust(trust_id)
except Exception as ex:
logger.exception(ex)
stack_status = self.FAILED
reason = "Error deleting trust: %s" % str(ex)
self.state_set(action, stack_status, reason)
if stack_status != self.FAILED:
# delete the stack
db_api.stack_delete(self.context, self.id)
self.id = None
def stack_task(self,
action,
reverse=False,
post_func=None,
error_wait_time=None,
aggregate_exceptions=False):
'''
A task to perform an action on the stack and all of the resources
in forward or reverse dependency order as specified by reverse
'''
try:
lifecycle_plugin_utils.do_pre_ops(self.context, self, None, action)
except Exception as e:
self.state_set(
action, self.FAILED,
e.args[0] if e.args else 'Failed stack pre-ops: %s' %
six.text_type(e))
if callable(post_func):
post_func()
return
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action
def resource_action(r):
# Find e.g resource.create and call it
action_l = action.lower()
handle = getattr(r, '%s' % action_l)
# If a local _$action_kwargs function exists, call it to get the
# action specific argument list, otherwise an empty arg list
handle_kwargs = getattr(self, '_%s_kwargs' % action_l,
lambda x: {})
return handle(**handle_kwargs(r))
action_task = scheduler.DependencyTaskGroup(
self.dependencies,
resource_action,
reverse,
error_wait_time=error_wait_time,
aggregate_exceptions=aggregate_exceptions)
try:
yield action_task()
except (exception.ResourceFailure, scheduler.ExceptionGroup) as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action, six.text_type(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
self.state_set(action, stack_status, reason)
if callable(post_func):
post_func()
lifecycle_plugin_utils.do_post_ops(self.context, self, None, action,
(self.status == self.FAILED))
def __call__(self):
"""Return a co-routine that updates the stack."""
cleanup_prev = scheduler.DependencyTaskGroup(
self.previous_stack.dependencies,
self._remove_backup_resource,
reverse=True)
update = scheduler.DependencyTaskGroup(self.dependencies(),
self._resource_update)
if not self.rollback:
yield cleanup_prev()
try:
yield update()
finally:
self.previous_stack.reset_dependencies()
def __call__(self):
"""Return a co-routine that updates the stack."""
cleanup_prev = scheduler.DependencyTaskGroup(
self.previous_stack.dependencies,
self._remove_backup_resource,
reverse=True)
def get_error_wait_time(resource):
return resource.cancel_grace_period()
updater = scheduler.DependencyTaskGroup(
self.dependencies(),
self._resource_update,
error_wait_time=get_error_wait_time)
if not self.rollback:
yield cleanup_prev()
try:
yield updater()
finally:
self.previous_stack.reset_dependencies()
def _dep_test(self, *edges):
dummy = DummyTask(getattr(self, 'steps', 3))
deps = dependencies.Dependencies(edges)
tg = scheduler.DependencyTaskGroup(deps, dummy)
self.mox.StubOutWithMock(dummy, 'do_step')
yield dummy
self.mox.ReplayAll()
scheduler.TaskRunner(tg)(wait_time=None)
self.mox.VerifyAll()
def _dep_test(self, *edges):
dummy = DummyTask(getattr(self, 'steps', 3))
deps = dependencies.Dependencies(edges)
tg = scheduler.DependencyTaskGroup(
deps, dummy, aggregate_exceptions=self.aggregate_exceptions,
reverse=self.reverse_order)
self.m.StubOutWithMock(dummy, 'do_step')
yield dummy
self.m.ReplayAll()
scheduler.TaskRunner(tg)(wait_time=None)
def stack_task(self, action, reverse=False, post_func=None):
'''
A task to perform an action on the stack and all of the resources
in forward or reverse dependency order as specfifed by reverse
'''
self.state_set(action, self.IN_PROGRESS,
'Stack %s started' % action)
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action
def resource_action(r):
# Find e.g resource.create and call it
action_l = action.lower()
handle = getattr(r, '%s' % action_l)
# If a local _$action_kwargs function exists, call it to get the
# action specific argument list, otherwise an empty arg list
handle_kwargs = getattr(self,
'_%s_kwargs' % action_l, lambda x: {})
return handle(**handle_kwargs(r))
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource_action,
reverse)
try:
yield action_task()
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action, six.text_type(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
self.state_set(action, stack_status, reason)
if callable(post_func):
post_func()
def create(self):
'''
Create the stack and all of the resources.
Creation will fail if it exceeds the specified timeout. The default is
60 minutes, set in the constructor
'''
self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started')
stack_status = self.CREATE_COMPLETE
reason = 'Stack successfully created'
res = None
def resource_create(r):
return r.create
create_task = scheduler.DependencyTaskGroup(self.dependencies,
resource_create)
create = scheduler.TaskRunner(create_task)
with eventlet.Timeout(self.timeout_mins * 60) as tmo:
try:
create()
except exception.ResourceFailure as ex:
stack_status = self.CREATE_FAILED
reason = 'Resource failed: %s' % str(ex)
except eventlet.Timeout as t:
if t is not tmo: # not my timeout
raise
stack_status = self.CREATE_FAILED
reason = 'Timed out'
self.state_set(stack_status, reason)
if stack_status == self.CREATE_FAILED and not self.disable_rollback:
self.delete(action=self.ROLLBACK)
def _dep_test(self, *edges):
dummy = DummyTask()
class TaskMaker(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return 'Dummy task "%s"' % self.name
def __call__(self, *args, **kwargs):
return dummy(self.name, *args, **kwargs)
deps = dependencies.Dependencies(edges)
tg = scheduler.DependencyTaskGroup(deps, TaskMaker)
self.mox.StubOutWithMock(dummy, 'do_step')
yield dummy
self.mox.ReplayAll()
scheduler.TaskRunner(tg)(wait_time=None)
self.mox.VerifyAll()
def delete(self, action=DELETE, backup=False, abandon=False):
'''
Delete all of the resources, and then the stack itself.
The action parameter is used to differentiate between a user
initiated delete and an automatic stack rollback after a failed
create, which amount to the same thing, but the states are recorded
differently.
Note abandon is a delete where all resources have been set to a
RETAIN deletion policy, but we also don't want to delete anything
required for those resources, e.g the stack_user_project.
'''
if action not in (self.DELETE, self.ROLLBACK):
LOG.error(_LE("Unexpected action %s passed to delete!"), action)
self.state_set(self.DELETE, self.FAILED,
"Invalid action %s" % action)
return
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
backup_stack = self._backup_stack(False)
if backup_stack:
self._delete_backup_stack(backup_stack)
if backup_stack.status != backup_stack.COMPLETE:
errs = backup_stack.status_reason
failure = 'Error deleting backup resources: %s' % errs
self.state_set(action, self.FAILED,
'Failed to %s : %s' % (action, failure))
return
snapshots = db_api.snapshot_get_all(self.context, self.id)
for snapshot in snapshots:
self.delete_snapshot(snapshot)
if not backup:
try:
lifecycle_plugin_utils.do_pre_ops(self.context, self, None,
action)
except Exception as e:
self.state_set(
action, self.FAILED,
e.args[0] if e.args else 'Failed stack pre-ops: %s' %
six.text_type(e))
return
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource.Resource.destroy,
reverse=True)
try:
scheduler.TaskRunner(action_task)(timeout=self.timeout_secs())
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action, six.text_type(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
# If the stack delete succeeded, this is not a backup stack and it's
# not a nested stack, we should delete the credentials
if stack_status != self.FAILED and not backup and not self.owner_id:
stack_status, reason = self._delete_credentials(
stack_status, reason, abandon)
try:
self.state_set(action, stack_status, reason)
except exception.NotFound:
LOG.info(_LI("Tried to delete stack that does not exist "
"%s "), self.id)
if not backup:
lifecycle_plugin_utils.do_post_ops(self.context, self, None,
action,
(self.status == self.FAILED))
if stack_status != self.FAILED:
# delete the stack
try:
db_api.stack_delete(self.context, self.id)
except exception.NotFound:
LOG.info(
_LI("Tried to delete stack that does not exist "
"%s "), self.id)
self.id = None
def delete(self, action=DELETE, backup=False):
'''
Delete all of the resources, and then the stack itself.
The action parameter is used to differentiate between a user
initiated delete and an automatic stack rollback after a failed
create, which amount to the same thing, but the states are recorded
differently.
'''
if action not in (self.DELETE, self.ROLLBACK):
LOG.error(_("Unexpected action %s passed to delete!") % action)
self.state_set(self.DELETE, self.FAILED,
"Invalid action %s" % action)
return
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
backup_stack = self._backup_stack(False)
if backup_stack:
for key, backup_resource in backup_stack.resources.items():
# If UpdateReplace is failed, we must restore backup_resource
# to existing_stack in case of it may have dependencies in
# these stacks. current_resource is the resource that just
# created and failed, so put into the backup_stack to delete
# anyway.
backup_resource_id = backup_resource.resource_id
current_resource = self.resources[key]
current_resource_id = current_resource.resource_id
if backup_resource_id:
child_failed = False
for child in self.dependencies[current_resource]:
# If child resource failed to update, current_resource
# should be replaced to resolve dependencies. But this
# is not fundamental solution. If there are update
# failer and success resources in the children, cannot
# delete the stack.
if (child.status == child.FAILED
and child.action == child.CREATE):
child_failed = True
if (current_resource.status == current_resource.FAILED
or child_failed):
# Stack class owns dependencies as set of resource's
# objects, so we switch members of the resource that is
# needed to delete it.
self.resources[key].resource_id = backup_resource_id
self.resources[
key].properties = backup_resource.properties
backup_stack.resources[
key].resource_id = current_resource_id
backup_stack.resources[
key].properties = current_resource.properties
backup_stack.delete(backup=True)
if backup_stack.status != backup_stack.COMPLETE:
errs = backup_stack.status_reason
failure = 'Error deleting backup resources: %s' % errs
self.state_set(action, self.FAILED,
'Failed to %s : %s' % (action, failure))
return
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource.Resource.destroy,
reverse=True)
try:
scheduler.TaskRunner(action_task)(timeout=self.timeout_secs())
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action, six.text_type(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
if stack_status != self.FAILED and not backup:
# Cleanup stored user_creds so they aren't accessible via
# the soft-deleted stack which remains in the DB
if self.user_creds_id:
user_creds = db_api.user_creds_get(self.user_creds_id)
# If we created a trust, delete it
if user_creds is not None:
trust_id = user_creds.get('trust_id')
if trust_id:
try:
self.clients.client('keystone').delete_trust(
trust_id)
except Exception as ex:
LOG.exception(ex)
stack_status = self.FAILED
reason = ("Error deleting trust: %s" %
six.text_type(ex))
# Delete the stored credentials
try:
db_api.user_creds_delete(self.context, self.user_creds_id)
except exception.NotFound:
LOG.info(
_("Tried to delete user_creds that do not exist "
"(stack=%(stack)s user_creds_id=%(uc)s)") % {
'stack': self.id,
'uc': self.user_creds_id
})
try:
self.user_creds_id = None
self.store()
except exception.NotFound:
LOG.info(
_("Tried to store a stack that does not exist "
"%s ") % self.id)
# If the stack has a domain project, delete it
if self.stack_user_project_id:
try:
keystone = self.clients.client('keystone')
keystone.delete_stack_domain_project(
project_id=self.stack_user_project_id)
except Exception as ex:
LOG.exception(ex)
stack_status = self.FAILED
reason = "Error deleting project: %s" % six.text_type(ex)
try:
self.state_set(action, stack_status, reason)
except exception.NotFound:
LOG.info(
_("Tried to delete stack that does not exist "
"%s ") % self.id)
if stack_status != self.FAILED:
# delete the stack
try:
db_api.stack_delete(self.context, self.id)
except exception.NotFound:
LOG.info(
_("Tried to delete stack that does not exist "
"%s ") % self.id)
self.id = None
def delete(self, action=DELETE, backup=False, abandon=False):
'''
Delete all of the resources, and then the stack itself.
The action parameter is used to differentiate between a user
initiated delete and an automatic stack rollback after a failed
create, which amount to the same thing, but the states are recorded
differently.
Note abandon is a delete where all resources have been set to a
RETAIN deletion policy, but we also don't want to delete anything
required for those resources, e.g the stack_user_project.
'''
if action not in (self.DELETE, self.ROLLBACK):
LOG.error(_("Unexpected action %s passed to delete!") % action)
self.state_set(self.DELETE, self.FAILED,
"Invalid action %s" % action)
return
stack_status = self.COMPLETE
reason = 'Stack %s completed successfully' % action
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
backup_stack = self._backup_stack(False)
if backup_stack:
def failed(child):
return (child.action == child.CREATE
and child.status in (child.FAILED, child.IN_PROGRESS))
for key, backup_resource in backup_stack.resources.items():
# If UpdateReplace is failed, we must restore backup_resource
# to existing_stack in case of it may have dependencies in
# these stacks. current_resource is the resource that just
# created and failed, so put into the backup_stack to delete
# anyway.
backup_resource_id = backup_resource.resource_id
current_resource = self.resources[key]
current_resource_id = current_resource.resource_id
if backup_resource_id:
if (any(
failed(child)
for child in self.dependencies[current_resource])
or current_resource.status
in (current_resource.FAILED,
current_resource.IN_PROGRESS)):
# If child resource failed to update, current_resource
# should be replaced to resolve dependencies. But this
# is not fundamental solution. If there are update
# failer and success resources in the children, cannot
# delete the stack.
# Stack class owns dependencies as set of resource's
# objects, so we switch members of the resource that is
# needed to delete it.
self.resources[key].resource_id = backup_resource_id
self.resources[
key].properties = backup_resource.properties
backup_stack.resources[
key].resource_id = current_resource_id
backup_stack.resources[
key].properties = current_resource.properties
backup_stack.delete(backup=True)
if backup_stack.status != backup_stack.COMPLETE:
errs = backup_stack.status_reason
failure = 'Error deleting backup resources: %s' % errs
self.state_set(action, self.FAILED,
'Failed to %s : %s' % (action, failure))
return
snapshots = db_api.snapshot_get_all(self.context, self.id)
for snapshot in snapshots:
self.delete_snapshot(snapshot)
if not backup:
try:
lifecycle_plugin_utils.do_pre_ops(self.context, self, None,
action)
except Exception as e:
self.state_set(
action, self.FAILED,
e.args[0] if e.args else 'Failed stack pre-ops: %s' %
six.text_type(e))
return
action_task = scheduler.DependencyTaskGroup(self.dependencies,
resource.Resource.destroy,
reverse=True)
try:
scheduler.TaskRunner(action_task)(timeout=self.timeout_secs())
except exception.ResourceFailure as ex:
stack_status = self.FAILED
reason = 'Resource %s failed: %s' % (action, six.text_type(ex))
except scheduler.Timeout:
stack_status = self.FAILED
reason = '%s timed out' % action.title()
# If the stack delete succeeded, this is not a backup stack and it's
# not a nested stack, we should delete the credentials
if stack_status != self.FAILED and not backup and not self.owner_id:
# Cleanup stored user_creds so they aren't accessible via
# the soft-deleted stack which remains in the DB
if self.user_creds_id:
user_creds = db_api.user_creds_get(self.user_creds_id)
# If we created a trust, delete it
if user_creds is not None:
trust_id = user_creds.get('trust_id')
if trust_id:
try:
# If the trustor doesn't match the context user the
# we have to use the stored context to cleanup the
# trust, as although the user evidently has
# permission to delete the stack, they don't have
# rights to delete the trust unless an admin
trustor_id = user_creds.get('trustor_user_id')
if self.context.user_id != trustor_id:
LOG.debug('Context user_id doesn\'t match '
'trustor, using stored context')
sc = self.stored_context()
sc.clients.client('keystone').delete_trust(
trust_id)
else:
self.clients.client('keystone').delete_trust(
trust_id)
except Exception as ex:
LOG.exception(ex)
stack_status = self.FAILED
reason = ("Error deleting trust: %s" %
six.text_type(ex))
# Delete the stored credentials
try:
db_api.user_creds_delete(self.context, self.user_creds_id)
except exception.NotFound:
LOG.info(
_("Tried to delete user_creds that do not exist "
"(stack=%(stack)s user_creds_id=%(uc)s)") % {
'stack': self.id,
'uc': self.user_creds_id
})
try:
self.user_creds_id = None
self.store()
except exception.NotFound:
LOG.info(
_("Tried to store a stack that does not exist "
"%s ") % self.id)
# If the stack has a domain project, delete it
if self.stack_user_project_id and not abandon:
try:
keystone = self.clients.client('keystone')
keystone.delete_stack_domain_project(
project_id=self.stack_user_project_id)
except Exception as ex:
LOG.exception(ex)
stack_status = self.FAILED
reason = "Error deleting project: %s" % six.text_type(ex)
try:
self.state_set(action, stack_status, reason)
except exception.NotFound:
LOG.info(
_("Tried to delete stack that does not exist "
"%s ") % self.id)
if not backup:
lifecycle_plugin_utils.do_post_ops(self.context, self, None,
action,
(self.status == self.FAILED))
if stack_status != self.FAILED:
# delete the stack
try:
db_api.stack_delete(self.context, self.id)
except exception.NotFound:
LOG.info(
_("Tried to delete stack that does not exist "
"%s ") % self.id)
self.id = None
