def __call__(self): """Return a co-routine that updates the stack.""" existing_deps = self.existing_stack.dependencies new_deps = self.new_stack.dependencies cleanup_prev = scheduler.DependencyTaskGroup( self.previous_stack.dependencies, self._remove_backup_resource, reverse=True) cleanup = scheduler.DependencyTaskGroup(existing_deps, self._remove_old_resource, reverse=True) create_new = scheduler.DependencyTaskGroup(new_deps, self._create_new_resource) update = scheduler.DependencyTaskGroup(new_deps, self._update_resource) if not self.rollback: yield cleanup_prev() yield create_new() try: yield update() finally: prev_deps = self.previous_stack._get_dependencies( self.previous_stack.resources.itervalues()) self.previous_stack.dependencies = prev_deps yield cleanup()
def stack_task(self, action, reverse=False, post_func=None): ''' A task to perform an action on the stack and all of the resources in forward or reverse dependency order as specfifed by reverse ''' self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action.lower() res = None def resource_action(r): # Find e.g resource.create and call it action_l = action.lower() handle = getattr(r, '%s' % action_l) return handle() action_task = scheduler.DependencyTaskGroup(self.dependencies, resource_action, reverse) try: yield action_task() except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action.lower(), str(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() self.state_set(action, stack_status, reason) if callable(post_func): post_func()
def create_task(self): ''' A task to create the stack and all of the resources. ''' self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started') stack_status = self.CREATE_COMPLETE reason = 'Stack successfully created' res = None def resource_create(r): return r.create create_task = scheduler.DependencyTaskGroup(self.dependencies, resource_create) try: yield create_task() except exception.ResourceFailure as ex: stack_status = self.CREATE_FAILED reason = 'Resource failed: %s' % str(ex) except scheduler.Timeout: stack_status = self.CREATE_FAILED reason = 'Timed out' self.state_set(stack_status, reason) if stack_status == self.CREATE_FAILED and not self.disable_rollback: self.delete(action=self.ROLLBACK)
def __call__(self): """Return a co-routine that updates the stack.""" existing_deps = self.existing_stack.dependencies new_deps = self.new_stack.dependencies cleanup = scheduler.DependencyTaskGroup(existing_deps, self._remove_old_resource, reverse=True) create_new = scheduler.DependencyTaskGroup(new_deps, self._create_new_resource) update = scheduler.DependencyTaskGroup(new_deps, self._update_resource) yield cleanup() yield create_new() yield update()
def delete(self, action=DELETE): ''' Delete all of the resources, and then the stack itself. The action parameter is used to differentiate between a user initiated delete and an automatic stack rollback after a failed create, which amount to the same thing, but the states are recorded differently. ''' if action not in (self.DELETE, self.ROLLBACK): logger.error(_("Unexpected action %s passed to delete!") % action) self.state_set(self.DELETE, self.FAILED, "Invalid action %s" % action) return stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action.lower() self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) backup_stack = self._backup_stack(False) if backup_stack is not None: backup_stack.delete() if backup_stack.status != backup_stack.COMPLETE: errs = backup_stack.status_reason failure = 'Error deleting backup resources: %s' % errs self.state_set(action, self.FAILED, 'Failed to %s : %s' % (action, failure)) return action_task = scheduler.DependencyTaskGroup(self.dependencies, resource.Resource.destroy, reverse=True) try: scheduler.TaskRunner(action_task)(timeout=self.timeout_secs()) except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action.lower(), str(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() if stack_status != self.FAILED: # If we created a trust, delete it stack = db_api.stack_get(self.context, self.id) user_creds = db_api.user_creds_get(stack.user_creds_id) trust_id = user_creds.get('trust_id') if trust_id: try: self.clients.keystone().delete_trust(trust_id) except Exception as ex: logger.exception(ex) stack_status = self.FAILED reason = "Error deleting trust: %s" % str(ex) self.state_set(action, stack_status, reason) if stack_status != self.FAILED: # delete the stack db_api.stack_delete(self.context, self.id) self.id = None
def stack_task(self, action, reverse=False, post_func=None, error_wait_time=None, aggregate_exceptions=False): ''' A task to perform an action on the stack and all of the resources in forward or reverse dependency order as specified by reverse ''' try: lifecycle_plugin_utils.do_pre_ops(self.context, self, None, action) except Exception as e: self.state_set( action, self.FAILED, e.args[0] if e.args else 'Failed stack pre-ops: %s' % six.text_type(e)) if callable(post_func): post_func() return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action def resource_action(r): # Find e.g resource.create and call it action_l = action.lower() handle = getattr(r, '%s' % action_l) # If a local _$action_kwargs function exists, call it to get the # action specific argument list, otherwise an empty arg list handle_kwargs = getattr(self, '_%s_kwargs' % action_l, lambda x: {}) return handle(**handle_kwargs(r)) action_task = scheduler.DependencyTaskGroup( self.dependencies, resource_action, reverse, error_wait_time=error_wait_time, aggregate_exceptions=aggregate_exceptions) try: yield action_task() except (exception.ResourceFailure, scheduler.ExceptionGroup) as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() self.state_set(action, stack_status, reason) if callable(post_func): post_func() lifecycle_plugin_utils.do_post_ops(self.context, self, None, action, (self.status == self.FAILED))
def __call__(self): """Return a co-routine that updates the stack.""" cleanup_prev = scheduler.DependencyTaskGroup( self.previous_stack.dependencies, self._remove_backup_resource, reverse=True) update = scheduler.DependencyTaskGroup(self.dependencies(), self._resource_update) if not self.rollback: yield cleanup_prev() try: yield update() finally: self.previous_stack.reset_dependencies()
def __call__(self): """Return a co-routine that updates the stack.""" cleanup_prev = scheduler.DependencyTaskGroup( self.previous_stack.dependencies, self._remove_backup_resource, reverse=True) def get_error_wait_time(resource): return resource.cancel_grace_period() updater = scheduler.DependencyTaskGroup( self.dependencies(), self._resource_update, error_wait_time=get_error_wait_time) if not self.rollback: yield cleanup_prev() try: yield updater() finally: self.previous_stack.reset_dependencies()
def _dep_test(self, *edges): dummy = DummyTask(getattr(self, 'steps', 3)) deps = dependencies.Dependencies(edges) tg = scheduler.DependencyTaskGroup(deps, dummy) self.mox.StubOutWithMock(dummy, 'do_step') yield dummy self.mox.ReplayAll() scheduler.TaskRunner(tg)(wait_time=None) self.mox.VerifyAll()
def _dep_test(self, *edges): dummy = DummyTask(getattr(self, 'steps', 3)) deps = dependencies.Dependencies(edges) tg = scheduler.DependencyTaskGroup( deps, dummy, aggregate_exceptions=self.aggregate_exceptions, reverse=self.reverse_order) self.m.StubOutWithMock(dummy, 'do_step') yield dummy self.m.ReplayAll() scheduler.TaskRunner(tg)(wait_time=None)
def stack_task(self, action, reverse=False, post_func=None): ''' A task to perform an action on the stack and all of the resources in forward or reverse dependency order as specfifed by reverse ''' self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action def resource_action(r): # Find e.g resource.create and call it action_l = action.lower() handle = getattr(r, '%s' % action_l) # If a local _$action_kwargs function exists, call it to get the # action specific argument list, otherwise an empty arg list handle_kwargs = getattr(self, '_%s_kwargs' % action_l, lambda x: {}) return handle(**handle_kwargs(r)) action_task = scheduler.DependencyTaskGroup(self.dependencies, resource_action, reverse) try: yield action_task() except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() self.state_set(action, stack_status, reason) if callable(post_func): post_func()
def create(self): ''' Create the stack and all of the resources. Creation will fail if it exceeds the specified timeout. The default is 60 minutes, set in the constructor ''' self.state_set(self.CREATE_IN_PROGRESS, 'Stack creation started') stack_status = self.CREATE_COMPLETE reason = 'Stack successfully created' res = None def resource_create(r): return r.create create_task = scheduler.DependencyTaskGroup(self.dependencies, resource_create) create = scheduler.TaskRunner(create_task) with eventlet.Timeout(self.timeout_mins * 60) as tmo: try: create() except exception.ResourceFailure as ex: stack_status = self.CREATE_FAILED reason = 'Resource failed: %s' % str(ex) except eventlet.Timeout as t: if t is not tmo: # not my timeout raise stack_status = self.CREATE_FAILED reason = 'Timed out' self.state_set(stack_status, reason) if stack_status == self.CREATE_FAILED and not self.disable_rollback: self.delete(action=self.ROLLBACK)
def _dep_test(self, *edges): dummy = DummyTask() class TaskMaker(object): def __init__(self, name): self.name = name def __repr__(self): return 'Dummy task "%s"' % self.name def __call__(self, *args, **kwargs): return dummy(self.name, *args, **kwargs) deps = dependencies.Dependencies(edges) tg = scheduler.DependencyTaskGroup(deps, TaskMaker) self.mox.StubOutWithMock(dummy, 'do_step') yield dummy self.mox.ReplayAll() scheduler.TaskRunner(tg)(wait_time=None) self.mox.VerifyAll()
def delete(self, action=DELETE, backup=False, abandon=False): ''' Delete all of the resources, and then the stack itself. The action parameter is used to differentiate between a user initiated delete and an automatic stack rollback after a failed create, which amount to the same thing, but the states are recorded differently. Note abandon is a delete where all resources have been set to a RETAIN deletion policy, but we also don't want to delete anything required for those resources, e.g the stack_user_project. ''' if action not in (self.DELETE, self.ROLLBACK): LOG.error(_LE("Unexpected action %s passed to delete!"), action) self.state_set(self.DELETE, self.FAILED, "Invalid action %s" % action) return stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) backup_stack = self._backup_stack(False) if backup_stack: self._delete_backup_stack(backup_stack) if backup_stack.status != backup_stack.COMPLETE: errs = backup_stack.status_reason failure = 'Error deleting backup resources: %s' % errs self.state_set(action, self.FAILED, 'Failed to %s : %s' % (action, failure)) return snapshots = db_api.snapshot_get_all(self.context, self.id) for snapshot in snapshots: self.delete_snapshot(snapshot) if not backup: try: lifecycle_plugin_utils.do_pre_ops(self.context, self, None, action) except Exception as e: self.state_set( action, self.FAILED, e.args[0] if e.args else 'Failed stack pre-ops: %s' % six.text_type(e)) return action_task = scheduler.DependencyTaskGroup(self.dependencies, resource.Resource.destroy, reverse=True) try: scheduler.TaskRunner(action_task)(timeout=self.timeout_secs()) except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() # If the stack delete succeeded, this is not a backup stack and it's # not a nested stack, we should delete the credentials if stack_status != self.FAILED and not backup and not self.owner_id: stack_status, reason = self._delete_credentials( stack_status, reason, abandon) try: self.state_set(action, stack_status, reason) except exception.NotFound: LOG.info(_LI("Tried to delete stack that does not exist " "%s "), self.id) if not backup: lifecycle_plugin_utils.do_post_ops(self.context, self, None, action, (self.status == self.FAILED)) if stack_status != self.FAILED: # delete the stack try: db_api.stack_delete(self.context, self.id) except exception.NotFound: LOG.info( _LI("Tried to delete stack that does not exist " "%s "), self.id) self.id = None
def delete(self, action=DELETE, backup=False): ''' Delete all of the resources, and then the stack itself. The action parameter is used to differentiate between a user initiated delete and an automatic stack rollback after a failed create, which amount to the same thing, but the states are recorded differently. ''' if action not in (self.DELETE, self.ROLLBACK): LOG.error(_("Unexpected action %s passed to delete!") % action) self.state_set(self.DELETE, self.FAILED, "Invalid action %s" % action) return stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) backup_stack = self._backup_stack(False) if backup_stack: for key, backup_resource in backup_stack.resources.items(): # If UpdateReplace is failed, we must restore backup_resource # to existing_stack in case of it may have dependencies in # these stacks. current_resource is the resource that just # created and failed, so put into the backup_stack to delete # anyway. backup_resource_id = backup_resource.resource_id current_resource = self.resources[key] current_resource_id = current_resource.resource_id if backup_resource_id: child_failed = False for child in self.dependencies[current_resource]: # If child resource failed to update, current_resource # should be replaced to resolve dependencies. But this # is not fundamental solution. If there are update # failer and success resources in the children, cannot # delete the stack. if (child.status == child.FAILED and child.action == child.CREATE): child_failed = True if (current_resource.status == current_resource.FAILED or child_failed): # Stack class owns dependencies as set of resource's # objects, so we switch members of the resource that is # needed to delete it. self.resources[key].resource_id = backup_resource_id self.resources[ key].properties = backup_resource.properties backup_stack.resources[ key].resource_id = current_resource_id backup_stack.resources[ key].properties = current_resource.properties backup_stack.delete(backup=True) if backup_stack.status != backup_stack.COMPLETE: errs = backup_stack.status_reason failure = 'Error deleting backup resources: %s' % errs self.state_set(action, self.FAILED, 'Failed to %s : %s' % (action, failure)) return action_task = scheduler.DependencyTaskGroup(self.dependencies, resource.Resource.destroy, reverse=True) try: scheduler.TaskRunner(action_task)(timeout=self.timeout_secs()) except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() if stack_status != self.FAILED and not backup: # Cleanup stored user_creds so they aren't accessible via # the soft-deleted stack which remains in the DB if self.user_creds_id: user_creds = db_api.user_creds_get(self.user_creds_id) # If we created a trust, delete it if user_creds is not None: trust_id = user_creds.get('trust_id') if trust_id: try: self.clients.client('keystone').delete_trust( trust_id) except Exception as ex: LOG.exception(ex) stack_status = self.FAILED reason = ("Error deleting trust: %s" % six.text_type(ex)) # Delete the stored credentials try: db_api.user_creds_delete(self.context, self.user_creds_id) except exception.NotFound: LOG.info( _("Tried to delete user_creds that do not exist " "(stack=%(stack)s user_creds_id=%(uc)s)") % { 'stack': self.id, 'uc': self.user_creds_id }) try: self.user_creds_id = None self.store() except exception.NotFound: LOG.info( _("Tried to store a stack that does not exist " "%s ") % self.id) # If the stack has a domain project, delete it if self.stack_user_project_id: try: keystone = self.clients.client('keystone') keystone.delete_stack_domain_project( project_id=self.stack_user_project_id) except Exception as ex: LOG.exception(ex) stack_status = self.FAILED reason = "Error deleting project: %s" % six.text_type(ex) try: self.state_set(action, stack_status, reason) except exception.NotFound: LOG.info( _("Tried to delete stack that does not exist " "%s ") % self.id) if stack_status != self.FAILED: # delete the stack try: db_api.stack_delete(self.context, self.id) except exception.NotFound: LOG.info( _("Tried to delete stack that does not exist " "%s ") % self.id) self.id = None
def delete(self, action=DELETE, backup=False, abandon=False): ''' Delete all of the resources, and then the stack itself. The action parameter is used to differentiate between a user initiated delete and an automatic stack rollback after a failed create, which amount to the same thing, but the states are recorded differently. Note abandon is a delete where all resources have been set to a RETAIN deletion policy, but we also don't want to delete anything required for those resources, e.g the stack_user_project. ''' if action not in (self.DELETE, self.ROLLBACK): LOG.error(_("Unexpected action %s passed to delete!") % action) self.state_set(self.DELETE, self.FAILED, "Invalid action %s" % action) return stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) backup_stack = self._backup_stack(False) if backup_stack: def failed(child): return (child.action == child.CREATE and child.status in (child.FAILED, child.IN_PROGRESS)) for key, backup_resource in backup_stack.resources.items(): # If UpdateReplace is failed, we must restore backup_resource # to existing_stack in case of it may have dependencies in # these stacks. current_resource is the resource that just # created and failed, so put into the backup_stack to delete # anyway. backup_resource_id = backup_resource.resource_id current_resource = self.resources[key] current_resource_id = current_resource.resource_id if backup_resource_id: if (any( failed(child) for child in self.dependencies[current_resource]) or current_resource.status in (current_resource.FAILED, current_resource.IN_PROGRESS)): # If child resource failed to update, current_resource # should be replaced to resolve dependencies. But this # is not fundamental solution. If there are update # failer and success resources in the children, cannot # delete the stack. # Stack class owns dependencies as set of resource's # objects, so we switch members of the resource that is # needed to delete it. self.resources[key].resource_id = backup_resource_id self.resources[ key].properties = backup_resource.properties backup_stack.resources[ key].resource_id = current_resource_id backup_stack.resources[ key].properties = current_resource.properties backup_stack.delete(backup=True) if backup_stack.status != backup_stack.COMPLETE: errs = backup_stack.status_reason failure = 'Error deleting backup resources: %s' % errs self.state_set(action, self.FAILED, 'Failed to %s : %s' % (action, failure)) return snapshots = db_api.snapshot_get_all(self.context, self.id) for snapshot in snapshots: self.delete_snapshot(snapshot) if not backup: try: lifecycle_plugin_utils.do_pre_ops(self.context, self, None, action) except Exception as e: self.state_set( action, self.FAILED, e.args[0] if e.args else 'Failed stack pre-ops: %s' % six.text_type(e)) return action_task = scheduler.DependencyTaskGroup(self.dependencies, resource.Resource.destroy, reverse=True) try: scheduler.TaskRunner(action_task)(timeout=self.timeout_secs()) except exception.ResourceFailure as ex: stack_status = self.FAILED reason = 'Resource %s failed: %s' % (action, six.text_type(ex)) except scheduler.Timeout: stack_status = self.FAILED reason = '%s timed out' % action.title() # If the stack delete succeeded, this is not a backup stack and it's # not a nested stack, we should delete the credentials if stack_status != self.FAILED and not backup and not self.owner_id: # Cleanup stored user_creds so they aren't accessible via # the soft-deleted stack which remains in the DB if self.user_creds_id: user_creds = db_api.user_creds_get(self.user_creds_id) # If we created a trust, delete it if user_creds is not None: trust_id = user_creds.get('trust_id') if trust_id: try: # If the trustor doesn't match the context user the # we have to use the stored context to cleanup the # trust, as although the user evidently has # permission to delete the stack, they don't have # rights to delete the trust unless an admin trustor_id = user_creds.get('trustor_user_id') if self.context.user_id != trustor_id: LOG.debug('Context user_id doesn\'t match ' 'trustor, using stored context') sc = self.stored_context() sc.clients.client('keystone').delete_trust( trust_id) else: self.clients.client('keystone').delete_trust( trust_id) except Exception as ex: LOG.exception(ex) stack_status = self.FAILED reason = ("Error deleting trust: %s" % six.text_type(ex)) # Delete the stored credentials try: db_api.user_creds_delete(self.context, self.user_creds_id) except exception.NotFound: LOG.info( _("Tried to delete user_creds that do not exist " "(stack=%(stack)s user_creds_id=%(uc)s)") % { 'stack': self.id, 'uc': self.user_creds_id }) try: self.user_creds_id = None self.store() except exception.NotFound: LOG.info( _("Tried to store a stack that does not exist " "%s ") % self.id) # If the stack has a domain project, delete it if self.stack_user_project_id and not abandon: try: keystone = self.clients.client('keystone') keystone.delete_stack_domain_project( project_id=self.stack_user_project_id) except Exception as ex: LOG.exception(ex) stack_status = self.FAILED reason = "Error deleting project: %s" % six.text_type(ex) try: self.state_set(action, stack_status, reason) except exception.NotFound: LOG.info( _("Tried to delete stack that does not exist " "%s ") % self.id) if not backup: lifecycle_plugin_utils.do_post_ops(self.context, self, None, action, (self.status == self.FAILED)) if stack_status != self.FAILED: # delete the stack try: db_api.stack_delete(self.context, self.id) except exception.NotFound: LOG.info( _("Tried to delete stack that does not exist " "%s ") % self.id) self.id = None