def state_set(self, action, status, reason): '''Update the stack state in the database.''' if action not in self.ACTIONS: raise ValueError(_("Invalid action %s") % action) if status not in self.STATUSES: raise ValueError(_("Invalid status %s") % status) self.action = action self.status = status self.status_reason = reason if self.id is None: return stack = db_api.stack_get(self.context, self.id) if stack is not None: stack.update_and_save({ 'action': action, 'status': status, 'status_reason': reason }) LOG.info( _LI('Stack %(action)s %(status)s (%(name)s): ' '%(reason)s'), { 'action': action, 'status': status, 'name': self.name, 'reason': reason }) notification.send(self)
def _stub_grp_replace(self, num_creates_expected_on_updt=0, num_deletes_expected_on_updt=0, num_reloads_expected_on_updt=0): """ Expect replacement of the capacity by batch size """ # for load balancer setup self._stub_lb_reload(num_reloads_expected_on_updt) self.m.StubOutWithMock(notification, 'send') notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None) # for instances in the group self.m.StubOutWithMock(clients.OpenStackClients, 'nova') self.m.StubOutWithMock(instance.Instance, 'handle_create') self.m.StubOutWithMock(instance.Instance, 'check_create_complete') self.m.StubOutWithMock(instance.Instance, 'destroy') if num_reloads_expected_on_updt > 1: clients.OpenStackClients.nova().MultipleTimes().AndReturn(self.fc) cookie = object() for i in range(num_creates_expected_on_updt): instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True) for i in range(num_deletes_expected_on_updt): instance.Instance.destroy().AndReturn(None)
def state_set(self, action, status, reason): '''Update the stack state in the database.''' if action not in self.ACTIONS: raise ValueError(_("Invalid action %s") % action) if status not in self.STATUSES: raise ValueError(_("Invalid status %s") % status) self.action = action self.status = status self.status_reason = reason if self.id is None: return stack = db_api.stack_get(self.context, self.id) if stack is not None: stack.update_and_save({'action': action, 'status': status, 'status_reason': reason}) msg = _('Stack %(action)s %(status)s (%(name)s): %(reason)s') LOG.info(msg % {'action': action, 'status': status, 'name': self.name, 'reason': reason}) notification.send(self)
def _stub_grp_create(self, capacity=0, setup_lb=True): """ Expect creation of instances to capacity. By default, expect creation of load balancer unless specified. """ self._stub_validate() self.m.StubOutWithMock(clients.OpenStackClients, 'nova') self.m.StubOutWithMock(instance.Instance, 'handle_create') self.m.StubOutWithMock(instance.Instance, 'check_create_complete') self.m.StubOutWithMock(notification, 'send') notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None) cookie = object() clients.OpenStackClients.nova().MultipleTimes().AndReturn(self.fc) # for load balancer setup if setup_lb: self._stub_lb_create() self._stub_lb_reload() instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True) # for each instance in group for i in range(capacity): instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True)
def _stub_grp_replace(self, num_creates_expected_on_updt=0, num_deletes_expected_on_updt=0, num_reloads_expected_on_updt=0): """ Expect replacement of the capacity by batch size """ # for load balancer setup self._stub_lb_reload(num_reloads_expected_on_updt) self.m.StubOutWithMock(notification, 'send') notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None) # for instances in the group self.m.StubOutWithMock(clients.OpenStackClients, '_nova') self.m.StubOutWithMock(instance.Instance, 'handle_create') self.m.StubOutWithMock(instance.Instance, 'check_create_complete') self.m.StubOutWithMock(instance.Instance, 'destroy') if num_reloads_expected_on_updt > 1: clients.OpenStackClients._nova().MultipleTimes().AndReturn(self.fc) cookie = object() for i in range(num_creates_expected_on_updt): instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True) for i in range(num_deletes_expected_on_updt): instance.Instance.destroy().AndReturn(None)
def _stub_grp_create(self, capacity=0, setup_lb=True): """ Expect creation of instances to capacity. By default, expect creation of load balancer unless specified. """ self._stub_validate() self.m.StubOutWithMock(clients.OpenStackClients, '_nova') self.m.StubOutWithMock(instance.Instance, 'handle_create') self.m.StubOutWithMock(instance.Instance, 'check_create_complete') self.m.StubOutWithMock(notification, 'send') notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None) cookie = object() clients.OpenStackClients._nova().MultipleTimes().AndReturn(self.fc) # for load balancer setup if setup_lb: self._stub_lb_create() self._stub_lb_reload() instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True) # for each instance in group for i in range(capacity): instance.Instance.handle_create().AndReturn(cookie) instance.Instance.check_create_complete(cookie).AndReturn(True)
def state_set(self, action, status, reason): """Update the stack state in the database.""" if action not in self.ACTIONS: raise ValueError(_("Invalid action %s") % action) if status not in self.STATUSES: raise ValueError(_("Invalid status %s") % status) self.action = action self.status = status self.status_reason = reason if self.id is None: return stack = db_api.stack_get(self.context, self.id) if stack is not None: stack.update_and_save({"action": action, "status": status, "status_reason": reason}) msg = _("Stack %(action)s %(status)s (%(name)s): %(reason)s") LOG.info(msg % {"action": action, "status": status, "name": self.name, "reason": reason}) notification.send(self)
def update_task(self, newstack, action=UPDATE, event=None): if action not in (self.UPDATE, self.ROLLBACK, self.RESTORE): LOG.error(_LE("Unexpected action %s passed to update!"), action) self.state_set(self.UPDATE, self.FAILED, "Invalid action %s" % action) return try: lifecycle_plugin_utils.do_pre_ops(self.context, self, newstack, action) except Exception as e: self.state_set( action, self.FAILED, e.args[0] if e.args else 'Failed stack pre-ops: %s' % six.text_type(e)) return if self.status == self.IN_PROGRESS: if action == self.ROLLBACK: LOG.debug("Starting update rollback for %s" % self.name) else: self.state_set(action, self.FAILED, 'State invalid for %s' % action) return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) if action == self.UPDATE: # Oldstack is useless when the action is not UPDATE , so we don't # need to build it, this can avoid some unexpected errors. oldstack = Stack(self.context, self.name, copy.deepcopy(self.t), self.env) backup_stack = self._backup_stack() try: update_task = update.StackUpdate( self, newstack, backup_stack, rollback=action == self.ROLLBACK, error_wait_time=cfg.CONF.error_wait_time) updater = scheduler.TaskRunner(update_task) self.env = newstack.env self.parameters = newstack.parameters self.t.files = newstack.t.files self.disable_rollback = newstack.disable_rollback self.timeout_mins = newstack.timeout_mins self._set_param_stackid() try: updater.start(timeout=self.timeout_secs()) yield while not updater.step(): if event is None or not event.ready(): yield else: message = event.wait() if message == rpc_api.THREAD_CANCEL: raise ForcedCancel() finally: self.reset_dependencies() if action == self.UPDATE: reason = 'Stack successfully updated' elif action == self.RESTORE: reason = 'Stack successfully restored' else: reason = 'Stack rollback completed' stack_status = self.COMPLETE except scheduler.Timeout: stack_status = self.FAILED reason = 'Timed out' except ForcedCancel as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: update_task.updater.cancel_all() yield self.update_task(oldstack, action=self.ROLLBACK) return except exception.ResourceFailure as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: # If rollback is enabled, we do another update, with the # existing template, so we roll back to the original state if not self.disable_rollback: yield self.update_task(oldstack, action=self.ROLLBACK) return else: LOG.debug('Deleting backup stack') backup_stack.delete(backup=True) # flip the template to the newstack values self.t = newstack.t template_outputs = self.t[self.t.OUTPUTS] self.outputs = self.resolve_static_data(template_outputs) # Don't use state_set to do only one update query and avoid race # condition with the COMPLETE status self.action = action self.status = stack_status self.status_reason = reason self.store() lifecycle_plugin_utils.do_post_ops(self.context, self, newstack, action, (self.status == self.FAILED)) notification.send(self)
def update_task(self, newstack, action=UPDATE): if action not in (self.UPDATE, self.ROLLBACK): LOG.error(_("Unexpected action %s passed to update!") % action) self.state_set(self.UPDATE, self.FAILED, "Invalid action %s" % action) return if self.status != self.COMPLETE: if (action == self.ROLLBACK and self.state == (self.UPDATE, self.IN_PROGRESS)): LOG.debug("Starting update rollback for %s" % self.name) else: self.state_set(action, self.FAILED, 'State invalid for %s' % action) return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) oldstack = Stack(self.context, self.name, copy.deepcopy(self.t), self.env) backup_stack = self._backup_stack() try: update_task = update.StackUpdate(self, newstack, backup_stack, rollback=action == self.ROLLBACK) updater = scheduler.TaskRunner(update_task) self.env = newstack.env self.parameters = newstack.parameters self.t.files = newstack.t.files self.disable_rollback = newstack.disable_rollback self.timeout_mins = newstack.timeout_mins self._set_param_stackid() try: updater.start(timeout=self.timeout_secs()) yield while not updater.step(): yield finally: self.reset_dependencies() if action == self.UPDATE: reason = 'Stack successfully updated' else: reason = 'Stack rollback completed' stack_status = self.COMPLETE except scheduler.Timeout: stack_status = self.FAILED reason = 'Timed out' except exception.ResourceFailure as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: # If rollback is enabled, we do another update, with the # existing template, so we roll back to the original state if not self.disable_rollback: yield self.update_task(oldstack, action=self.ROLLBACK) return else: LOG.debug('Deleting backup stack') backup_stack.delete(backup=True) # flip the template to the newstack values self.t = newstack.t template_outputs = self.t[self.t.OUTPUTS] self.outputs = self.resolve_static_data(template_outputs) # Don't use state_set to do only one update query and avoid race # condition with the COMPLETE status self.action = action self.status = stack_status self.status_reason = reason self.store() notification.send(self)
def update_task(self, newstack, action=UPDATE): if action not in (self.UPDATE, self.ROLLBACK): LOG.error(_("Unexpected action %s passed to update!") % action) self.state_set(self.UPDATE, self.FAILED, "Invalid action %s" % action) return if self.status != self.COMPLETE: if (action == self.ROLLBACK and self.state == (self.UPDATE, self.IN_PROGRESS)): LOG.debug("Starting update rollback for %s" % self.name) else: self.state_set(action, self.FAILED, 'State invalid for %s' % action) return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) oldstack = Stack(self.context, self.name, self.t, self.env) backup_stack = self._backup_stack() try: update_task = update.StackUpdate(self, newstack, backup_stack, rollback=action == self.ROLLBACK) updater = scheduler.TaskRunner(update_task) self.env = newstack.env self.parameters = newstack.parameters self.t.files = newstack.t.files self.disable_rollback = newstack.disable_rollback self.timeout_mins = newstack.timeout_mins self._set_param_stackid() try: updater.start(timeout=self.timeout_secs()) yield while not updater.step(): yield finally: self.reset_dependencies() if action == self.UPDATE: reason = 'Stack successfully updated' else: reason = 'Stack rollback completed' stack_status = self.COMPLETE except scheduler.Timeout: stack_status = self.FAILED reason = 'Timed out' except exception.ResourceFailure as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: # If rollback is enabled, we do another update, with the # existing template, so we roll back to the original state if not self.disable_rollback: yield self.update_task(oldstack, action=self.ROLLBACK) return else: LOG.debug('Deleting backup stack') backup_stack.delete(backup=True) # flip the template to the newstack values # Note we do this on success and failure, so the current # stack resources are stored, even if one is in a failed # state (otherwise we won't remove them on delete) self.t = newstack.t template_outputs = self.t[self.t.OUTPUTS] self.outputs = self.resolve_static_data(template_outputs) # Don't use state_set to do only one update query and avoid race # condition with the COMPLETE status self.action = action self.status = stack_status self.status_reason = reason self.store() notification.send(self)
def update_task(self, newstack, action=UPDATE, event=None): if action not in (self.UPDATE, self.ROLLBACK): LOG.error(_("Unexpected action %s passed to update!") % action) self.state_set(self.UPDATE, self.FAILED, "Invalid action %s" % action) return try: lifecycle_plugin_utils.do_pre_ops(self.context, self, newstack, action) except Exception as e: self.state_set(action, self.FAILED, e.args[0] if e.args else 'Failed stack pre-ops: %s' % six.text_type(e)) return if self.status == self.IN_PROGRESS: if action == self.ROLLBACK: LOG.debug("Starting update rollback for %s" % self.name) else: self.state_set(action, self.FAILED, 'State invalid for %s' % action) return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) if action == self.UPDATE: # Oldstack is useless when the action is not UPDATE , so we don't # need to build it, this can avoid some unexpected errors. oldstack = Stack(self.context, self.name, copy.deepcopy(self.t), self.env) backup_stack = self._backup_stack() try: update_task = update.StackUpdate(self, newstack, backup_stack, rollback=action == self.ROLLBACK, error_wait_time=ERROR_WAIT_TIME) updater = scheduler.TaskRunner(update_task) self.env = newstack.env self.parameters = newstack.parameters self.t.files = newstack.t.files self.disable_rollback = newstack.disable_rollback self.timeout_mins = newstack.timeout_mins self._set_param_stackid() try: updater.start(timeout=self.timeout_secs()) yield while not updater.step(): if event is None or not event.ready(): yield else: message = event.wait() if message == rpc_api.THREAD_CANCEL: raise ForcedCancel() finally: self.reset_dependencies() if action == self.UPDATE: reason = 'Stack successfully updated' else: reason = 'Stack rollback completed' stack_status = self.COMPLETE except scheduler.Timeout: stack_status = self.FAILED reason = 'Timed out' except ForcedCancel as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: update_task.updater.cancel_all() yield self.update_task(oldstack, action=self.ROLLBACK) return except exception.ResourceFailure as e: reason = six.text_type(e) stack_status = self.FAILED if action == self.UPDATE: # If rollback is enabled, we do another update, with the # existing template, so we roll back to the original state if not self.disable_rollback: yield self.update_task(oldstack, action=self.ROLLBACK) return else: LOG.debug('Deleting backup stack') backup_stack.delete(backup=True) # flip the template to the newstack values self.t = newstack.t template_outputs = self.t[self.t.OUTPUTS] self.outputs = self.resolve_static_data(template_outputs) # Don't use state_set to do only one update query and avoid race # condition with the COMPLETE status self.action = action self.status = stack_status self.status_reason = reason self.store() lifecycle_plugin_utils.do_post_ops(self.context, self, newstack, action, (self.status == self.FAILED)) notification.send(self)