def state_set(self, action, status, reason):
'''Update the stack state in the database.'''
if action not in self.ACTIONS:
raise ValueError(_("Invalid action %s") % action)
if status not in self.STATUSES:
raise ValueError(_("Invalid status %s") % status)
self.action = action
self.status = status
self.status_reason = reason
if self.id is None:
return
stack = db_api.stack_get(self.context, self.id)
if stack is not None:
stack.update_and_save({
'action': action,
'status': status,
'status_reason': reason
})
LOG.info(
_LI('Stack %(action)s %(status)s (%(name)s): '
'%(reason)s'), {
'action': action,
'status': status,
'name': self.name,
'reason': reason
})
notification.send(self)
def _stub_grp_replace(self,
num_creates_expected_on_updt=0,
num_deletes_expected_on_updt=0,
num_reloads_expected_on_updt=0):
"""
Expect replacement of the capacity by batch size
"""
# for load balancer setup
self._stub_lb_reload(num_reloads_expected_on_updt)
self.m.StubOutWithMock(notification, 'send')
notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None)
# for instances in the group
self.m.StubOutWithMock(clients.OpenStackClients, 'nova')
self.m.StubOutWithMock(instance.Instance, 'handle_create')
self.m.StubOutWithMock(instance.Instance, 'check_create_complete')
self.m.StubOutWithMock(instance.Instance, 'destroy')
if num_reloads_expected_on_updt > 1:
clients.OpenStackClients.nova().MultipleTimes().AndReturn(self.fc)
cookie = object()
for i in range(num_creates_expected_on_updt):
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
for i in range(num_deletes_expected_on_updt):
instance.Instance.destroy().AndReturn(None)
def state_set(self, action, status, reason):
'''Update the stack state in the database.'''
if action not in self.ACTIONS:
raise ValueError(_("Invalid action %s") % action)
if status not in self.STATUSES:
raise ValueError(_("Invalid status %s") % status)
self.action = action
self.status = status
self.status_reason = reason
if self.id is None:
return
stack = db_api.stack_get(self.context, self.id)
if stack is not None:
stack.update_and_save({'action': action,
'status': status,
'status_reason': reason})
msg = _('Stack %(action)s %(status)s (%(name)s): %(reason)s')
LOG.info(msg % {'action': action,
'status': status,
'name': self.name,
'reason': reason})
notification.send(self)
def _stub_grp_create(self, capacity=0, setup_lb=True):
"""
Expect creation of instances to capacity. By default, expect creation
of load balancer unless specified.
"""
self._stub_validate()
self.m.StubOutWithMock(clients.OpenStackClients, 'nova')
self.m.StubOutWithMock(instance.Instance, 'handle_create')
self.m.StubOutWithMock(instance.Instance, 'check_create_complete')
self.m.StubOutWithMock(notification, 'send')
notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None)
cookie = object()
clients.OpenStackClients.nova().MultipleTimes().AndReturn(self.fc)
# for load balancer setup
if setup_lb:
self._stub_lb_create()
self._stub_lb_reload()
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
# for each instance in group
for i in range(capacity):
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
def _stub_grp_replace(self,
num_creates_expected_on_updt=0,
num_deletes_expected_on_updt=0,
num_reloads_expected_on_updt=0):
"""
Expect replacement of the capacity by batch size
"""
# for load balancer setup
self._stub_lb_reload(num_reloads_expected_on_updt)
self.m.StubOutWithMock(notification, 'send')
notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None)
# for instances in the group
self.m.StubOutWithMock(clients.OpenStackClients, '_nova')
self.m.StubOutWithMock(instance.Instance, 'handle_create')
self.m.StubOutWithMock(instance.Instance, 'check_create_complete')
self.m.StubOutWithMock(instance.Instance, 'destroy')
if num_reloads_expected_on_updt > 1:
clients.OpenStackClients._nova().MultipleTimes().AndReturn(self.fc)
cookie = object()
for i in range(num_creates_expected_on_updt):
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
for i in range(num_deletes_expected_on_updt):
instance.Instance.destroy().AndReturn(None)
def _stub_grp_create(self, capacity=0, setup_lb=True):
"""
Expect creation of instances to capacity. By default, expect creation
of load balancer unless specified.
"""
self._stub_validate()
self.m.StubOutWithMock(clients.OpenStackClients, '_nova')
self.m.StubOutWithMock(instance.Instance, 'handle_create')
self.m.StubOutWithMock(instance.Instance, 'check_create_complete')
self.m.StubOutWithMock(notification, 'send')
notification.send(mox.IgnoreArg()).MultipleTimes().AndReturn(None)
cookie = object()
clients.OpenStackClients._nova().MultipleTimes().AndReturn(self.fc)
# for load balancer setup
if setup_lb:
self._stub_lb_create()
self._stub_lb_reload()
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
# for each instance in group
for i in range(capacity):
instance.Instance.handle_create().AndReturn(cookie)
instance.Instance.check_create_complete(cookie).AndReturn(True)
def state_set(self, action, status, reason):
"""Update the stack state in the database."""
if action not in self.ACTIONS:
raise ValueError(_("Invalid action %s") % action)
if status not in self.STATUSES:
raise ValueError(_("Invalid status %s") % status)
self.action = action
self.status = status
self.status_reason = reason
if self.id is None:
return
stack = db_api.stack_get(self.context, self.id)
if stack is not None:
stack.update_and_save({"action": action, "status": status, "status_reason": reason})
msg = _("Stack %(action)s %(status)s (%(name)s): %(reason)s")
LOG.info(msg % {"action": action, "status": status, "name": self.name, "reason": reason})
notification.send(self)
def update_task(self, newstack, action=UPDATE, event=None):
if action not in (self.UPDATE, self.ROLLBACK, self.RESTORE):
LOG.error(_LE("Unexpected action %s passed to update!"), action)
self.state_set(self.UPDATE, self.FAILED,
"Invalid action %s" % action)
return
try:
lifecycle_plugin_utils.do_pre_ops(self.context, self, newstack,
action)
except Exception as e:
self.state_set(
action, self.FAILED,
e.args[0] if e.args else 'Failed stack pre-ops: %s' %
six.text_type(e))
return
if self.status == self.IN_PROGRESS:
if action == self.ROLLBACK:
LOG.debug("Starting update rollback for %s" % self.name)
else:
self.state_set(action, self.FAILED,
'State invalid for %s' % action)
return
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
if action == self.UPDATE:
# Oldstack is useless when the action is not UPDATE , so we don't
# need to build it, this can avoid some unexpected errors.
oldstack = Stack(self.context, self.name, copy.deepcopy(self.t),
self.env)
backup_stack = self._backup_stack()
try:
update_task = update.StackUpdate(
self,
newstack,
backup_stack,
rollback=action == self.ROLLBACK,
error_wait_time=cfg.CONF.error_wait_time)
updater = scheduler.TaskRunner(update_task)
self.env = newstack.env
self.parameters = newstack.parameters
self.t.files = newstack.t.files
self.disable_rollback = newstack.disable_rollback
self.timeout_mins = newstack.timeout_mins
self._set_param_stackid()
try:
updater.start(timeout=self.timeout_secs())
yield
while not updater.step():
if event is None or not event.ready():
yield
else:
message = event.wait()
if message == rpc_api.THREAD_CANCEL:
raise ForcedCancel()
finally:
self.reset_dependencies()
if action == self.UPDATE:
reason = 'Stack successfully updated'
elif action == self.RESTORE:
reason = 'Stack successfully restored'
else:
reason = 'Stack rollback completed'
stack_status = self.COMPLETE
except scheduler.Timeout:
stack_status = self.FAILED
reason = 'Timed out'
except ForcedCancel as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
update_task.updater.cancel_all()
yield self.update_task(oldstack, action=self.ROLLBACK)
return
except exception.ResourceFailure as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
# If rollback is enabled, we do another update, with the
# existing template, so we roll back to the original state
if not self.disable_rollback:
yield self.update_task(oldstack, action=self.ROLLBACK)
return
else:
LOG.debug('Deleting backup stack')
backup_stack.delete(backup=True)
# flip the template to the newstack values
self.t = newstack.t
template_outputs = self.t[self.t.OUTPUTS]
self.outputs = self.resolve_static_data(template_outputs)
# Don't use state_set to do only one update query and avoid race
# condition with the COMPLETE status
self.action = action
self.status = stack_status
self.status_reason = reason
self.store()
lifecycle_plugin_utils.do_post_ops(self.context, self, newstack,
action,
(self.status == self.FAILED))
notification.send(self)
def update_task(self, newstack, action=UPDATE):
if action not in (self.UPDATE, self.ROLLBACK):
LOG.error(_("Unexpected action %s passed to update!") % action)
self.state_set(self.UPDATE, self.FAILED,
"Invalid action %s" % action)
return
if self.status != self.COMPLETE:
if (action == self.ROLLBACK
and self.state == (self.UPDATE, self.IN_PROGRESS)):
LOG.debug("Starting update rollback for %s" % self.name)
else:
self.state_set(action, self.FAILED,
'State invalid for %s' % action)
return
self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action)
oldstack = Stack(self.context, self.name, copy.deepcopy(self.t),
self.env)
backup_stack = self._backup_stack()
try:
update_task = update.StackUpdate(self,
newstack,
backup_stack,
rollback=action == self.ROLLBACK)
updater = scheduler.TaskRunner(update_task)
self.env = newstack.env
self.parameters = newstack.parameters
self.t.files = newstack.t.files
self.disable_rollback = newstack.disable_rollback
self.timeout_mins = newstack.timeout_mins
self._set_param_stackid()
try:
updater.start(timeout=self.timeout_secs())
yield
while not updater.step():
yield
finally:
self.reset_dependencies()
if action == self.UPDATE:
reason = 'Stack successfully updated'
else:
reason = 'Stack rollback completed'
stack_status = self.COMPLETE
except scheduler.Timeout:
stack_status = self.FAILED
reason = 'Timed out'
except exception.ResourceFailure as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
# If rollback is enabled, we do another update, with the
# existing template, so we roll back to the original state
if not self.disable_rollback:
yield self.update_task(oldstack, action=self.ROLLBACK)
return
else:
LOG.debug('Deleting backup stack')
backup_stack.delete(backup=True)
# flip the template to the newstack values
self.t = newstack.t
template_outputs = self.t[self.t.OUTPUTS]
self.outputs = self.resolve_static_data(template_outputs)
# Don't use state_set to do only one update query and avoid race
# condition with the COMPLETE status
self.action = action
self.status = stack_status
self.status_reason = reason
self.store()
notification.send(self)
def update_task(self, newstack, action=UPDATE):
if action not in (self.UPDATE, self.ROLLBACK):
LOG.error(_("Unexpected action %s passed to update!") % action)
self.state_set(self.UPDATE, self.FAILED,
"Invalid action %s" % action)
return
if self.status != self.COMPLETE:
if (action == self.ROLLBACK and
self.state == (self.UPDATE, self.IN_PROGRESS)):
LOG.debug("Starting update rollback for %s" % self.name)
else:
self.state_set(action, self.FAILED,
'State invalid for %s' % action)
return
self.state_set(action, self.IN_PROGRESS,
'Stack %s started' % action)
oldstack = Stack(self.context, self.name, self.t, self.env)
backup_stack = self._backup_stack()
try:
update_task = update.StackUpdate(self, newstack, backup_stack,
rollback=action == self.ROLLBACK)
updater = scheduler.TaskRunner(update_task)
self.env = newstack.env
self.parameters = newstack.parameters
self.t.files = newstack.t.files
self.disable_rollback = newstack.disable_rollback
self.timeout_mins = newstack.timeout_mins
self._set_param_stackid()
try:
updater.start(timeout=self.timeout_secs())
yield
while not updater.step():
yield
finally:
self.reset_dependencies()
if action == self.UPDATE:
reason = 'Stack successfully updated'
else:
reason = 'Stack rollback completed'
stack_status = self.COMPLETE
except scheduler.Timeout:
stack_status = self.FAILED
reason = 'Timed out'
except exception.ResourceFailure as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
# If rollback is enabled, we do another update, with the
# existing template, so we roll back to the original state
if not self.disable_rollback:
yield self.update_task(oldstack, action=self.ROLLBACK)
return
else:
LOG.debug('Deleting backup stack')
backup_stack.delete(backup=True)
# flip the template to the newstack values
# Note we do this on success and failure, so the current
# stack resources are stored, even if one is in a failed
# state (otherwise we won't remove them on delete)
self.t = newstack.t
template_outputs = self.t[self.t.OUTPUTS]
self.outputs = self.resolve_static_data(template_outputs)
# Don't use state_set to do only one update query and avoid race
# condition with the COMPLETE status
self.action = action
self.status = stack_status
self.status_reason = reason
self.store()
notification.send(self)
def update_task(self, newstack, action=UPDATE, event=None):
if action not in (self.UPDATE, self.ROLLBACK):
LOG.error(_("Unexpected action %s passed to update!") % action)
self.state_set(self.UPDATE, self.FAILED,
"Invalid action %s" % action)
return
try:
lifecycle_plugin_utils.do_pre_ops(self.context, self,
newstack, action)
except Exception as e:
self.state_set(action, self.FAILED, e.args[0] if e.args else
'Failed stack pre-ops: %s' % six.text_type(e))
return
if self.status == self.IN_PROGRESS:
if action == self.ROLLBACK:
LOG.debug("Starting update rollback for %s" % self.name)
else:
self.state_set(action, self.FAILED,
'State invalid for %s' % action)
return
self.state_set(action, self.IN_PROGRESS,
'Stack %s started' % action)
if action == self.UPDATE:
# Oldstack is useless when the action is not UPDATE , so we don't
# need to build it, this can avoid some unexpected errors.
oldstack = Stack(self.context, self.name, copy.deepcopy(self.t),
self.env)
backup_stack = self._backup_stack()
try:
update_task = update.StackUpdate(self, newstack, backup_stack,
rollback=action == self.ROLLBACK,
error_wait_time=ERROR_WAIT_TIME)
updater = scheduler.TaskRunner(update_task)
self.env = newstack.env
self.parameters = newstack.parameters
self.t.files = newstack.t.files
self.disable_rollback = newstack.disable_rollback
self.timeout_mins = newstack.timeout_mins
self._set_param_stackid()
try:
updater.start(timeout=self.timeout_secs())
yield
while not updater.step():
if event is None or not event.ready():
yield
else:
message = event.wait()
if message == rpc_api.THREAD_CANCEL:
raise ForcedCancel()
finally:
self.reset_dependencies()
if action == self.UPDATE:
reason = 'Stack successfully updated'
else:
reason = 'Stack rollback completed'
stack_status = self.COMPLETE
except scheduler.Timeout:
stack_status = self.FAILED
reason = 'Timed out'
except ForcedCancel as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
update_task.updater.cancel_all()
yield self.update_task(oldstack, action=self.ROLLBACK)
return
except exception.ResourceFailure as e:
reason = six.text_type(e)
stack_status = self.FAILED
if action == self.UPDATE:
# If rollback is enabled, we do another update, with the
# existing template, so we roll back to the original state
if not self.disable_rollback:
yield self.update_task(oldstack, action=self.ROLLBACK)
return
else:
LOG.debug('Deleting backup stack')
backup_stack.delete(backup=True)
# flip the template to the newstack values
self.t = newstack.t
template_outputs = self.t[self.t.OUTPUTS]
self.outputs = self.resolve_static_data(template_outputs)
# Don't use state_set to do only one update query and avoid race
# condition with the COMPLETE status
self.action = action
self.status = stack_status
self.status_reason = reason
self.store()
lifecycle_plugin_utils.do_post_ops(self.context, self,
newstack, action,
(self.status == self.FAILED))
notification.send(self)
