def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
self.entity.do_check(self.context)
child = []
res = self.RES_OK
reason = _('Cluster checking completed.')
for node in self.entity.nodes:
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_CHECK,
name='node_check_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.entity.eval_status(self.context, consts.CLUSTER_CHECK)
return res, reason
def _update_nodes(self, profile_id, nodes_obj):
# Get batching policy data if any
LOG.info("Updating cluster '%(cluster)s': profile='%(profile)s'.", {
'cluster': self.entity.id,
'profile': profile_id
})
plan = []
pd = self.data.get('update', None)
if pd:
pause_time = pd.get('pause_time')
plan = pd.get('plan')
else:
pause_time = 0
nodes_list = []
for node in self.entity.nodes:
nodes_list.append(node.id)
plan.append(set(nodes_list))
for node_set in plan:
child = []
nodes = list(node_set)
for node in nodes:
kwargs = {
'name': 'node_update_%s' % node[:8],
'cause': consts.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
}
action_id = base.Action.create(self.context, node,
consts.NODE_UPDATE, **kwargs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child],
self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# clear the action list
child = []
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
self.entity.eval_status(self.context,
consts.CLUSTER_UPDATE)
return result, 'Failed in updating nodes.'
# pause time
if pause_time != 0:
self._sleep(pause_time)
self.entity.profile_id = profile_id
self.entity.eval_status(self.context,
consts.CLUSTER_UPDATE,
profile_id=profile_id,
updated_at=timeutils.utcnow(True))
return self.RES_OK, 'Cluster update completed.'
def CompleteLifecycleProc(context, action_id):
"""Complete lifecycle process."""
action = base.Action.load(context, action_id=action_id, project_safe=False)
if action is None:
LOG.error("Action %s could not be found.", action_id)
raise exception.ResourceNotFound(type='action', id=action_id)
if action.get_status() == consts.ACTION_WAITING_LIFECYCLE_COMPLETION:
# update action status and reset owner back to None
# so that the action will get picked up by dispatcher
ao.Action.update(
context, action_id, {
'status': consts.ACTION_READY,
'status_reason': 'Lifecycle complete.',
'owner': None
})
dispatcher.start_action()
else:
LOG.debug(
'Action %s status is not WAITING_LIFECYCLE. '
'Skip CompleteLifecycleProc', action_id)
return False
return True
def _remove_nodes_normally(self, action_name, node_ids, inputs=None):
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': consts.CAUSE_DERIVED,
'inputs': inputs or {},
}
action_id = base.Action.create(self.context, node_id, action_name,
**kwargs)
child.append((action_id, node_id))
if child:
dobj.Dependency.create(self.context, [aid for aid, nid in child],
self.id)
for action_id, node_id in child:
ao.Action.update(self.context, action_id,
{'status': base.Action.READY})
dispatcher.start_action()
res, reason = self._wait_for_dependents()
return res, reason
return self.RES_OK, ''
def release(thread, action_id):
'''Callback function that will be passed to GreenThread.link().'''
# Remove action thread from thread list
self.workers.pop(action_id)
action = action_mod.Action.load(self.db_session, action_id)
# This is for actions with RETRY
if action.status == action.READY:
dispatcher.start_action(action_id=action_id)
def release(thread, action_id):
'''Callback function that will be passed to GreenThread.link().'''
# Remove action thread from thread list
self.workers.pop(action_id)
action = action_mod.Action.load(self.db_session, action_id)
# This is for actions with RETRY
if action.status == action.READY:
dispatcher.start_action(action_id=action_id)
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id is not None:
errors.append(_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {'n': node_id,
'c': node.cluster_id})
continue
if node.status != node.ACTIVE:
errors.append(_('Node [%s] is not in ACTIVE status.'
) % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
for node in nodes:
kwargs = {
'name': 'node_join_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {'cluster_id': self.target},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_JOIN', **kwargs)
action.store(self.context)
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(self.READY)
dispatcher.start_action(action_id=action.id)
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.outputs['nodes_added'] = [node.id for node in nodes]
for node in nodes:
self.cluster.add_node(node)
return result, reason
def set_status(self, result, reason=None):
"""Set action status based on return value from execute."""
timestamp = wallclock()
if result == self.RES_OK:
status = self.SUCCEEDED
ao.Action.mark_succeeded(self.context, self.id, timestamp)
elif result == self.RES_ERROR:
status = self.FAILED
ao.Action.mark_failed(self.context, self.id, timestamp,
reason or 'ERROR')
elif result == self.RES_TIMEOUT:
status = self.FAILED
ao.Action.mark_failed(self.context, self.id, timestamp,
reason or 'TIMEOUT')
elif result == self.RES_CANCEL:
status = self.CANCELLED
ao.Action.mark_cancelled(self.context, self.id, timestamp)
elif result == self.RES_LIFECYCLE_COMPLETE:
status = self.SUCCEEDED
ao.Action.mark_ready(self.context, self.id, timestamp)
else: # result == self.RES_RETRY:
retries = self.data.get('retries', 0)
# Action failed at the moment, but can be retried
# retries time is configurable
if retries < cfg.CONF.lock_retry_times:
status = self.READY
retries += 1
self.data.update({'retries': retries})
ao.Action.abandon(self.context, self.id, {'data': self.data})
# sleep for a while
eventlet.sleep(cfg.CONF.lock_retry_interval)
dispatcher.start_action(self.id)
else:
status = self.RES_ERROR
if not reason:
reason = ('Exceeded maximum number of retries (%d)'
'') % cfg.CONF.lock_retry_times
ao.Action.mark_failed(self.context, self.id, timestamp, reason)
if status == self.SUCCEEDED:
EVENT.info(self, consts.PHASE_END, reason or 'SUCCEEDED')
elif status == self.READY:
EVENT.warning(self, consts.PHASE_ERROR, reason or 'RETRY')
else:
EVENT.error(self, consts.PHASE_ERROR, reason or 'ERROR')
self.status = status
self.status_reason = reason
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
res = self.cluster.do_recover(self.context)
if not res:
reason = _('Cluster recovery failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
# process data from health_policy
pd = self.data.get('health', None)
if pd is None:
pd = {
'health': {
'recover_action': 'RECREATE',
}
}
self.data.update(pd)
recover_action = pd.get('recover_action', 'RECREATE')
reason = _('Cluster recovery succeeded.')
children = []
for node in self.cluster.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_RECOVER,
name='node_recover_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
inputs={'operation': recover_action})
children.append(action_id)
if children:
dobj.Dependency.create(self.context, [c for c in children],
self.id)
for cid in children:
ao.Action.update(self.context, cid, {'status': 'READY'})
dispatcher.start_action()
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
if res != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.ERROR,
reason)
return res, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
res = self.cluster.do_recover(self.context)
if not res:
reason = _('Cluster recovery failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
# process data from health_policy
pd = self.data.get('health', None)
if pd is None:
pd = {
'health': {
'recover_action': 'RECREATE',
}
}
self.data.update(pd)
recover_action = pd.get('recover_action', 'RECREATE')
reason = _('Cluster recovery succeeded.')
children = []
for node in self.cluster.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
action_id = base.Action.create(
self.context, node_id, consts.NODE_RECOVER,
name='node_recover_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
inputs={'operation': recover_action}
)
children.append(action_id)
if children:
dobj.Dependency.create(self.context, [c for c in children],
self.id)
for cid in children:
ao.Action.update(self.context, cid, {'status': 'READY'})
dispatcher.start_action()
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
if res != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.ERROR,
reason)
return res, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def _wait_for_dependents(self, lifecycle_hook_timeout=None):
"""Wait for dependent actions to complete.
:returns: A tuple containing the result and the corresponding reason.
"""
status = self.get_status()
while status != self.READY:
if status == self.FAILED:
reason = ('%(action)s [%(id)s] failed' % {
'action': self.action, 'id': self.id[:8]})
LOG.debug(reason)
return self.RES_ERROR, reason
if self.is_cancelled():
# During this period, if cancel request comes, cancel this
# operation immediately after signaling children to cancel,
# then release the cluster lock
reason = ('%(action)s [%(id)s] cancelled' % {
'action': self.action, 'id': self.id[:8]})
LOG.debug(reason)
return self.RES_CANCEL, reason
# When a child action is cancelled the parent action will update
# its status to cancelled as well this allows it to exit.
if status == self.CANCELLED:
if self.check_children_complete():
reason = ('%(action)s [%(id)s] cancelled' % {
'action': self.action, 'id': self.id[:8]})
LOG.debug(reason)
return self.RES_CANCEL, reason
if self.is_timeout():
# Action timeout, return
reason = ('%(action)s [%(id)s] timeout' % {
'action': self.action, 'id': self.id[:8]})
LOG.debug(reason)
return self.RES_TIMEOUT, reason
if (lifecycle_hook_timeout is not None and
self.is_timeout(lifecycle_hook_timeout)):
# if lifecycle hook timeout is specified and Lifecycle hook
# timeout is reached, return
reason = ('%(action)s [%(id)s] lifecycle hook timeout'
'') % {'action': self.action, 'id': self.id[:8]}
LOG.debug(reason)
return self.RES_LIFECYCLE_HOOK_TIMEOUT, reason
# Continue waiting (with reschedule)
LOG.debug('Action %s sleep for 3 seconds ', self.id)
self._sleep(3)
status = self.get_status()
dispatcher.start_action()
return self.RES_OK, 'All dependents ended with success'
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
self.entity.do_recover(self.context)
inputs = {}
check = self.inputs.get('check', False)
inputs['operation'] = self.inputs.get('operation', None)
inputs['operation_params'] = self.inputs.get('operation_params', None)
children = []
for node in self.entity.nodes:
node_id = node.id
if check:
node = node_mod.Node.load(self.context, node_id=node_id)
node.do_check(self.context)
if node.status == consts.NS_ACTIVE:
continue
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_RECOVER,
name='node_recover_%s' % node_id[:8],
cause=consts.CAUSE_DERIVED,
inputs=inputs,
)
children.append(action_id)
res = self.RES_OK
reason = 'Cluster recovery succeeded.'
if children:
dobj.Dependency.create(self.context, [c for c in children],
self.id)
for cid in children:
ao.Action.update(self.context, cid,
{'status': consts.ACTION_READY})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
check_capacity = self.inputs.get('check_capacity', False)
if check_capacity is True:
self._check_capacity()
self.entity.eval_status(self.context, consts.CLUSTER_RECOVER)
return res, reason
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
self.entity.do_recover(self.context)
# process data from health_policy
pd = self.data.get('health', None)
inputs = {}
if pd:
recover_action = pd.get('recover_action', None)
fencing = pd.get('fencing', None)
if recover_action is not None:
inputs['operation'] = recover_action
if fencing is not None and 'COMPUTE' in fencing:
inputs['force'] = True
children = []
for node in self.entity.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_RECOVER,
name='node_recover_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
inputs=inputs,
)
children.append(action_id)
res = self.RES_OK
reason = _('Cluster recovery succeeded.')
if children:
dobj.Dependency.create(self.context, [c for c in children],
self.id)
for cid in children:
ao.Action.update(self.context, cid, {'status': 'READY'})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.entity.eval_status(self.context, consts.CLUSTER_RECOVER)
return res, reason
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
profile_id = self.inputs.get('new_profile_id')
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_UPDATE', **kwargs)
action.store(self.context)
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
# Wait for nodes to complete update
if len(self.cluster.nodes) > 0:
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
reason = _('Cluster update completed.')
self.cluster.set_status(self.context,
self.cluster.ACTIVE,
reason,
profile_id=profile_id)
return self.RES_OK, reason
def CompleteLifecycleProc(context, action_id):
"""Complete lifecycle process."""
action = base.Action.load(context, action_id=action_id, project_safe=False)
if action is None:
LOG.error("Action %s could not be found.", action_id)
raise exception.ResourceNotFound(type='action', id=action_id)
if action.get_status() == consts.ACTION_WAITING_LIFECYCLE_COMPLETION:
action.set_status(base.Action.RES_LIFECYCLE_COMPLETE)
dispatcher.start_action()
else:
LOG.debug('Action %s status is not WAITING_LIFECYCLE. '
'Skip CompleteLifecycleProc', action_id)
return False
return True
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
self.entity.do_check(self.context)
child = []
res = self.RES_OK
reason = 'Cluster checking completed.'
for node in self.entity.nodes:
node_id = node.id
need_delete = self.inputs.get('delete_check_action', False)
# delete some records of NODE_CHECK
if need_delete:
ao.Action.delete_by_target(
self.context,
node_id,
action=[consts.NODE_CHECK],
status=[consts.ACTION_SUCCEEDED, consts.ACTION_FAILED])
name = 'node_check_%s' % node_id[:8]
action_id = base.Action.create(self.context,
node_id,
consts.NODE_CHECK,
name=name,
cause=consts.CAUSE_DERIVED,
inputs=self.inputs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.entity.eval_status(self.context, consts.CLUSTER_CHECK)
return res, reason
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
profile_id = self.inputs.get('new_profile_id')
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_UPDATE', **kwargs)
action.store(self.context)
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
# Wait for nodes to complete update
if len(self.cluster.nodes) > 0:
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
reason = _('Cluster update completed.')
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason,
profile_id=profile_id)
return self.RES_OK, reason
def do_operation(self):
"""Handler for CLUSTER_OPERATION action.
Note that the inputs for the action should contain the following items:
* ``nodes``: The nodes to operate on;
* ``operation``: The operation to be performed;
* ``params``: The parameters corresponding to the operation.
:returns: A tuple containing the result and the corresponding reason.
"""
inputs = copy.deepcopy(self.inputs)
operation = inputs['operation']
self.entity.do_operation(self.context, operation=operation)
child = []
res = self.RES_OK
reason = "Cluster operation '%s' completed." % operation
nodes = inputs.pop('nodes')
for node_id in nodes:
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_OPERATION,
name='node_%s_%s' % (operation, node_id[:8]),
cause=consts.CAUSE_DERIVED,
inputs=inputs,
)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.entity.eval_status(self.context, operation)
return res, reason
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
saved_status = self.cluster.status
saved_reason = self.cluster.status_reason
res = self.cluster.do_check(self.context)
if not res:
reason = _('Cluster checking failed.')
self.cluster.set_status(self.context, saved_status, saved_reason)
return self.RES_ERROR, reason
child = []
res = self.RES_OK
reason = _('Cluster checking completed.')
for node in self.cluster.nodes:
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_CHECK,
name='node_check_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.cluster.set_status(self.context, saved_status, saved_reason)
return res, reason
def notify(self, context, params=None):
queue_name = self.channel['queue_name']
# Claim message(s) from queue
# TODO(Yanyanhu) carefully handling claim ttl to avoid
# potential race condition.
try:
claim = self.zaqar().claim_create(queue_name)
messages = claim.messages
except exc.InternalError as ex:
LOG.error(_('Failed in claiming message: %s') % ex.message)
return
# Build actions
actions = []
if messages:
for message in messages:
try:
action_id = self._build_action(context, message)
actions.append(action_id)
except exc.InternalError as ex:
LOG.error(_('Failed in building action: %s') % ex.message)
try:
self.zaqar().message_delete(queue_name, message['id'],
claim.id)
except exc.InternalError as ex:
LOG.error(
_('Failed in deleting message %(id)s: %(reason)s'), {
'id': message['id'],
'reason': ex.message
})
self.zaqar().claim_delete(queue_name, claim.id)
msg = _('Actions %(actions)s were successfully built.') % {
'actions': actions
}
LOG.info(msg)
dispatcher.start_action()
return actions
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
saved_status = self.cluster.status
saved_reason = self.cluster.status_reason
res = self.cluster.do_check(self.context)
if not res:
reason = _('Cluster checking failed.')
self.cluster.set_status(self.context, saved_status, saved_reason)
return self.RES_ERROR, reason
child = []
res = self.RES_OK
reason = _('Cluster checking completed.')
for node in self.cluster.nodes:
node_id = node.id
action_id = base.Action.create(
self.context, node_id, consts.NODE_CHECK,
name='node_check_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.cluster.set_status(self.context, saved_status, saved_reason)
return res, reason
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_deletion', True)
if not destroy:
action_name = consts.NODE_LEAVE
child_actions = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, action_name, **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context, [c.id for c in child_actions],
self.id)
for child in child_actions:
# Build dependency and make the new action ready
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
return res, reason
return self.RES_OK, ''
def set_status(self, result, reason=None):
"""Set action status based on return value from execute."""
timestamp = wallclock()
if result == self.RES_OK:
status = self.SUCCEEDED
ao.Action.mark_succeeded(self.context, self.id, timestamp)
elif result == self.RES_ERROR:
status = self.FAILED
ao.Action.mark_failed(self.context, self.id, timestamp,
reason or 'ERROR')
elif result == self.RES_TIMEOUT:
status = self.FAILED
ao.Action.mark_failed(self.context, self.id, timestamp,
reason or 'TIMEOUT')
elif result == self.RES_CANCEL:
status = self.CANCELLED
ao.Action.mark_cancelled(self.context, self.id, timestamp)
else: # result == self.RES_RETRY:
status = self.READY
# Action failed at the moment, but can be retried
# We abandon it and then notify other dispatchers to execute it
ao.Action.abandon(self.context, self.id)
dispatcher.start_action()
if status == self.SUCCEEDED:
EVENT.info(self, consts.PHASE_END, reason or 'SUCCEEDED')
elif status == self.READY:
EVENT.warning(self, consts.PHASE_ERROR, reason or 'RETRY')
else:
EVENT.error(self, consts.PHASE_ERROR, reason or 'ERROR')
self.status = status
self.status_reason = reason
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
saved_status = self.cluster.status
saved_reason = self.cluster.status_reason
self.cluster.do_check(self.context)
child_actions = []
for node in self.cluster.nodes:
node_id = node.id
kwargs = {
'name': 'node_check_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, 'NODE_CHECK', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
self.cluster.set_status(self.context, saved_status, saved_reason)
return res, reason
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_deletion', True)
if not destroy:
action_name = consts.NODE_LEAVE
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node_id, action_name,
**kwargs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
# Build dependency and make the new action ready
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
else:
reason = _('Failed in deleting nodes.')
return res, reason
return self.RES_OK, ''
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_deletion', True)
if not destroy:
action_name = consts.NODE_LEAVE
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node_id, action_name,
**kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
# Build dependency and make the new action ready
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
else:
reason = _('Failed in deleting nodes.')
return res, reason
return self.RES_OK, ''
def _delete_nodes(self, nodes):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_delete', True)
if not destroy:
action_name = consts.NODE_LEAVE
for node_id in nodes:
kwargs = {
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED
}
action = base.Action(node_id, action_name, **kwargs)
action.store(self.context)
# Build dependency and make the new action ready
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
if len(nodes) > 0:
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
# TODO(anyone): avoid passing nodes in this way.
self.data['nodes'] = nodes
for node_id in nodes:
self.cluster.remove_node(node_id)
return res, reason
return self.RES_OK, ''
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_delete', True)
if not destroy:
action_name = consts.NODE_LEAVE
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, action_name, **kwargs)
action.store(self.context)
# Build dependency and make the new action ready
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
if len(node_ids) > 0:
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
return res, reason
return self.RES_OK, ''
def do_update(self):
profile_id = self.inputs.get('new_profile_id')
for node in self.cluster.nodes:
kwargs = {
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
}
}
action = base.Action(node.id, 'NODE_UPDATE', **kwargs)
action.store(self.context)
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
# Wait for nodes to complete update
result = self.RES_OK
reason = _('Cluster update completed.')
if len(self.cluster.nodes) > 0:
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
return result, new_reason
# TODO(anyone): this seems an overhead
self.cluster.profile_id = profile_id
self.cluster.store(self.context)
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
name = self.inputs.get('name')
metadata = self.inputs.get('metadata')
timeout = self.inputs.get('timeout')
profile_id = self.inputs.get('new_profile_id')
if name is not None:
self.cluster.name = name
if metadata is not None:
self.cluster.metadata = metadata
if timeout is not None:
self.cluster.timeout = timeout
self.cluster.store(self.context)
reason = _('Cluster update completed.')
if profile_id is None:
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
fmt = _LI("Updating cluster '%(cluster)s': profile='%(profile)s'.")
LOG.info(fmt, {'cluster': self.cluster.id, 'profile': profile_id})
child = []
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_UPDATE, **kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
new_reason = _('Failed in updating nodes.')
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
self.cluster.set_status(self.context,
self.cluster.ACTIVE,
reason,
profile_id=profile_id)
return self.RES_OK, reason
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
child = []
for m in range(count):
index = db_api.cluster_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement['placements'][m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name,
self.cluster.profile_id,
self.cluster.id,
context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_CREATE, **kwargs)
child.append(action_id)
if child:
# Build dependency and make the new action ready
db_api.dependency_add(self.context, [a for a in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
else:
reason = _('Failed in creating nodes.')
return res, reason
return self.RES_OK, ''
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id == self.cluster.id:
node_ids.remove(node_id)
continue
if node.cluster_id is not None:
errors.append(
_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {
'n': node_id,
'c': node.cluster_id
})
continue
if node.status != node.ACTIVE:
errors.append(
_('Node [%s] is not in ACTIVE status.') % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
if len(node_ids) == 0:
return self.RES_OK, reason
for node_id in node_ids:
kwargs = {
'name': 'node_join_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'cluster_id': self.target
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, 'NODE_JOIN', **kwargs)
action.store(self.context)
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(self.READY)
dispatcher.start_action(action_id=action.id)
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.outputs['nodes_added'] = node_ids
for node in nodes:
self.cluster.add_node(node)
return result, reason
def do_replace_nodes(self):
"""Handler for the CLUSTER_REPLACE_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_dict = self.inputs
errors = []
original_nodes = []
replacement_nodes = []
for (original, replacement) in node_dict.items():
original_node = no.Node.get(self.context, original)
replacement_node = no.Node.get(self.context, replacement)
# The return value is None if node not found
if not original_node:
errors.append('Original node %s not found.' % original)
continue
if not replacement_node:
errors.append('Replacement node %s not found.' % replacement)
continue
if original_node.cluster_id != self.target:
errors.append('Node %(o)s is not a member of the '
'cluster %(c)s.' % {
'o': original,
'c': self.target
})
continue
if replacement_node.cluster_id:
errors.append(
('Node %(r)s is already owned by cluster %(c)s.') % {
'r': replacement,
'c': replacement_node.cluster_id
})
continue
if replacement_node.status != consts.NS_ACTIVE:
errors.append('Node %s is not in ACTIVE status.' % replacement)
continue
original_nodes.append(original_node)
replacement_nodes.append(replacement_node)
if len(errors) > 0:
return self.RES_ERROR, '\n'.join(errors)
result = self.RES_OK
reason = 'Completed replacing nodes.'
children = []
for (original, replacement) in node_dict.items():
kwargs = {
'cause': consts.CAUSE_DERIVED,
}
# node_leave action
kwargs['name'] = 'node_leave_%s' % original[:8]
leave_action_id = base.Action.create(self.context, original,
consts.NODE_LEAVE, **kwargs)
# node_join action
kwargs['name'] = 'node_join_%s' % replacement[:8]
kwargs['inputs'] = {'cluster_id': self.target}
join_action_id = base.Action.create(self.context, replacement,
consts.NODE_JOIN, **kwargs)
children.append((join_action_id, leave_action_id))
if children:
dobj.Dependency.create(self.context, [c[0] for c in children],
self.id)
for child in children:
join_id = child[0]
leave_id = child[1]
ao.Action.update(self.context, join_id,
{'status': base.Action.READY})
dobj.Dependency.create(self.context, [join_id], leave_id)
ao.Action.update(self.context, leave_id,
{'status': base.Action.READY})
dispatcher.start_action()
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
for n in range(len(original_nodes)):
self.entity.remove_node(original_nodes[n])
self.entity.add_node(replacement_nodes[n])
self.entity.eval_status(self.context, consts.CLUSTER_REPLACE_NODES)
return result, reason
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
child = []
for m in range(count):
index = co.Cluster.get_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement['placements'][m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name, self.cluster.profile_id,
self.cluster.id, context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_CREATE, **kwargs)
child.append(action_id)
if child:
# Build dependency and make the new action ready
dobj.Dependency.create(self.context, [a for a in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
else:
reason = _('Failed in creating nodes.')
return res, reason
return self.RES_OK, ''
def _delete_nodes_with_hook(self, action_name, node_ids, lifecycle_hook):
lifecycle_hook_timeout = lifecycle_hook.get('timeout')
lifecycle_hook_type = lifecycle_hook.get('type', None)
lifecycle_hook_params = lifecycle_hook.get('params')
if lifecycle_hook_type == "zaqar":
lifecycle_hook_target = lifecycle_hook_params.get('queue')
else:
# lifecycle_hook_target = lifecycle_hook_params.get('url')
return self.RES_ERROR, ("Lifecycle hook type '%s' is not "
"implemented") % lifecycle_hook_type
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': consts.CAUSE_DERIVED_LCH,
}
action_id = base.Action.create(self.context, node_id, action_name,
**kwargs)
child.append((action_id, node_id))
if child:
dobj.Dependency.create(self.context, [aid for aid, nid in child],
self.id)
# lifecycle_hook_type has to be "zaqar"
# post message to zaqar
kwargs = {
'user': self.context.user_id,
'project': self.context.project_id,
'domain': self.context.domain_id
}
notifier = msg.Message(lifecycle_hook_target, **kwargs)
for action_id, node_id in child:
# wait lifecycle complete if node exists and is active
node = no.Node.get(self.context, node_id)
if not node:
LOG.warning(
'Node %s is not found. '
'Skipping wait for lifecycle completion.', node_id)
status = base.Action.READY
child.remove((action_id, node_id))
elif node.status != consts.NS_ACTIVE or not node.physical_id:
LOG.warning(
'Node %s is not in ACTIVE status. '
'Skipping wait for lifecycle completion.', node_id)
status = base.Action.READY
child.remove((action_id, node_id))
else:
status = base.Action.WAITING_LIFECYCLE_COMPLETION
ao.Action.update(self.context, action_id, {'status': status})
if status == base.Action.WAITING_LIFECYCLE_COMPLETION:
notifier.post_lifecycle_hook_message(
action_id, node_id, node.physical_id,
consts.LIFECYCLE_NODE_TERMINATION)
dispatcher.start_action()
res, reason = self._wait_for_dependents(lifecycle_hook_timeout)
if res == self.RES_LIFECYCLE_HOOK_TIMEOUT:
self._handle_lifecycle_timeout(child)
if res is None or res == self.RES_LIFECYCLE_HOOK_TIMEOUT:
dispatcher.start_action()
res, reason = self._wait_for_dependents()
return res, reason
return self.RES_OK, ''
def test_start_action_function(self, mock_notify):
dispatcher.start_action(engine_id='FAKE_ENGINE')
mock_notify.assert_called_once_with(dispatcher.START_ACTION,
'FAKE_ENGINE')
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
for m in range(count):
index = db_api.cluster_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement[m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name, self.cluster.profile_id,
self.cluster.id, context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_CREATE', **kwargs)
action.store(self.context)
# Build dependency and make the new action ready
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
if count > 0:
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_added'] = [n.id for n in nodes]
for node in nodes:
self.cluster.add_node(node)
return res, reason
return self.RES_OK, ''
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
name = self.inputs.get('name')
metadata = self.inputs.get('metadata')
timeout = self.inputs.get('timeout')
profile_id = self.inputs.get('new_profile_id')
if name is not None:
self.cluster.name = name
if metadata is not None:
self.cluster.metadata = metadata
if timeout is not None:
self.cluster.timeout = timeout
self.cluster.store(self.context)
reason = _('Cluster update completed.')
if profile_id is not None:
fmt = _LI("Updating cluster '%(cluster)s': profile='%(profile)s'.")
LOG.info(fmt, {'cluster': self.cluster.id, 'profile': profile_id})
child_actions = []
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_UPDATE', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
self.cluster.set_status(self.context, self.cluster.ACTIVE,
reason, profile_id=profile_id)
return self.RES_OK, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
res = self.cluster.do_recover(self.context)
if not res:
reason = _('Cluster recover failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
# process data from health_policy
pd = self.data.get('health', None)
if pd is None:
pd = {
'health': {
'recover_action': 'RECREATE',
}
}
self.data.update(pd)
recover_action = pd.get('recover_action', 'RECREATE')
reason = _('Cluster recovery succeeded.')
child_actions = []
for node in self.cluster.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
kwargs = {
'name': 'node_recover_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'operation': recover_action,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, 'NODE_RECOVER', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
if res != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.ERROR,
reason)
return res, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id:
errors.append(
_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {
'n': node_id,
'c': node.cluster_id
})
continue
if node.status != node.ACTIVE:
errors.append(
_('Node [%s] is not in ACTIVE status.') % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
child = []
for node in nodes:
kwargs = {
'name': 'node_join_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'cluster_id': self.target
},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_JOIN, **kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.cluster.desired_capacity += len(nodes)
self.cluster.store(self.context)
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
return result, reason
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
TODO(anyone): handle placement data
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
errors = []
nodes = []
for nid in node_ids:
node = no.Node.get(self.context, nid)
if not node:
errors.append('Node %s is not found.' % nid)
continue
if node.cluster_id:
errors.append('Node %(n)s is already owned by cluster %(c)s.'
'' % {
'n': nid,
'c': node.cluster_id
})
continue
if node.status != consts.NS_ACTIVE:
errors.append('Node %s is not in ACTIVE status.' % nid)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, '\n'.join(errors)
reason = 'Completed adding nodes.'
# check the size constraint
current = no.Node.count_by_cluster(self.context, self.target)
desired = current + len(node_ids)
res = scaleutils.check_size_params(self.entity, desired, None, None,
True)
if res:
return self.RES_ERROR, res
child = []
for node in nodes:
nid = node.id
kwargs = {
'name': 'node_join_%s' % nid[:8],
'cause': consts.CAUSE_DERIVED,
'inputs': {
'cluster_id': self.target
},
}
action_id = base.Action.create(self.context, nid, consts.NODE_JOIN,
**kwargs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.entity.eval_status(self.context,
consts.CLUSTER_ADD_NODES,
desired_capacity=desired)
self.outputs['nodes_added'] = node_ids
creation = self.data.get('creation', {})
creation['nodes'] = node_ids
self.data['creation'] = creation
for node in nodes:
obj = node_mod.Node.load(self.context, db_node=node)
self.entity.add_node(obj)
return result, reason
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id:
errors.append(_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {'n': node_id,
'c': node.cluster_id})
continue
if node.status != node.ACTIVE:
errors.append(_('Node [%s] is not in ACTIVE status.'
) % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
child = []
for node in nodes:
kwargs = {
'name': 'node_join_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {'cluster_id': self.target},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_JOIN, **kwargs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.cluster = cluster_mod.Cluster.load(self.context,
self.cluster.id)
self.cluster.desired_capacity += len(nodes)
self.cluster.store(self.context)
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
return result, reason
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
name = self.inputs.get('name')
metadata = self.inputs.get('metadata')
timeout = self.inputs.get('timeout')
profile_id = self.inputs.get('new_profile_id')
if name is not None:
self.cluster.name = name
if metadata is not None:
self.cluster.metadata = metadata
if timeout is not None:
self.cluster.timeout = timeout
self.cluster.store(self.context)
reason = _('Cluster update completed.')
if profile_id is None:
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
fmt = _LI("Updating cluster '%(cluster)s': profile='%(profile)s'.")
LOG.info(fmt, {'cluster': self.cluster.id, 'profile': profile_id})
child = []
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_UPDATE, **kwargs)
child.append(action_id)
if child:
dobj.Dependency.create(self.context, [c for c in child], self.id)
for cid in child:
ao.Action.update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action()
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
new_reason = _('Failed in updating nodes.')
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
self.cluster.set_status(self.context, self.cluster.ACTIVE,
reason, profile_id=profile_id)
return self.RES_OK, reason
def test_start_action_function(self, mock_notify):
dispatcher.start_action(engine_id='FAKE_ENGINE')
mock_notify.assert_called_once_with(dispatcher.START_ACTION,
'FAKE_ENGINE')
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
for m in range(count):
index = db_api.cluster_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement[m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name,
self.cluster.profile_id,
self.cluster.id,
context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_CREATE', **kwargs)
action.store(self.context)
# Build dependency and make the new action ready
db_api.action_add_dependency(self.context, action.id, self.id)
action.set_status(action.READY)
dispatcher.start_action(action_id=action.id)
if count > 0:
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_added'] = [n.id for n in nodes]
for node in nodes:
self.cluster.add_node(node)
return res, reason
return self.RES_OK, ''
