def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
res = self.cluster.do_recover(self.context)
if not res:
reason = _('Cluster recovery failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
# process data from health_policy
pd = self.data.get('health', None)
if pd is None:
pd = {
'health': {
'recover_action': 'RECREATE',
}
}
self.data.update(pd)
recover_action = pd.get('recover_action', 'RECREATE')
reason = _('Cluster recovery succeeded.')
children = []
for node in self.cluster.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_RECOVER,
name='node_recover_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
inputs={'operation': recover_action})
children.append(action_id)
if children:
db_api.dependency_add(self.context, [c for c in children], self.id)
for cid in children:
db_api.action_update(self.context, cid, {'status': 'READY'})
dispatcher.start_action(action_id=cid)
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
if res != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.ERROR,
reason)
return res, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def store(self, context):
'''Store the action record into database table.'''
values = {
'name': self.name,
'context': self.context.to_dict(),
'target': self.target,
'action': self.action,
'cause': self.cause,
'owner': self.owner,
'interval': self.interval,
'start_time': self.start_time,
'end_time': self.end_time,
'timeout': self.timeout,
'status': self.status,
'status_reason': self.status_reason,
'inputs': self.inputs,
'outputs': self.outputs,
'depends_on': self.depends_on,
'depended_by': self.depended_by,
'created_time': datetime.datetime.utcnow(),
'updated_time': self.updated_time,
'deleted_time': self.deleted_time,
}
if self.id:
values['updated_time'] = datetime.datetime.utcnow()
action = db_api.action_update(context, values)
else:
values['created_time'] = datetime.datetime.utcnow()
action = db_api.action_create(context, values)
self.id = action.id
return self.id
def store(self, context):
"""Store the action record into database table.
:param context: An instance of the request context.
:return: The ID of the stored object.
"""
timestamp = timeutils.utcnow()
values = {
'name': self.name,
'context': self.context.to_dict(),
'target': self.target,
'action': self.action,
'cause': self.cause,
'owner': self.owner,
'interval': self.interval,
'start_time': self.start_time,
'end_time': self.end_time,
'timeout': self.timeout,
'status': self.status,
'status_reason': self.status_reason,
'inputs': self.inputs,
'outputs': self.outputs,
'created_at': self.created_at,
'updated_at': self.updated_at,
'data': self.data,
'user': self.user,
'project': self.project,
'domain': self.domain,
}
if self.id:
self.updated_at = timestamp
values['updated_at'] = timestamp
db_api.action_update(context, self.id, values)
else:
self.created_at = timestamp
values['created_at'] = timestamp
action = db_api.action_create(context, values)
self.id = action.id
return self.id
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
saved_status = self.cluster.status
saved_reason = self.cluster.status_reason
res = self.cluster.do_check(self.context)
if not res:
reason = _('Cluster checking failed.')
self.cluster.set_status(self.context, saved_status, saved_reason)
return self.RES_ERROR, reason
child = []
res = self.RES_OK
reason = _('Cluster checking completed.')
for node in self.cluster.nodes:
node_id = node.id
action_id = base.Action.create(
self.context,
node_id,
consts.NODE_CHECK,
name='node_check_%s' % node_id[:8],
cause=base.CAUSE_DERIVED,
)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for dependent action if any
res, new_reason = self._wait_for_dependents()
if res != self.RES_OK:
reason = new_reason
self.cluster.set_status(self.context, saved_status, saved_reason)
return res, reason
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_deletion', True)
if not destroy:
action_name = consts.NODE_LEAVE
child_actions = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, action_name, **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context, [c.id for c in child_actions],
self.id)
for child in child_actions:
# Build dependency and make the new action ready
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
return res, reason
return self.RES_OK, ''
def store(self, context):
'''Store the action record into database table.'''
timestamp = timeutils.utcnow()
values = {
'name': self.name,
'context': self.context.to_dict(),
'target': self.target,
'action': self.action,
'cause': self.cause,
'owner': self.owner,
'interval': self.interval,
'start_time': self.start_time,
'end_time': self.end_time,
'timeout': self.timeout,
'status': self.status,
'status_reason': self.status_reason,
'inputs': self.inputs,
'outputs': self.outputs,
'depends_on': self.depends_on,
'depended_by': self.depended_by,
'created_time': self.created_time,
'updated_time': self.updated_time,
'deleted_time': self.deleted_time,
'data': self.data,
}
if self.id:
self.updated_time = timestamp
values['updated_time'] = timestamp
db_api.action_update(context, self.id, values)
else:
self.created_time = timestamp
values['created_time'] = timestamp
action = db_api.action_create(context, values)
self.id = action.id
return self.id
def _delete_nodes(self, node_ids):
action_name = consts.NODE_DELETE
pd = self.data.get('deletion', None)
if pd is not None:
destroy = pd.get('destroy_after_deletion', True)
if not destroy:
action_name = consts.NODE_LEAVE
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node_id, action_name,
**kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
# Build dependency and make the new action ready
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids
for node_id in node_ids:
self.cluster.remove_node(node_id)
else:
reason = _('Failed in deleting nodes.')
return res, reason
return self.RES_OK, ''
def do_check(self):
"""Handler for CLUSTER_CHECK action.
:returns: A tuple containing the result and the corresponding reason.
"""
saved_status = self.cluster.status
saved_reason = self.cluster.status_reason
self.cluster.do_check(self.context)
child_actions = []
for node in self.cluster.nodes:
node_id = node.id
kwargs = {
'name': 'node_check_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, 'NODE_CHECK', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
self.cluster.set_status(self.context, saved_status, saved_reason)
return res, reason
def update(cls, context, action_id, values):
return db_api.action_update(context, action_id, values)
def update(cls, context, action_id, values):
return db_api.action_update(context, action_id, values)
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
name = self.inputs.get('name')
metadata = self.inputs.get('metadata')
timeout = self.inputs.get('timeout')
profile_id = self.inputs.get('new_profile_id')
if name is not None:
self.cluster.name = name
if metadata is not None:
self.cluster.metadata = metadata
if timeout is not None:
self.cluster.timeout = timeout
self.cluster.store(self.context)
reason = _('Cluster update completed.')
if profile_id is not None:
fmt = _LI("Updating cluster '%(cluster)s': profile='%(profile)s'.")
LOG.info(fmt, {'cluster': self.cluster.id, 'profile': profile_id})
child_actions = []
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_UPDATE', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
self.cluster.set_status(self.context, self.cluster.ACTIVE,
reason, profile_id=profile_id)
return self.RES_OK, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id:
errors.append(_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {'n': node_id,
'c': node.cluster_id})
continue
if node.status != node.ACTIVE:
errors.append(_('Node [%s] is not in ACTIVE status.'
) % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
child_actions = []
for node in nodes:
kwargs = {
'name': 'node_join_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {'cluster_id': self.target},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_JOIN', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context, [c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
return result, reason
def do_add_nodes(self):
"""Handler for the CLUSTER_ADD_NODES action.
:returns: A tuple containing the result and the corresponding reason.
"""
node_ids = self.inputs.get('nodes')
# TODO(anyone): handle placement data
errors = []
nodes = []
for node_id in node_ids:
try:
node = node_mod.Node.load(self.context, node_id)
except exception.NodeNotFound:
errors.append(_('Node [%s] is not found.') % node_id)
continue
if node.cluster_id:
errors.append(
_('Node [%(n)s] is already owned by cluster '
'[%(c)s].') % {
'n': node_id,
'c': node.cluster_id
})
continue
if node.status != node.ACTIVE:
errors.append(
_('Node [%s] is not in ACTIVE status.') % node_id)
continue
nodes.append(node)
if len(errors) > 0:
return self.RES_ERROR, ''.join(errors)
reason = _('Completed adding nodes.')
child = []
for node in nodes:
kwargs = {
'name': 'node_join_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'cluster_id': self.target
},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_JOIN, **kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for dependent action if any
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
reason = new_reason
else:
self.cluster.desired_capacity += len(nodes)
self.cluster.store(self.context)
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
return result, reason
def do_update(self):
"""Handler for CLUSTER_UPDATE action.
:returns: A tuple consisting the result and the corresponding reason.
"""
res = self.cluster.do_update(self.context)
if not res:
reason = _('Cluster update failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
name = self.inputs.get('name')
metadata = self.inputs.get('metadata')
timeout = self.inputs.get('timeout')
profile_id = self.inputs.get('new_profile_id')
if name is not None:
self.cluster.name = name
if metadata is not None:
self.cluster.metadata = metadata
if timeout is not None:
self.cluster.timeout = timeout
self.cluster.store(self.context)
reason = _('Cluster update completed.')
if profile_id is None:
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
fmt = _LI("Updating cluster '%(cluster)s': profile='%(profile)s'.")
LOG.info(fmt, {'cluster': self.cluster.id, 'profile': profile_id})
child = []
for node in self.cluster.nodes:
kwargs = {
'name': 'node_update_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'new_profile_id': profile_id,
},
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_UPDATE, **kwargs)
child.append(action_id)
if child:
db_api.dependency_add(self.context, [c for c in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
result, new_reason = self._wait_for_dependents()
if result != self.RES_OK:
new_reason = _('Failed in updating nodes.')
self.cluster.set_status(self.context, self.cluster.WARNING,
new_reason)
return result, new_reason
self.cluster.set_status(self.context,
self.cluster.ACTIVE,
reason,
profile_id=profile_id)
return self.RES_OK, reason
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
child = []
for m in range(count):
index = db_api.cluster_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement['placements'][m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name,
self.cluster.profile_id,
self.cluster.id,
context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
}
action_id = base.Action.create(self.context, node.id,
consts.NODE_CREATE, **kwargs)
child.append(action_id)
if child:
# Build dependency and make the new action ready
db_api.dependency_add(self.context, [a for a in child], self.id)
for cid in child:
db_api.action_update(self.context, cid,
{'status': base.Action.READY})
dispatcher.start_action(action_id=cid)
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
else:
reason = _('Failed in creating nodes.')
return res, reason
return self.RES_OK, ''
def _create_nodes(self, count):
"""Utility method for node creation.
:param count: Number of nodes to create.
:returns: A tuple comprised of the result and reason.
"""
if count == 0:
return self.RES_OK, ''
placement = self.data.get('placement', None)
nodes = []
child_actions = []
for m in range(count):
index = db_api.cluster_next_index(self.context, self.cluster.id)
kwargs = {
'index': index,
'metadata': {},
'user': self.cluster.user,
'project': self.cluster.project,
'domain': self.cluster.domain,
}
if placement is not None:
# We assume placement is a list
kwargs['data'] = {'placement': placement['placements'][m]}
name = 'node-%s-%003d' % (self.cluster.id[:8], index)
node = node_mod.Node(name, self.cluster.profile_id,
self.cluster.id, context=self.context,
**kwargs)
node.store(self.context)
nodes.append(node)
kwargs = {
'name': 'node_create_%s' % node.id[:8],
'cause': base.CAUSE_DERIVED,
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node.id, 'NODE_CREATE', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
# Build dependency and make the new action ready
db_api.dependency_add(self.context,
[a.id for a in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for cluster creation to complete
res, reason = self._wait_for_dependents()
if res == self.RES_OK:
nodes_added = [n.id for n in nodes]
self.outputs['nodes_added'] = nodes_added
creation = self.data.get('creation', {})
creation['nodes'] = nodes_added
self.data['creation'] = creation
for node in nodes:
self.cluster.add_node(node)
return res, reason
return self.RES_OK, ''
def do_recover(self):
"""Handler for the CLUSTER_RECOVER action.
:returns: A tuple containing the result and the corresponding reason.
"""
res = self.cluster.do_recover(self.context)
if not res:
reason = _('Cluster recover failed.')
self.cluster.set_status(self.context, self.cluster.ERROR, reason)
return self.RES_ERROR, reason
# process data from health_policy
pd = self.data.get('health', None)
if pd is None:
pd = {
'health': {
'recover_action': 'RECREATE',
}
}
self.data.update(pd)
recover_action = pd.get('recover_action', 'RECREATE')
reason = _('Cluster recovery succeeded.')
child_actions = []
for node in self.cluster.nodes:
if node.status == 'ACTIVE':
continue
node_id = node.id
kwargs = {
'name': 'node_recover_%s' % node_id[:8],
'cause': base.CAUSE_DERIVED,
'inputs': {
'operation': recover_action,
},
'user': self.context.user,
'project': self.context.project,
'domain': self.context.domain,
}
action = base.Action(node_id, 'NODE_RECOVER', **kwargs)
action.store(self.context)
child_actions.append(action)
if child_actions:
db_api.dependency_add(self.context,
[c.id for c in child_actions],
self.id)
for child in child_actions:
db_api.action_update(self.context, child.id,
{'status': child.READY})
dispatcher.start_action(action_id=child.id)
# Wait for dependent action if any
res, reason = self._wait_for_dependents()
if res != self.RES_OK:
self.cluster.set_status(self.context, self.cluster.ERROR,
reason)
return res, reason
self.cluster.set_status(self.context, self.cluster.ACTIVE, reason)
return self.RES_OK, reason
