def do_rebuild(self, obj):
if not obj.physical_id:
return False
self.server_id = obj.physical_id
driver = self.compute(obj)
try:
server = driver.server_get(self.server_id)
except exc.InternalError as ex:
raise exc.EResourceOperation(op='rebuilding',
type='server',
id=self.server_id,
message=six.text_type(ex))
if server is None or server.image is None:
return False
image_id = server.image['id']
admin_pass = self.properties.get(self.ADMIN_PASS)
try:
driver.server_rebuild(self.server_id, image_id,
self.properties.get(self.NAME), admin_pass)
driver.wait_for_server(self.server_id, 'ACTIVE')
except exc.InternalError as ex:
raise exc.EResourceOperation(op='rebuilding',
type='server',
id=self.server_id,
message=six.text_type(ex))
return True
def do_recover(self, obj, **options):
"""Default recover operation.
:param obj: The node object to operate on.
:param options: Keyword arguments for the recover operation.
"""
operation = options.pop('operation', None)
# TODO(Qiming): The operation input could be a list of operations.
if operation and not isinstance(operation, six.string_types):
operation = operation[0]
if operation and operation != consts.RECOVER_RECREATE:
LOG.error(_LE("Recover operation not supported: %s"), operation)
return False
try:
self.do_delete(obj, **options)
except exc.EResourceDeletion as ex:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex))
res = None
try:
res = self.do_create(obj)
except exc.EResourceCreation as ex:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex))
return res
def do_recover(self, obj, **options):
"""Default recover operation.
This is provided as a fallback if a specific profile type does not
override this method.
:param obj: The node object to operate on.
:param options: Keyword arguments for the recover operation.
:return id: New id of the recovered resource or None if recovery
failed.
:return status: True indicates successful recovery, False indicates
failure.
"""
operation = options.get('operation', None)
force_recreate = options.get('force_recreate', None)
delete_timeout = options.get('delete_timeout', None)
if operation.upper() != consts.RECOVER_RECREATE:
LOG.error("Recover operation not supported: %s", operation)
return None, False
extra_params = options.get('operation_params', None)
fence_compute = False
if extra_params:
fence_compute = extra_params.get('fence_compute', False)
try:
self.do_delete(obj, force=fence_compute, timeout=delete_timeout)
except exc.EResourceDeletion as ex:
if force_recreate:
# log error and continue on to creating the node
LOG.warning('Failed to delete node during recovery action: %s',
ex)
else:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex))
# pause to allow deleted resource to get reclaimed by nova
# this is needed to avoid a problem when the compute resources are
# at their quota limit. The deleted resource has to become available
# so that the new node can be created.
eventlet.sleep(cfg.CONF.batch_interval)
res = None
try:
res = self.do_create(obj)
except exc.EResourceCreation as ex:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex),
resource_id=ex.resource_id)
return res, True
def test_node_recover_check_exception(self, mock_recover, mock_status):
def set_status(*args, **kwargs):
if args[1] == 'ACTIVE':
node.physical_id = new_id
node = nodem.Node('node1', PROFILE_ID, '')
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
new_id = '166db83b-b4a4-49ef-96a8-6c0fdd882d1a'
mock_recover.return_value = new_id
mock_status.side_effect = set_status
mock_check = self.patchobject(pb.Profile, 'check_object')
mock_check.side_effect = exception.EResourceOperation(
op='checking', type='server', id=node.physical_id, reason='Boom!')
action = mock.Mock(inputs={'operation': ['boom'], 'check': True})
res = node.do_recover(self.context, action)
self.assertTrue(res)
mock_check.assert_called_once_with(self.context, node)
mock_recover.assert_called_once_with(self.context, node,
**action.inputs)
self.assertEqual('node1', node.name)
self.assertEqual(new_id, node.physical_id)
self.assertEqual(PROFILE_ID, node.profile_id)
mock_status.assert_has_calls([
mock.call(self.context, 'RECOVERING',
reason='Recover in progress'),
mock.call(self.context,
consts.NS_ACTIVE,
reason='Recover succeeded',
physical_id=new_id)
])
def run_workflow(self, **options):
if not self.physical_id:
return False
workflow_name = options.pop('workflow_name')
inputs = options.pop('inputs')
definition = inputs.pop('definition', None)
params = {
'cluster_id': self.cluster_id,
'node_id': self.physical_id,
}
params.update(inputs)
try:
profile = self.rt['profile']
wfc = profile.workflow(self)
workflow = wfc.workflow_find(workflow_name)
if workflow is None:
wfc.workflow_create(definition, scope="private")
else:
definition = workflow.definition
inputs_str = jsonutils.dumps(params)
wfc.execution_create(workflow_name, str(inputs_str))
except exc.InternalError as ex:
raise exc.EResourceOperation(op='executing',
type='workflow',
id=workflow_name,
message=six.text_type(ex))
return True
def do_check(self, obj):
"""Check stack status.
:param obj: Node object to operate.
:returns: True if check succeeded, or False otherwise.
"""
stack_id = obj.physical_id
if stack_id is None:
return False
hc = self.orchestration(obj)
try:
# Timeout = None means we will use the 'default_action_timeout'
# It can be overridden by the TIMEOUT profile properties
timeout = None
if self.properties[self.TIMEOUT]:
timeout = self.properties[self.TIMEOUT] * 60
hc.stack_check(stack_id)
hc.wait_for_stack(stack_id, 'CHECK_COMPLETE', timeout=timeout)
except exc.InternalError as ex:
raise exc.EResourceOperation(op='checking', type='stack',
id=stack_id,
message=six.text_type(ex))
return True
def do_recover(self, obj, **options):
"""Default recover operation.
This is provided as a fallback if a specific profile type does not
override this method.
:param obj: The node object to operate on.
:param options: Keyword arguments for the recover operation.
:return id: New id of the recovered resource or None if recovery
failed.
:return status: True indicates successful recovery, False indicates
failure.
"""
operation = options.pop('operation', None)
force_recreate = options.pop('force_recreate', None)
delete_timeout = options.pop('delete_timeout', None)
# The operation is a list of action names with optional parameters
if operation and not isinstance(operation, six.string_types):
operation = operation[0]
if operation and operation['name'] != consts.RECOVER_RECREATE:
LOG.error("Recover operation not supported: %s", operation)
return None, False
extra_params = options.get('params', {})
fence_compute = extra_params.get('fence_compute', False)
try:
self.do_delete(obj, force=fence_compute, timeout=delete_timeout,
delete_ports_on_failure=force_recreate)
except exc.EResourceDeletion as ex:
if force_recreate:
# log error and continue on to creating the node
LOG.warning('Failed to delete node during recovery action: %s',
ex)
else:
raise exc.EResourceOperation(op='recovering', type='node',
id=obj.id,
message=six.text_type(ex))
res = None
try:
res = self.do_create(obj)
except exc.EResourceCreation as ex:
raise exc.EResourceOperation(op='recovering', type='node',
id=obj.id, message=six.text_type(ex))
return res, True
def handle_stop(self, obj, **options):
"""Handler for the stop operation."""
if not obj.physical_id:
return
timeout = options.get('timeout', None)
if timeout:
timeout = int(timeout)
try:
self.docker(obj).stop(obj.physical_id, timeout=timeout)
except exc.InternalError as ex:
raise exc.EResourceOperation(type='container',
id=obj.physical_id[:8],
op='stop',
message=str(ex))
def do_recover(self, obj, **options):
"""Default recover operation.
This is provided as a fallback if a specific profile type does not
override this method.
:param obj: The node object to operate on.
:param options: Keyword arguments for the recover operation.
"""
operation = options.pop('operation', None)
# The operation is a list of action names with optional parameters
if operation and not isinstance(operation, six.string_types):
operation = operation[0]
if operation and operation['name'] != consts.RECOVER_RECREATE:
LOG.error("Recover operation not supported: %s", operation)
return False
extra_params = options.get('params', {})
fence_compute = extra_params.get('fence_compute', False)
try:
self.do_delete(obj, force=fence_compute)
except exc.EResourceDeletion as ex:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex))
res = None
try:
res = self.do_create(obj)
except exc.EResourceCreation as ex:
raise exc.EResourceOperation(op='recovering',
type='node',
id=obj.id,
message=six.text_type(ex))
return res
def _disassociate_floatingip(self, obj, server):
ctx = context.get_service_context(user_id=obj.user,
project_id=obj.project)
if obj.cluster_id:
cluster = cluster_obj.Cluster.get(ctx, obj.cluster_id)
fip = cluster.data.get(self.KUBE_MASTER_FLOATINGIP)
if fip:
try:
self.compute(obj).server_floatingip_disassociate(
server, fip)
except exc.InternalError as ex:
raise exc.EResourceOperation(op='floatingip',
type='kubernetes',
id=fip,
message=str(ex))
def test_node_check_check_with_exc(self, mock_check, mock_status):
node = nodem.Node('node1', PROFILE_ID, '')
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
err = exception.EResourceOperation(op='checking',
type='server',
id=node.physical_id,
message='failed get')
mock_check.side_effect = err
res = node.do_check(self.context)
self.assertFalse(res)
mock_status.assert_called_once_with(
self.context, consts.NS_ERROR,
"Failed in checking server '%s': failed get." % node.physical_id)
def do_check(self, obj):
if not obj.physical_id:
return False
try:
server = self.compute(obj).server_get(obj.physical_id)
except exc.InternalError as ex:
raise exc.EResourceOperation(op='checking',
type='server',
id=obj.physical_id,
message=six.text_type(ex))
if (server is None or server.status != 'ACTIVE'):
return False
return True
def handle_unpause(self, obj):
"""Handler for an unpause operation.
:param obj: The node object representing the container.
:returns: None
"""
if not obj.physical_id:
return
try:
self.docker(obj).unpause(obj.physical_id)
except exc.InternalError as ex:
raise exc.EResourceOperation(type='container',
id=obj.physical_id[:8],
op='unpausing',
message=str(ex))
return
def test_node_operation_failed_op(self, mock_set_status):
node = nodem.Node('node1', PROFILE_ID, '')
node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1'
x_profile = mock.Mock()
err = exception.EResourceOperation(
op='dance', type='container', id='test_id', message='Boom')
x_profile.handle_dance = mock.Mock(side_effect=err)
node.rt['profile'] = x_profile
inputs = {'operation': 'dance', 'params': {'style': 'tango'}}
res = node.do_operation(self.context, **inputs)
self.assertFalse(res)
mock_set_status.assert_has_calls([
mock.call(self.context, consts.NS_OPERATING,
reason="Operation 'dance' in progress"),
mock.call(self.context, consts.NS_ERROR,
reason="Failed in dance container 'test_id': Boom.")
])
x_profile.handle_dance.assert_called_once_with(node, style='tango')
def handle_reboot(self, obj, **options):
"""Handler for a reboot operation.
:param obj: The node object representing the container.
:returns: None
"""
if not obj.physical_id:
return
if 'timeout' in options:
params = {'timeout': options['timeout']}
else:
params = {}
try:
self.docker(obj).restart(obj.physical_id, **params)
except exc.InternalError as ex:
raise exc.EResourceOperation(type='container',
id=obj.physical_id[:8],
op='rebooting',
message=str(ex))
return