def do_rebuild(self, obj): if not obj.physical_id: return False self.server_id = obj.physical_id driver = self.compute(obj) try: server = driver.server_get(self.server_id) except exc.InternalError as ex: raise exc.EResourceOperation(op='rebuilding', type='server', id=self.server_id, message=six.text_type(ex)) if server is None or server.image is None: return False image_id = server.image['id'] admin_pass = self.properties.get(self.ADMIN_PASS) try: driver.server_rebuild(self.server_id, image_id, self.properties.get(self.NAME), admin_pass) driver.wait_for_server(self.server_id, 'ACTIVE') except exc.InternalError as ex: raise exc.EResourceOperation(op='rebuilding', type='server', id=self.server_id, message=six.text_type(ex)) return True
def do_recover(self, obj, **options): """Default recover operation. :param obj: The node object to operate on. :param options: Keyword arguments for the recover operation. """ operation = options.pop('operation', None) # TODO(Qiming): The operation input could be a list of operations. if operation and not isinstance(operation, six.string_types): operation = operation[0] if operation and operation != consts.RECOVER_RECREATE: LOG.error(_LE("Recover operation not supported: %s"), operation) return False try: self.do_delete(obj, **options) except exc.EResourceDeletion as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) res = None try: res = self.do_create(obj) except exc.EResourceCreation as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) return res
def do_recover(self, obj, **options): """Default recover operation. This is provided as a fallback if a specific profile type does not override this method. :param obj: The node object to operate on. :param options: Keyword arguments for the recover operation. :return id: New id of the recovered resource or None if recovery failed. :return status: True indicates successful recovery, False indicates failure. """ operation = options.get('operation', None) force_recreate = options.get('force_recreate', None) delete_timeout = options.get('delete_timeout', None) if operation.upper() != consts.RECOVER_RECREATE: LOG.error("Recover operation not supported: %s", operation) return None, False extra_params = options.get('operation_params', None) fence_compute = False if extra_params: fence_compute = extra_params.get('fence_compute', False) try: self.do_delete(obj, force=fence_compute, timeout=delete_timeout) except exc.EResourceDeletion as ex: if force_recreate: # log error and continue on to creating the node LOG.warning('Failed to delete node during recovery action: %s', ex) else: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) # pause to allow deleted resource to get reclaimed by nova # this is needed to avoid a problem when the compute resources are # at their quota limit. The deleted resource has to become available # so that the new node can be created. eventlet.sleep(cfg.CONF.batch_interval) res = None try: res = self.do_create(obj) except exc.EResourceCreation as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex), resource_id=ex.resource_id) return res, True
def test_node_recover_check_exception(self, mock_recover, mock_status): def set_status(*args, **kwargs): if args[1] == 'ACTIVE': node.physical_id = new_id node = nodem.Node('node1', PROFILE_ID, '') node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1' new_id = '166db83b-b4a4-49ef-96a8-6c0fdd882d1a' mock_recover.return_value = new_id mock_status.side_effect = set_status mock_check = self.patchobject(pb.Profile, 'check_object') mock_check.side_effect = exception.EResourceOperation( op='checking', type='server', id=node.physical_id, reason='Boom!') action = mock.Mock(inputs={'operation': ['boom'], 'check': True}) res = node.do_recover(self.context, action) self.assertTrue(res) mock_check.assert_called_once_with(self.context, node) mock_recover.assert_called_once_with(self.context, node, **action.inputs) self.assertEqual('node1', node.name) self.assertEqual(new_id, node.physical_id) self.assertEqual(PROFILE_ID, node.profile_id) mock_status.assert_has_calls([ mock.call(self.context, 'RECOVERING', reason='Recover in progress'), mock.call(self.context, consts.NS_ACTIVE, reason='Recover succeeded', physical_id=new_id) ])
def run_workflow(self, **options): if not self.physical_id: return False workflow_name = options.pop('workflow_name') inputs = options.pop('inputs') definition = inputs.pop('definition', None) params = { 'cluster_id': self.cluster_id, 'node_id': self.physical_id, } params.update(inputs) try: profile = self.rt['profile'] wfc = profile.workflow(self) workflow = wfc.workflow_find(workflow_name) if workflow is None: wfc.workflow_create(definition, scope="private") else: definition = workflow.definition inputs_str = jsonutils.dumps(params) wfc.execution_create(workflow_name, str(inputs_str)) except exc.InternalError as ex: raise exc.EResourceOperation(op='executing', type='workflow', id=workflow_name, message=six.text_type(ex)) return True
def do_check(self, obj): """Check stack status. :param obj: Node object to operate. :returns: True if check succeeded, or False otherwise. """ stack_id = obj.physical_id if stack_id is None: return False hc = self.orchestration(obj) try: # Timeout = None means we will use the 'default_action_timeout' # It can be overridden by the TIMEOUT profile properties timeout = None if self.properties[self.TIMEOUT]: timeout = self.properties[self.TIMEOUT] * 60 hc.stack_check(stack_id) hc.wait_for_stack(stack_id, 'CHECK_COMPLETE', timeout=timeout) except exc.InternalError as ex: raise exc.EResourceOperation(op='checking', type='stack', id=stack_id, message=six.text_type(ex)) return True
def do_recover(self, obj, **options): """Default recover operation. This is provided as a fallback if a specific profile type does not override this method. :param obj: The node object to operate on. :param options: Keyword arguments for the recover operation. :return id: New id of the recovered resource or None if recovery failed. :return status: True indicates successful recovery, False indicates failure. """ operation = options.pop('operation', None) force_recreate = options.pop('force_recreate', None) delete_timeout = options.pop('delete_timeout', None) # The operation is a list of action names with optional parameters if operation and not isinstance(operation, six.string_types): operation = operation[0] if operation and operation['name'] != consts.RECOVER_RECREATE: LOG.error("Recover operation not supported: %s", operation) return None, False extra_params = options.get('params', {}) fence_compute = extra_params.get('fence_compute', False) try: self.do_delete(obj, force=fence_compute, timeout=delete_timeout, delete_ports_on_failure=force_recreate) except exc.EResourceDeletion as ex: if force_recreate: # log error and continue on to creating the node LOG.warning('Failed to delete node during recovery action: %s', ex) else: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) res = None try: res = self.do_create(obj) except exc.EResourceCreation as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) return res, True
def handle_stop(self, obj, **options): """Handler for the stop operation.""" if not obj.physical_id: return timeout = options.get('timeout', None) if timeout: timeout = int(timeout) try: self.docker(obj).stop(obj.physical_id, timeout=timeout) except exc.InternalError as ex: raise exc.EResourceOperation(type='container', id=obj.physical_id[:8], op='stop', message=str(ex))
def do_recover(self, obj, **options): """Default recover operation. This is provided as a fallback if a specific profile type does not override this method. :param obj: The node object to operate on. :param options: Keyword arguments for the recover operation. """ operation = options.pop('operation', None) # The operation is a list of action names with optional parameters if operation and not isinstance(operation, six.string_types): operation = operation[0] if operation and operation['name'] != consts.RECOVER_RECREATE: LOG.error("Recover operation not supported: %s", operation) return False extra_params = options.get('params', {}) fence_compute = extra_params.get('fence_compute', False) try: self.do_delete(obj, force=fence_compute) except exc.EResourceDeletion as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) res = None try: res = self.do_create(obj) except exc.EResourceCreation as ex: raise exc.EResourceOperation(op='recovering', type='node', id=obj.id, message=six.text_type(ex)) return res
def _disassociate_floatingip(self, obj, server): ctx = context.get_service_context(user_id=obj.user, project_id=obj.project) if obj.cluster_id: cluster = cluster_obj.Cluster.get(ctx, obj.cluster_id) fip = cluster.data.get(self.KUBE_MASTER_FLOATINGIP) if fip: try: self.compute(obj).server_floatingip_disassociate( server, fip) except exc.InternalError as ex: raise exc.EResourceOperation(op='floatingip', type='kubernetes', id=fip, message=str(ex))
def test_node_check_check_with_exc(self, mock_check, mock_status): node = nodem.Node('node1', PROFILE_ID, '') node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1' err = exception.EResourceOperation(op='checking', type='server', id=node.physical_id, message='failed get') mock_check.side_effect = err res = node.do_check(self.context) self.assertFalse(res) mock_status.assert_called_once_with( self.context, consts.NS_ERROR, "Failed in checking server '%s': failed get." % node.physical_id)
def do_check(self, obj): if not obj.physical_id: return False try: server = self.compute(obj).server_get(obj.physical_id) except exc.InternalError as ex: raise exc.EResourceOperation(op='checking', type='server', id=obj.physical_id, message=six.text_type(ex)) if (server is None or server.status != 'ACTIVE'): return False return True
def handle_unpause(self, obj): """Handler for an unpause operation. :param obj: The node object representing the container. :returns: None """ if not obj.physical_id: return try: self.docker(obj).unpause(obj.physical_id) except exc.InternalError as ex: raise exc.EResourceOperation(type='container', id=obj.physical_id[:8], op='unpausing', message=str(ex)) return
def test_node_operation_failed_op(self, mock_set_status): node = nodem.Node('node1', PROFILE_ID, '') node.physical_id = 'd94d6333-82e6-4f87-b7ab-b786776df9d1' x_profile = mock.Mock() err = exception.EResourceOperation( op='dance', type='container', id='test_id', message='Boom') x_profile.handle_dance = mock.Mock(side_effect=err) node.rt['profile'] = x_profile inputs = {'operation': 'dance', 'params': {'style': 'tango'}} res = node.do_operation(self.context, **inputs) self.assertFalse(res) mock_set_status.assert_has_calls([ mock.call(self.context, consts.NS_OPERATING, reason="Operation 'dance' in progress"), mock.call(self.context, consts.NS_ERROR, reason="Failed in dance container 'test_id': Boom.") ]) x_profile.handle_dance.assert_called_once_with(node, style='tango')
def handle_reboot(self, obj, **options): """Handler for a reboot operation. :param obj: The node object representing the container. :returns: None """ if not obj.physical_id: return if 'timeout' in options: params = {'timeout': options['timeout']} else: params = {} try: self.docker(obj).restart(obj.physical_id, **params) except exc.InternalError as ex: raise exc.EResourceOperation(type='container', id=obj.physical_id[:8], op='rebooting', message=str(ex)) return