def detach_volumes(): try: # NOTE(TheJulia): If the node is in ACTIVE state, we can # tolerate failures detaching as the node is likely being # powered down to cause a detachment event. allow_errors = (task.node.provision_state == states.ACTIVE or aborting_attach and outer_args['attempt'] > 0) cinder.detach_volumes(task, targets, connector, allow_errors=allow_errors) except exception.StorageError as e: with excutils.save_and_reraise_exception(): # NOTE(TheJulia): In the event that the node is not in # ACTIVE state, we need to fail hard as we need to ensure # all attachments are removed. if aborting_attach: msg_format = ("Error on aborting volume detach for " "node %(node)s: %(err)s.") else: msg_format = ("Error detaching volume for " "node %(node)s: %(err)s.") msg = (msg_format) % {'node': node.uuid, 'err': e} if outer_args['attempt'] < CONF.cinder.action_retries: outer_args['attempt'] += 1 msg += " Re-attempting detachment." LOG.warning(msg) else: LOG.error(msg)
def detach_volumes(): try: # NOTE(TheJulia): If the node is in ACTIVE state, we can # tolerate failures detaching as the node is likely being # powered down to cause a detachment event. allow_errors = (task.node.provision_state == states.ACTIVE) cinder.detach_volumes(task, targets, connector, allow_errors=allow_errors) except exception.StorageError as e: # NOTE(TheJulia): In the event that the node is not in # ACTIVE state, we need to fail hard as we need to ensure # all attachments are removed. msg = ("Error detaching volumes for " "node %(node)s: %(err)s.") % { 'node': node.uuid, 'err': e } if outer_args['attempt'] < CONF.cinder.action_retries: outer_args['attempt'] += 1 msg += " Re-attempting detachment." LOG.warning(msg) else: LOG.error(msg) raise
def test_detach_volumes_one_detached(self, mock_create_meta, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): """Iterate with two volumes, one already detached.""" volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id, 'detached'] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_get.side_effect = [ mock.Mock(attachments=[{ 'server_id': self.node.uuid, 'attachment_id': 'qux' }]), mock.Mock(attachments=[]) ] with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_begin.assert_called_once_with(mock.ANY, volume_id) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_begin_detaching_failure( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} volume = mock.Mock(attachments=[]) mock_get.return_value = volume mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_begin.side_effect = cinder_exceptions.NotAcceptable( http_client.NOT_ACCEPTABLE) with task_manager.acquire(self.context, self.node.uuid) as task: self.assertRaises(exception.StorageError, cinder.detach_volumes, task, volumes, connector) mock_is_attached.assert_called_once_with(mock.ANY, volume) cinder.detach_volumes(task, volumes, connector, allow_errors=True) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, None) mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_begin_detaching_failure( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} volume = mock.Mock(attachments=[]) mock_get.return_value = volume mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_begin.side_effect = cinder_exceptions.NotAcceptable( http_client.NOT_ACCEPTABLE) with task_manager.acquire(self.context, self.node.uuid) as task: self.assertRaises(exception.StorageError, cinder.detach_volumes, task, volumes, connector) mock_is_attached.assert_called_once_with(mock.ANY, volume) cinder.detach_volumes(task, volumes, connector, allow_errors=True) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, None) mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_vol_not_found(self, mock_get, mock_set_meta, mock_session): """Raise an error if the volume lookup fails""" volumes = ['vol1'] connector = {'foo': 'bar'} mock_get.side_effect = cinder_exceptions.NotFound(404, message='error') with task_manager.acquire(self.context, self.node.uuid) as task: self.assertRaises(exception.StorageError, cinder.detach_volumes, task, volumes, connector) self.assertFalse(mock_set_meta.called) # We should not raise any exception when issuing a command # with errors being permitted. cinder.detach_volumes(task, volumes, connector, allow_errors=True) self.assertFalse(mock_set_meta.called)
def test_detach_volumes_detach_meta_failure_errors_not_allowed( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_get.return_value = mock.Mock(attachments=[ {'server_id': self.node.uuid, 'attachment_id': 'qux'}]) mock_set_meta.side_effect = cinder_exceptions.NotAcceptable( http_client.NOT_ACCEPTABLE) with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_vol_not_found(self, mock_get, mock_set_meta, mock_session): """Raise an error if the volume lookup fails""" volumes = ['vol1'] connector = {'foo': 'bar'} mock_get.side_effect = cinder_exceptions.NotFound( http_client.NOT_FOUND, message='error') with task_manager.acquire(self.context, self.node.uuid) as task: self.assertRaises(exception.StorageError, cinder.detach_volumes, task, volumes, connector) self.assertFalse(mock_set_meta.called) # We should not raise any exception when issuing a command # with errors being permitted. cinder.detach_volumes(task, volumes, connector, allow_errors=True) self.assertFalse(mock_set_meta.called)
def test_detach_volumes_detach_meta_failure_errors_not_allowed( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_get.return_value = mock.Mock(attachments=[ {'server_id': self.node.uuid, 'attachment_id': 'qux'}]) mock_set_meta.side_effect = cinder_exceptions.NotAcceptable( http_client.NOT_ACCEPTABLE) with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_term_failure(self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_get, mock_set_meta, mock_session): volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_get.return_value = {'id': volume_id, 'attachments': []} mock_term.side_effect = cinder_exceptions.NotAcceptable(406) with task_manager.acquire(self.context, self.node.uuid) as task: self.assertRaises(exception.StorageError, cinder.detach_volumes, task, volumes, connector) mock_begin.assert_called_once_with(mock.ANY, volume_id) mock_term.assert_called_once_with(mock.ANY, volume_id, connector) cinder.detach_volumes(task, volumes, connector, allow_errors=True) self.assertFalse(mock_set_meta.called)
def _abort_attach_volume(self, task, connector): """Detach volumes as a result of failed attachment. If detachment fails, it will be retried with allow_errors=True. :param task: The task object. :param connector: The dictionary representing a node's connectivity as define by _generate_connector. """ targets = [target.volume_id for target in task.volume_targets] try: cinder.detach_volumes(task, targets, connector) except exception.StorageError as e: LOG.warning( "Error detaching volume for node %(node)s on " "aborting volume attach: %(err)s. Retrying with " "errors allowed.", { 'node': task.node.uuid, 'err': e }) cinder.detach_volumes(task, targets, connector, allow_errors=True)
def test_detach_volumes( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): """Iterate once and detach a volume without issues.""" volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_get.return_value = mock.Mock(attachments=[ {'server_id': self.node.uuid, 'attachment_id': 'qux'}]) with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_begin.assert_called_once_with(mock.ANY, volume_id) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes( self, mock_create_meta, mock_is_attached, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): """Iterate once and detach a volume without issues.""" volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_is_attached.return_value = True mock_get.return_value = mock.Mock(attachments=[ {'server_id': self.node.uuid, 'attachment_id': 'qux'}]) with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_begin.assert_called_once_with(mock.ANY, volume_id) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})
def test_detach_volumes_one_detached( self, mock_create_meta, mock_begin, mock_term, mock_detach, mock_get, mock_set_meta, mock_session): """Iterate with two volumes, one already detached.""" volume_id = '111111111-0000-0000-0000-000000000003' volumes = [volume_id, 'detached'] connector = {'foo': 'bar'} mock_create_meta.return_value = {'bar': 'baz'} mock_get.side_effect = [ mock.Mock(attachments=[ {'server_id': self.node.uuid, 'attachment_id': 'qux'}]), mock.Mock(attachments=[]) ] with task_manager.acquire(self.context, self.node.uuid) as task: cinder.detach_volumes(task, volumes, connector, allow_errors=False) mock_begin.assert_called_once_with(mock.ANY, volume_id) mock_term.assert_called_once_with(mock.ANY, volume_id, {'foo': 'bar'}) mock_detach.assert_called_once_with(mock.ANY, volume_id, 'qux') mock_set_meta.assert_called_once_with(mock.ANY, volume_id, {'bar': 'baz'})