def reserve_nodes(self, tag, nodes): # Ensure consistent sort order so we don't run into deadlocks. nodes.sort() result = [] session = get_session() with session.begin(): # TODO(deva): Optimize this by trying to reserve all the nodes # at once, and fall back to reserving one at a time # only if needed to determine the cause of an error. for node in nodes: query = model_query(models.Node, session=session) query = add_identity_filter(query, node) # Be optimistic and assume we usually get a reservation. count = query.filter_by(reservation=None).\ update({'reservation': tag}) if count != 1: try: ref = query.one() except NoResultFound: raise exception.NodeNotFound(node=node) else: raise exception.NodeLocked(node=node) ref = query.one() result.append(ref) return result
def destroy_node(self, node): session = get_session() with session.begin(): query = model_query(models.Node, session=session) query = add_identity_filter(query, node) try: node_ref = query.one() except NoResultFound: raise exception.NodeNotFound(node=node) if node_ref['reservation'] is not None: raise exception.NodeLocked(node=node) # Get node ID, if an UUID was supplied. The ID is # required for deleting all ports, attached to the node. if uuidutils.is_uuid_like(node): node_id = node_ref['id'] else: node_id = node port_query = model_query(models.Port, session=session) port_query = add_port_filter_by_node(port_query, node_id) port_query.delete() query.delete()
def test_node_locked(self, mock_check, mock_acquire): mgr = mock.Mock(spec=['iter_nodes']) mgr.iter_nodes.return_value = [('1', 'd1'), ('2', 'd2')] mock_acquire.side_effect = exception.NodeLocked("boom") discoverd.DiscoverdInspect()._periodic_check_result( mgr, mock.sentinel.context) self.assertFalse(mock_check.called) self.assertEqual(2, mock_acquire.call_count)
def test_delete_port_node_locked(self, mock_dpt): self.node.reserve(self.context, 'fake', self.node.uuid) mock_dpt.side_effect = exception.NodeLocked(node='fake-node', host='fake-host') ret = self.delete('/ports/%s' % self.port.uuid, expect_errors=True) self.assertEqual(http_client.CONFLICT, ret.status_code) self.assertTrue(ret.json['error_message']) self.assertTrue(mock_dpt.called)
def _check_port_change_forbidden(port, session): node_id = port['node_id'] if node_id is not None: query = model_query(models.Node, session=session) query = query.filter_by(id=node_id) node_ref = query.one() if node_ref['reservation'] is not None: raise exception.NodeLocked(node=node_id)
def test_node_locked(self, mock_check, mock_acquire): iter_nodes_ret = [('1', 'd1'), ('2', 'd2')] mock_acquire.side_effect = iter([exception.NodeLocked("boom")] * len(iter_nodes_ret)) mgr = mock.MagicMock(spec=['iter_nodes']) mgr.iter_nodes.return_value = iter_nodes_ret inspector.Inspector()._periodic_check_result(mgr, mock.sentinel.context) self.assertFalse(mock_check.called) self.assertEqual(2, mock_acquire.call_count)
def test_excl_lock_reserve_exception(self, get_ports_mock, get_driver_mock, reserve_mock, release_mock, node_get_mock): reserve_mock.side_effect = exception.NodeLocked(node='foo', host='foo') self.assertRaises(exception.NodeLocked, task_manager.TaskManager, self.context, 'fake-node-id') reserve_mock.assert_called_once_with(self.host, 'fake-node-id') self.assertFalse(get_ports_mock.called) self.assertFalse(get_driver_mock.called) self.assertFalse(release_mock.called) self.assertFalse(node_get_mock.called)
def test_excl_lock_exception_then_lock(self, get_ports_mock, get_driver_mock, reserve_mock, release_mock, node_get_mock): retry_attempts = 3 self.config(node_locked_retry_attempts=retry_attempts, group='conductor') # Fail on the first lock attempt, succeed on the second. reserve_mock.side_effect = [ exception.NodeLocked(node='foo', host='foo'), self.node ] with task_manager.TaskManager(self.context, 'fake-node-id') as task: self.assertFalse(task.shared) reserve_mock.assert_called(self.context, self.host, 'fake-node-id') self.assertEqual(2, reserve_mock.call_count)
def release_node(self, tag, node_id): with _session_for_write(): query = model_query(models.Node) query = add_identity_filter(query, node_id) # be optimistic and assume we usually release a reservation count = query.filter_by(reservation=tag).update( {'reservation': None}, synchronize_session=False) try: if count != 1: node = query.one() if node['reservation'] is None: raise exception.NodeNotLocked(node=node.uuid) else: raise exception.NodeLocked(node=node.uuid, host=node['reservation']) except NoResultFound: raise exception.NodeNotFound(node_id)
def reserve_node(self, tag, node_id): with _session_for_write(): query = _get_node_query_with_tags() query = add_identity_filter(query, node_id) # be optimistic and assume we usually create a reservation count = query.filter_by(reservation=None).update( {'reservation': tag}, synchronize_session=False) try: node = query.one() if count != 1: # Nothing updated and node exists. Must already be # locked. raise exception.NodeLocked(node=node.uuid, host=node['reservation']) return node except NoResultFound: raise exception.NodeNotFound(node_id)
def test_delete_volume_connector_node_locked(self, mock_notify, mock_dvc): self.node.reserve(self.context, 'fake', self.node.uuid) mock_dvc.side_effect = exception.NodeLocked(node='fake-node', host='fake-host') ret = self.delete('/volume/connectors/%s' % self.connector.uuid, expect_errors=True, headers=self.headers) self.assertEqual(http_client.CONFLICT, ret.status_code) self.assertTrue(ret.json['error_message']) self.assertTrue(mock_dvc.called) mock_notify.assert_has_calls([mock.call(mock.ANY, mock.ANY, 'delete', obj_fields.NotificationLevel.INFO, obj_fields.NotificationStatus.START, node_uuid=self.node.uuid), mock.call(mock.ANY, mock.ANY, 'delete', obj_fields.NotificationLevel.ERROR, obj_fields.NotificationStatus.ERROR, node_uuid=self.node.uuid)])
def test_excl_lock_reserve_exception(self, get_ports_mock, get_driver_mock, reserve_mock, release_mock, node_get_mock): retry_attempts = 3 self.config(node_locked_retry_attempts=retry_attempts, group='conductor') reserve_mock.side_effect = exception.NodeLocked(node='foo', host='foo') self.assertRaises(exception.NodeLocked, task_manager.TaskManager, self.context, 'fake-node-id') reserve_mock.assert_called_with(self.context, self.host, 'fake-node-id') self.assertEqual(retry_attempts, reserve_mock.call_count) self.assertFalse(get_ports_mock.called) self.assertFalse(get_driver_mock.called) self.assertFalse(release_mock.called) self.assertFalse(node_get_mock.called)
def test_excl_lock_upgade_exception_no_retries( self, get_voltgt_mock, get_volconn_mock, get_portgroups_mock, get_ports_mock, build_driver_mock, reserve_mock, release_mock, node_get_mock): retry_attempts = 3 self.config(node_locked_retry_attempts=retry_attempts, group='conductor') node_get_mock.return_value = self.node # Fail on the first lock attempt, succeed on the second. reserve_mock.side_effect = [exception.NodeLocked(node='foo', host='foo'), self.node] task = task_manager.TaskManager(self.context, 'fake-node-id', shared=True) self.assertRaises(exception.NodeLocked, task.upgrade_lock, retry=False) reserve_mock.assert_called_once_with(self.context, self.host, 'fake-node-id')
def release_nodes(self, tag, nodes): session = get_session() with session.begin(): # TODO(deva): Optimize this by trying to release all the nodes # at once, and fall back to releasing one at a time # only if needed to determine the cause of an error. for node in nodes: query = model_query(models.Node, session=session) query = add_identity_filter(query, node) # be optimistic and assume we usually release a reservation count = query.filter_by(reservation=tag).\ update({'reservation': None}) if count != 1: try: ref = query.one() except NoResultFound: raise exception.NodeNotFound(node=node) else: if ref['reservation'] is not None: raise exception.NodeLocked(node=node)
def provision(self, node_ident, target, configdrive=None): """Asynchronous trigger the provisioning of the node. This will set the target provision state of the node, and a background task will begin which actually applies the state change. This call will return a 202 (Accepted) indicating the request was accepted and is in progress; the client should continue to GET the status of this node to observe the status of the requested action. :param node_ident: UUID or logical name of a node. :param target: The desired provision state of the node. :param configdrive: Optional. A gzipped and base64 encoded configdrive. Only valid when setting provision state to "active". :raises: NodeLocked (HTTP 409) if the node is currently locked. :raises: ClientSideError (HTTP 409) if the node is already being provisioned. :raises: InvalidStateRequested (HTTP 400) if the requested transition is not possible from the current state. :raises: NotAcceptable (HTTP 406) if the API version specified does not allow the requested state transition. """ check_allow_management_verbs(target) rpc_node = api_utils.get_rpc_node(node_ident) topic = pecan.request.rpcapi.get_topic_for(rpc_node) # Normally, we let the task manager recognize and deal with # NodeLocked exceptions. However, that isn't done until the RPC calls # below. In order to main backward compatibility with our API HTTP # response codes, we have this check here to deal with cases where # a node is already being operated on (DEPLOYING or such) and we # want to continue returning 409. Without it, we'd return 400. if rpc_node.reservation: raise exception.NodeLocked(node=rpc_node.uuid, host=rpc_node.reservation) if (target in (ir_states.ACTIVE, ir_states.REBUILD) and rpc_node.maintenance): raise exception.NodeInMaintenance(op=_('provisioning'), node=rpc_node.uuid) m = ir_states.machine.copy() m.initialize(rpc_node.provision_state) if not m.is_valid_event(ir_states.VERBS.get(target, target)): raise exception.InvalidStateRequested( action=target, node=rpc_node.uuid, state=rpc_node.provision_state) if configdrive and target != ir_states.ACTIVE: msg = (_('Adding a config drive is only supported when setting ' 'provision state to %s') % ir_states.ACTIVE) raise wsme.exc.ClientSideError(msg, status_code=400) # Note that there is a race condition. The node state(s) could change # by the time the RPC call is made and the TaskManager manager gets a # lock. if target == ir_states.ACTIVE: pecan.request.rpcapi.do_node_deploy(pecan.request.context, rpc_node.uuid, False, configdrive, topic) elif target == ir_states.REBUILD: pecan.request.rpcapi.do_node_deploy(pecan.request.context, rpc_node.uuid, True, None, topic) elif target == ir_states.DELETED: pecan.request.rpcapi.do_node_tear_down( pecan.request.context, rpc_node.uuid, topic) elif target == ir_states.VERBS['inspect']: pecan.request.rpcapi.inspect_hardware( pecan.request.context, rpc_node.uuid, topic=topic) elif target in ( ir_states.VERBS['manage'], ir_states.VERBS['provide']): pecan.request.rpcapi.do_provisioning_action( pecan.request.context, rpc_node.uuid, target, topic) else: msg = (_('The requested action "%(action)s" could not be ' 'understood.') % {'action': target}) raise exception.InvalidStateRequested(message=msg) # Set the HTTP Location Header url_args = '/'.join([node_ident, 'states']) pecan.response.location = link.build_url('nodes', url_args)
def _check_node_already_locked(query, query_by): no_reserv = None locked_ref = query.filter(models.Node.reservation != no_reserv).first() if locked_ref: raise exception.NodeLocked(node=locked_ref[query_by], host=locked_ref['reservation'])