def _create_workers(self, row_event, worker_num): self.mech_driver.nb_ovn.idl.notify_handler.watch_event(row_event) worker_list = [self.mech_driver.nb_ovn] # Create 10 fake workers for _ in range(worker_num): node_uuid = uuidutils.generate_uuid() db_hash_ring.add_node(self.context, ovn_const.HASH_RING_ML2_GROUP, node_uuid) fake_driver = mock.MagicMock( node_uuid=node_uuid, hash_ring_group=ovn_const.HASH_RING_ML2_GROUP) _idl = ovsdb_monitor.OvnNbIdl.from_server( self.ovsdb_server_mgr.get_ovsdb_connection_path(), self.nb_api.schema_helper, fake_driver) worker = self.useFixture( fixtures.OVNIdlConnectionFixture(idl=_idl, timeout=10)).connection worker.idl.notify_handler.watch_event(row_event) worker.start() worker_list.append(worker) # Refresh the hash rings just in case [worker.idl._hash_ring.refresh() for worker in worker_list] # Assert we have 11 active workers in the ring self.assertEqual( worker_num + 1, len( db_hash_ring.get_active_nodes( self.context, interval=ovn_const.HASH_RING_NODES_TIMEOUT, group_name=ovn_const.HASH_RING_ML2_GROUP))) return worker_list
def test_ring_rebalance(self): # Use pre-defined UUIDs to make the hashes predictable node_1_uuid = db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-1') node_2_uuid = db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-2') # Add another node from a different host with mock.patch.object(db_hash_ring, 'CONF') as mock_conf: mock_conf.host = 'another-host-52359446-c366' another_host_node = db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'another-host') # Assert all nodes are alive in the ring self.hash_ring_manager.refresh() self.assertEqual(3, len(self.hash_ring_manager._hash_ring.nodes)) # Hash certain values against the nodes hash_dict_before = { node_1_uuid: 'fake-uuid', node_2_uuid: 'fake-uuid-0', another_host_node: 'fake-uuid-ABCDE' } self._verify_hashes(hash_dict_before) # Mock utcnow() as the HASH_RING_NODES_TIMEOUT have expired # already and touch the nodes from our host fake_utcnow = timeutils.utcnow() - datetime.timedelta( seconds=constants.HASH_RING_NODES_TIMEOUT) with mock.patch.object(timeutils, 'utcnow') as mock_utcnow: mock_utcnow.return_value = fake_utcnow db_hash_ring.touch_nodes_from_host(self.admin_ctx, HASH_RING_TEST_GROUP) # Now assert that the ring was re-balanced and only the node from # another host is marked as alive self.hash_ring_manager.refresh() self.assertEqual([another_host_node], list(self.hash_ring_manager._hash_ring.nodes.keys())) # Now only "another_host_node" is alive, all values should hash to it hash_dict_after_rebalance = { another_host_node: 'fake-uuid', another_host_node: 'fake-uuid-0', another_host_node: 'fake-uuid-ABCDE' } self._verify_hashes(hash_dict_after_rebalance) # Now touch the nodes so they appear active again db_hash_ring.touch_nodes_from_host(self.admin_ctx, HASH_RING_TEST_GROUP) self.hash_ring_manager.refresh() # The ring should re-balance and as it was before self._verify_hashes(hash_dict_before)
def test_get_node(self): # Use pre-defined UUIDs to make the hashes predictable node_1_uuid = db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-1') node_2_uuid = db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-2') hash_dict_before = { node_1_uuid: 'fake-uuid', node_2_uuid: 'fake-uuid-0' } self._verify_hashes(hash_dict_before)
def test_distributed_lock(self): api_workers = 11 cfg.CONF.set_override('api_workers', api_workers) row_event = DistributedLockTestEvent() self.mech_driver._nb_ovn.idl.notify_handler.watch_event(row_event) worker_list = [ self.mech_driver._nb_ovn, ] # Create 10 fake workers for _ in range(api_workers - len(worker_list)): node_uuid = uuidutils.generate_uuid() db_hash_ring.add_node(self.context, ovn_const.HASH_RING_ML2_GROUP, node_uuid) fake_driver = mock.MagicMock( node_uuid=node_uuid, hash_ring_group=ovn_const.HASH_RING_ML2_GROUP) _idl = ovsdb_monitor.OvnNbIdl.from_server( self.ovsdb_server_mgr.get_ovsdb_connection_path(), 'OVN_Northbound', fake_driver) worker = self.useFixture( fixtures.OVNIdlConnectionFixture(idl=_idl, timeout=10)).connection worker.idl.notify_handler.watch_event(row_event) worker.start() worker_list.append(worker) # Refresh the hash rings just in case [worker.idl._hash_ring.refresh() for worker in worker_list] # Assert we have 11 active workers in the ring self.assertEqual( 11, len( db_hash_ring.get_active_nodes( self.context, interval=ovn_const.HASH_RING_NODES_TIMEOUT, group_name=ovn_const.HASH_RING_ML2_GROUP))) # Trigger the event self.create_port() # Wait for the event to complete self.assertTrue(row_event.wait()) # Assert that only one worker handled the event self.assertEqual(1, row_event.COUNTER)
def _add_nodes_and_assert_exists(self, count=1, group_name=HASH_RING_TEST_GROUP): nodes = [] for i in range(count): node_uuid = ovn_hash_ring_db.add_node(self.admin_ctx, group_name) self.assertIsNotNone(self._get_node_row(node_uuid)) nodes.append(node_uuid) return nodes
def test__wait_startup_before_caching(self, api_workers): db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-1') # Assert it will return True until until we equal api_workers self.assertTrue(self.hash_ring_manager._wait_startup_before_caching) self.assertTrue(self.hash_ring_manager._check_hashring_startup) db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-2') # Assert it's now False. Waiting is not needed anymore self.assertFalse(self.hash_ring_manager._wait_startup_before_caching) self.assertFalse(self.hash_ring_manager._check_hashring_startup) # Now assert that since the _check_hashring_startup has been # flipped, we no longer will read from the database with mock.patch.object(hash_ring_manager.db_hash_ring, 'get_active_nodes') as get_nodes_mock: self.assertFalse( self.hash_ring_manager._wait_startup_before_caching) self.assertFalse(get_nodes_mock.called)
def test__wait_startup_before_caching(self): db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-1') db_hash_ring.add_node(self.admin_ctx, HASH_RING_TEST_GROUP, 'node-2') # Assert it will return True until created_at != updated_at self.assertTrue(self.hash_ring_manager._wait_startup_before_caching) self.assertTrue(self.hash_ring_manager._cache_startup_timeout) # Touch the nodes (== update the updated_at column) db_hash_ring.touch_nodes_from_host(self.admin_ctx, HASH_RING_TEST_GROUP) # Assert it's now False. Waiting is not needed anymore self.assertFalse(self.hash_ring_manager._wait_startup_before_caching) self.assertFalse(self.hash_ring_manager._cache_startup_timeout) # Now assert that since the _cache_startup_timeout has been # flipped, we no longer will read from the database with mock.patch.object(hash_ring_manager.db_hash_ring, 'get_active_nodes') as get_nodes_mock: self.assertFalse( self.hash_ring_manager._wait_startup_before_caching) self.assertFalse(get_nodes_mock.called)