def test_random_host_selector_with_serverset(self): testutil.initialize_kazoo_client_manager(ZK_HOSTS) kazoo_client = KazooClientManager().get_client() kazoo_client.ensure_path(HostSelectorTestCase.SERVER_SET_PATH) host_provider = HostsProvider(HostSelectorTestCase.PORT_LIST, HostSelectorTestCase.SERVER_SET_PATH) self.assertTrue(host_provider.initialized) self.assertTrue(host_provider.hosts) # Since there is no live hosts in the server set, host provider should # still use the static host list. self.assertEqual(host_provider._current_host_tuple, host_provider._static_host_tuple) random_host_selector = RandomHostSelector( host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) self.assertTrue(random_host_selector.get_host() in HostSelectorTestCase.PORT_LIST) server_set = ServerSet(HostSelectorTestCase.SERVER_SET_PATH, ZK_HOSTS) g = server_set.join(HostSelectorTestCase.PORT_LIST[0], use_ip=False) g.get() no_of_iterations = 100 # After the first endpoint joins, random host selector should only # start to use hosts in the server set. returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] self.assertEqual(len(set(returned_hosts)), 1) self.assertEqual(len(host_provider.hosts), 1) g = server_set.join(HostSelectorTestCase.PORT_LIST[1], use_ip=False) g.get() # After the second endpoint joins the server set, random host selector # should return both endpoints now. returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] self.assertEqual(len(set(returned_hosts)), 2) self.assertEqual(len(host_provider.hosts), 2)
def test_random_host_selector(self): """Test the RandomHostSelector.""" host_provider = HostsProvider(HostSelectorTestCase.HOST_LIST) random_host_selector = RandomHostSelector( host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) # Note that we didn't have to mock _chose_host() call this time, # it should be im RandomHostSelector class already. some_host = random_host_selector.get_host() self.assertTrue(some_host in HostSelectorTestCase.HOST_LIST) self.assertEquals(random_host_selector._current, some_host) no_of_iterations = 250 # If I run get_host() about 100 times I expect to have relatively # even distribution and all hosts in the host_list returned by now. returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] host_counter = Counter(returned_hosts) # We expect that all calls happened. self.assertEquals(sum(host_counter.itervalues()), no_of_iterations) # We should have seen all the elements. self.assertEquals(set(host_counter), set(HostSelectorTestCase.HOST_LIST)) # But if we had left large expire_time only one host would be picked # up all the time, and we'll show that here. random_host_selector = RandomHostSelector(host_provider, invalidation_threshold=1.0) returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] host_counter = Counter(returned_hosts) self.assertEquals(len(list(host_counter)), 1) # Test invalidation hosts = [HostSelectorTestCase.HOST_LIST[0]] for i in xrange(4): hosts.append(HostSelectorTestCase.HOST_LIST[1]) def random_select(*args): return hosts.pop() mock = Mock(side_effect=random_select) with patch("random.choice", new=mock): random_host_selector = RandomHostSelector( host_provider, expire_time=0, retry_time=60, invalidation_threshold=1.0) host = random_host_selector.get_host() self.assertEqual(host, HostSelectorTestCase.HOST_LIST[1]) random_host_selector.invalidate() # Because mock will return the bad host three times in a row, # this will force it to compute the set of good hosts host = random_host_selector.get_host() self.assertEqual(host, HostSelectorTestCase.HOST_LIST[0]) # At this point, random.choice should have been called 5 times self.assertEqual(mock.call_count, 5)
def test_random_host_selector_with_serverset(self): fd, tmp_file = tempfile.mkstemp() # Add a new host into the local server set file to simulate a join f = open(tmp_file, 'w') f.write(HostSelectorWithLocalFileTestCase.HOST_LIST[0]) f.close() HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates() host_provider = HostsProvider( HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file) self.assertTrue(host_provider.initialized) self.assertTrue(host_provider.hosts) self.assertEqual(host_provider._current_host_tuple, (HostSelectorWithLocalFileTestCase.HOST_LIST[0], )) random_host_selector = RandomHostSelector(host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) self.assertTrue(random_host_selector.get_host() in HostSelectorWithLocalFileTestCase.HOST_LIST) no_of_iterations = 100 # After the first endpoint joins, random host selector should only # start to use hosts in the server set. returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] self.assertEqual(len(set(returned_hosts)), 1) self.assertEqual(len(host_provider.hosts), 1) time.sleep(1) f = open(tmp_file, 'a') f.write('\n' + HostSelectorWithLocalFileTestCase.HOST_LIST[1]) f.close() HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates() # After the second endpoint joins the server set, random host selector # should return both endpoints now. returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] self.assertEqual(len(set(returned_hosts)), 2) self.assertEqual(len(host_provider.hosts), 2) HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches() os.remove(tmp_file)
def test_random_host_selector_with_serverset(self): fd, tmp_file = tempfile.mkstemp() # Add a new host into the local server set file to simulate a join f = open(tmp_file, 'w') f.write(HostSelectorWithLocalFileTestCase.HOST_LIST[0]) f.close() HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates() host_provider = HostsProvider( HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file) self.assertTrue(host_provider.initialized) self.assertTrue(host_provider.hosts) self.assertEqual(host_provider._current_host_tuple, (HostSelectorWithLocalFileTestCase.HOST_LIST[0],)) random_host_selector = RandomHostSelector( host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) self.assertTrue(random_host_selector.get_host() in HostSelectorWithLocalFileTestCase.HOST_LIST) no_of_iterations = 100 # After the first endpoint joins, random host selector should only # start to use hosts in the server set. returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] self.assertEqual(len(set(returned_hosts)), 1) self.assertEqual(len(host_provider.hosts), 1) time.sleep(1) f = open(tmp_file, 'a') f.write('\n' + HostSelectorWithLocalFileTestCase.HOST_LIST[1]) f.close() HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates() # After the second endpoint joins the server set, random host selector # should return both endpoints now. returned_hosts = [random_host_selector.get_host() for i in xrange(no_of_iterations)] self.assertEqual(len(set(returned_hosts)), 2) self.assertEqual(len(host_provider.hosts), 2) HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches() os.remove(tmp_file)
def test_random_host_selector_with_serverset(self): testutil.initialize_kazoo_client_manager(ZK_HOSTS) kazoo_client = KazooClientManager().get_client() kazoo_client.ensure_path(HostSelectorTestCase.SERVER_SET_PATH) host_provider = HostsProvider(HostSelectorTestCase.PORT_LIST, HostSelectorTestCase.SERVER_SET_PATH) self.assertTrue(host_provider.initialized) self.assertTrue(host_provider.hosts) # Since there is no live hosts in the server set, host provider should # still use the static host list. self.assertEqual(host_provider._current_host_tuple, host_provider._static_host_tuple) random_host_selector = RandomHostSelector(host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) self.assertTrue( random_host_selector.get_host() in HostSelectorTestCase.PORT_LIST) server_set = ServerSet(HostSelectorTestCase.SERVER_SET_PATH, ZK_HOSTS) g = server_set.join(HostSelectorTestCase.PORT_LIST[0], use_ip=False) g.get() no_of_iterations = 100 # After the first endpoint joins, random host selector should only # start to use hosts in the server set. returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] self.assertEqual(len(set(returned_hosts)), 1) self.assertEqual(len(host_provider.hosts), 1) g = server_set.join(HostSelectorTestCase.PORT_LIST[1], use_ip=False) g.get() # After the second endpoint joins the server set, random host selector # should return both endpoints now. returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] self.assertEqual(len(set(returned_hosts)), 2) self.assertEqual(len(host_provider.hosts), 2)
def test_random_host_selector(self): """Test the RandomHostSelector.""" fd, tmp_file = tempfile.mkstemp() with open(tmp_file, 'w') as f: f.write('\n'.join(HostSelectorWithLocalFileTestCase.HOST_LIST)) host_provider = HostsProvider( HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file) random_host_selector = RandomHostSelector(host_provider, expire_time=0, retry_time=0, invalidation_threshold=1.0) # Note that we didn't have to mock _chose_host() call this time, # it should be im RandomHostSelector class already. some_host = random_host_selector.get_host() self.assertTrue( some_host in HostSelectorWithLocalFileTestCase.HOST_LIST) self.assertEquals(random_host_selector._current, some_host) no_of_iterations = 250 # If I run get_host() about 100 times I expect to have relatively # even distribution and all hosts in the host_list returned by now. returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] host_counter = Counter(returned_hosts) # We expect that all calls happened. self.assertEquals(sum(host_counter.itervalues()), no_of_iterations) # We should have seen all the elements. self.assertEquals(set(host_counter), set(HostSelectorWithLocalFileTestCase.HOST_LIST)) # But if we had left large expire_time only one host would be picked # up all the time, and we'll show that here. random_host_selector = RandomHostSelector(host_provider, invalidation_threshold=1.0) returned_hosts = [ random_host_selector.get_host() for i in xrange(no_of_iterations) ] host_counter = Counter(returned_hosts) self.assertEquals(len(list(host_counter)), 1) # Test invalidation hosts = [HostSelectorWithLocalFileTestCase.HOST_LIST[0]] for i in xrange(4): hosts.append(HostSelectorWithLocalFileTestCase.HOST_LIST[1]) def random_select(*args): return hosts.pop() mock = Mock(side_effect=random_select) with patch("random.choice", new=mock): random_host_selector = RandomHostSelector( host_provider, expire_time=0, retry_time=60, invalidation_threshold=1.0) host = random_host_selector.get_host() self.assertEqual(host, HostSelectorWithLocalFileTestCase.HOST_LIST[1]) random_host_selector.invalidate() # Because mock will return the bad host three times in a row, # this will force it to compute the set of good hosts host = random_host_selector.get_host() self.assertEqual(host, HostSelectorWithLocalFileTestCase.HOST_LIST[0]) # At this point, random.choice should have been called 5 times self.assertEqual(mock.call_count, 5) HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches() os.remove(tmp_file)