def test_random_host_selector_with_serverset(self):
testutil.initialize_kazoo_client_manager(ZK_HOSTS)
kazoo_client = KazooClientManager().get_client()
kazoo_client.ensure_path(HostSelectorTestCase.SERVER_SET_PATH)
host_provider = HostsProvider(HostSelectorTestCase.PORT_LIST,
HostSelectorTestCase.SERVER_SET_PATH)
self.assertTrue(host_provider.initialized)
self.assertTrue(host_provider.hosts)
# Since there is no live hosts in the server set, host provider should
# still use the static host list.
self.assertEqual(host_provider._current_host_tuple,
host_provider._static_host_tuple)
random_host_selector = RandomHostSelector(
host_provider, expire_time=0, retry_time=0,
invalidation_threshold=1.0)
self.assertTrue(random_host_selector.get_host() in
HostSelectorTestCase.PORT_LIST)
server_set = ServerSet(HostSelectorTestCase.SERVER_SET_PATH, ZK_HOSTS)
g = server_set.join(HostSelectorTestCase.PORT_LIST[0], use_ip=False)
g.get()
no_of_iterations = 100
# After the first endpoint joins, random host selector should only
# start to use hosts in the server set.
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
self.assertEqual(len(set(returned_hosts)), 1)
self.assertEqual(len(host_provider.hosts), 1)
g = server_set.join(HostSelectorTestCase.PORT_LIST[1], use_ip=False)
g.get()
# After the second endpoint joins the server set, random host selector
# should return both endpoints now.
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
self.assertEqual(len(set(returned_hosts)), 2)
self.assertEqual(len(host_provider.hosts), 2)
def test_random_host_selector(self):
"""Test the RandomHostSelector."""
host_provider = HostsProvider(HostSelectorTestCase.HOST_LIST)
random_host_selector = RandomHostSelector(
host_provider, expire_time=0, retry_time=0,
invalidation_threshold=1.0)
# Note that we didn't have to mock _chose_host() call this time,
# it should be im RandomHostSelector class already.
some_host = random_host_selector.get_host()
self.assertTrue(some_host in HostSelectorTestCase.HOST_LIST)
self.assertEquals(random_host_selector._current, some_host)
no_of_iterations = 250
# If I run get_host() about 100 times I expect to have relatively
# even distribution and all hosts in the host_list returned by now.
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
host_counter = Counter(returned_hosts)
# We expect that all calls happened.
self.assertEquals(sum(host_counter.itervalues()), no_of_iterations)
# We should have seen all the elements.
self.assertEquals(set(host_counter),
set(HostSelectorTestCase.HOST_LIST))
# But if we had left large expire_time only one host would be picked
# up all the time, and we'll show that here.
random_host_selector = RandomHostSelector(host_provider,
invalidation_threshold=1.0)
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
host_counter = Counter(returned_hosts)
self.assertEquals(len(list(host_counter)), 1)
# Test invalidation
hosts = [HostSelectorTestCase.HOST_LIST[0]]
for i in xrange(4):
hosts.append(HostSelectorTestCase.HOST_LIST[1])
def random_select(*args):
return hosts.pop()
mock = Mock(side_effect=random_select)
with patch("random.choice", new=mock):
random_host_selector = RandomHostSelector(
host_provider, expire_time=0, retry_time=60,
invalidation_threshold=1.0)
host = random_host_selector.get_host()
self.assertEqual(host, HostSelectorTestCase.HOST_LIST[1])
random_host_selector.invalidate()
# Because mock will return the bad host three times in a row,
# this will force it to compute the set of good hosts
host = random_host_selector.get_host()
self.assertEqual(host, HostSelectorTestCase.HOST_LIST[0])
# At this point, random.choice should have been called 5 times
self.assertEqual(mock.call_count, 5)
def test_random_host_selector_with_serverset(self):
fd, tmp_file = tempfile.mkstemp()
# Add a new host into the local server set file to simulate a join
f = open(tmp_file, 'w')
f.write(HostSelectorWithLocalFileTestCase.HOST_LIST[0])
f.close()
HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates()
host_provider = HostsProvider(
HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file)
self.assertTrue(host_provider.initialized)
self.assertTrue(host_provider.hosts)
self.assertEqual(host_provider._current_host_tuple,
(HostSelectorWithLocalFileTestCase.HOST_LIST[0], ))
random_host_selector = RandomHostSelector(host_provider,
expire_time=0,
retry_time=0,
invalidation_threshold=1.0)
self.assertTrue(random_host_selector.get_host() in
HostSelectorWithLocalFileTestCase.HOST_LIST)
no_of_iterations = 100
# After the first endpoint joins, random host selector should only
# start to use hosts in the server set.
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
self.assertEqual(len(set(returned_hosts)), 1)
self.assertEqual(len(host_provider.hosts), 1)
time.sleep(1)
f = open(tmp_file, 'a')
f.write('\n' + HostSelectorWithLocalFileTestCase.HOST_LIST[1])
f.close()
HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates()
# After the second endpoint joins the server set, random host selector
# should return both endpoints now.
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
self.assertEqual(len(set(returned_hosts)), 2)
self.assertEqual(len(host_provider.hosts), 2)
HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)
def test_random_host_selector_with_serverset(self):
fd, tmp_file = tempfile.mkstemp()
# Add a new host into the local server set file to simulate a join
f = open(tmp_file, 'w')
f.write(HostSelectorWithLocalFileTestCase.HOST_LIST[0])
f.close()
HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates()
host_provider = HostsProvider(
HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file)
self.assertTrue(host_provider.initialized)
self.assertTrue(host_provider.hosts)
self.assertEqual(host_provider._current_host_tuple,
(HostSelectorWithLocalFileTestCase.HOST_LIST[0],))
random_host_selector = RandomHostSelector(
host_provider, expire_time=0, retry_time=0,
invalidation_threshold=1.0)
self.assertTrue(random_host_selector.get_host() in
HostSelectorWithLocalFileTestCase.HOST_LIST)
no_of_iterations = 100
# After the first endpoint joins, random host selector should only
# start to use hosts in the server set.
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
self.assertEqual(len(set(returned_hosts)), 1)
self.assertEqual(len(host_provider.hosts), 1)
time.sleep(1)
f = open(tmp_file, 'a')
f.write('\n' + HostSelectorWithLocalFileTestCase.HOST_LIST[1])
f.close()
HostSelectorWithLocalFileTestCase.FILE_WATCH._check_file_updates()
# After the second endpoint joins the server set, random host selector
# should return both endpoints now.
returned_hosts = [random_host_selector.get_host()
for i in xrange(no_of_iterations)]
self.assertEqual(len(set(returned_hosts)), 2)
self.assertEqual(len(host_provider.hosts), 2)
HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)
def test_random_host_selector_with_serverset(self):
testutil.initialize_kazoo_client_manager(ZK_HOSTS)
kazoo_client = KazooClientManager().get_client()
kazoo_client.ensure_path(HostSelectorTestCase.SERVER_SET_PATH)
host_provider = HostsProvider(HostSelectorTestCase.PORT_LIST,
HostSelectorTestCase.SERVER_SET_PATH)
self.assertTrue(host_provider.initialized)
self.assertTrue(host_provider.hosts)
# Since there is no live hosts in the server set, host provider should
# still use the static host list.
self.assertEqual(host_provider._current_host_tuple,
host_provider._static_host_tuple)
random_host_selector = RandomHostSelector(host_provider,
expire_time=0,
retry_time=0,
invalidation_threshold=1.0)
self.assertTrue(
random_host_selector.get_host() in HostSelectorTestCase.PORT_LIST)
server_set = ServerSet(HostSelectorTestCase.SERVER_SET_PATH, ZK_HOSTS)
g = server_set.join(HostSelectorTestCase.PORT_LIST[0], use_ip=False)
g.get()
no_of_iterations = 100
# After the first endpoint joins, random host selector should only
# start to use hosts in the server set.
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
self.assertEqual(len(set(returned_hosts)), 1)
self.assertEqual(len(host_provider.hosts), 1)
g = server_set.join(HostSelectorTestCase.PORT_LIST[1], use_ip=False)
g.get()
# After the second endpoint joins the server set, random host selector
# should return both endpoints now.
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
self.assertEqual(len(set(returned_hosts)), 2)
self.assertEqual(len(host_provider.hosts), 2)
def test_random_host_selector(self):
"""Test the RandomHostSelector."""
fd, tmp_file = tempfile.mkstemp()
with open(tmp_file, 'w') as f:
f.write('\n'.join(HostSelectorWithLocalFileTestCase.HOST_LIST))
host_provider = HostsProvider(
HostSelectorWithLocalFileTestCase.HOST_LIST, file_path=tmp_file)
random_host_selector = RandomHostSelector(host_provider,
expire_time=0,
retry_time=0,
invalidation_threshold=1.0)
# Note that we didn't have to mock _chose_host() call this time,
# it should be im RandomHostSelector class already.
some_host = random_host_selector.get_host()
self.assertTrue(
some_host in HostSelectorWithLocalFileTestCase.HOST_LIST)
self.assertEquals(random_host_selector._current, some_host)
no_of_iterations = 250
# If I run get_host() about 100 times I expect to have relatively
# even distribution and all hosts in the host_list returned by now.
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
host_counter = Counter(returned_hosts)
# We expect that all calls happened.
self.assertEquals(sum(host_counter.itervalues()), no_of_iterations)
# We should have seen all the elements.
self.assertEquals(set(host_counter),
set(HostSelectorWithLocalFileTestCase.HOST_LIST))
# But if we had left large expire_time only one host would be picked
# up all the time, and we'll show that here.
random_host_selector = RandomHostSelector(host_provider,
invalidation_threshold=1.0)
returned_hosts = [
random_host_selector.get_host() for i in xrange(no_of_iterations)
]
host_counter = Counter(returned_hosts)
self.assertEquals(len(list(host_counter)), 1)
# Test invalidation
hosts = [HostSelectorWithLocalFileTestCase.HOST_LIST[0]]
for i in xrange(4):
hosts.append(HostSelectorWithLocalFileTestCase.HOST_LIST[1])
def random_select(*args):
return hosts.pop()
mock = Mock(side_effect=random_select)
with patch("random.choice", new=mock):
random_host_selector = RandomHostSelector(
host_provider,
expire_time=0,
retry_time=60,
invalidation_threshold=1.0)
host = random_host_selector.get_host()
self.assertEqual(host,
HostSelectorWithLocalFileTestCase.HOST_LIST[1])
random_host_selector.invalidate()
# Because mock will return the bad host three times in a row,
# this will force it to compute the set of good hosts
host = random_host_selector.get_host()
self.assertEqual(host,
HostSelectorWithLocalFileTestCase.HOST_LIST[0])
# At this point, random.choice should have been called 5 times
self.assertEqual(mock.call_count, 5)
HostSelectorWithLocalFileTestCase.FILE_WATCH._clear_all_watches()
os.remove(tmp_file)