def test_get_lock_concurrency_locking2(self):
# NOTE(sileht): some database based lock can have only
# one lock per connection, this test ensures acquiring a
# second lock doesn't release the first one.
lock1 = self._coord.get_lock(tests.get_random_uuid())
lock2 = self._coord.get_lock(tests.get_random_uuid())
graceful_ending = threading.Event()
thread_locked = threading.Event()
def thread():
with lock2:
try:
self.assertFalse(lock1.acquire(blocking=False))
except tooz.NotImplemented:
pass
thread_locked.set()
graceful_ending.set()
t = threading.Thread(target=thread)
t.daemon = True
with lock1:
t.start()
thread_locked.wait()
self.assertTrue(thread_locked.is_set())
t.join()
graceful_ending.wait()
self.assertTrue(graceful_ending.is_set())
def test_get_lock_concurrency_locking2(self):
# NOTE(sileht): some database based lock can have only
# one lock per connection, this test ensures acquiring a
# second lock doesn't release the first one.
lock1 = self._coord.get_lock(tests.get_random_uuid())
lock2 = self._coord.get_lock(tests.get_random_uuid())
graceful_ending = threading.Event()
thread_locked = threading.Event()
def thread():
with lock2:
try:
self.assertFalse(lock1.acquire(blocking=False))
except tooz.NotImplemented:
pass
thread_locked.set()
graceful_ending.set()
t = threading.Thread(target=thread)
t.daemon = True
with lock1:
t.start()
thread_locked.wait()
self.assertTrue(thread_locked.is_set())
t.join()
graceful_ending.wait()
self.assertTrue(graceful_ending.is_set())
def test_client_failure_leave(self, mock_client_cls):
mock_client = mock.MagicMock()
mock_client_cls.return_value = mock_client
member_id = tests.get_random_uuid()
coord = coordination.get_coordinator(self.FAKE_URL, member_id)
coord.start()
mock_client.gets.side_effect = socket.timeout('timed-out')
fut = coord.leave_group(tests.get_random_uuid())
self.assertRaises(coordination.ToozConnectionError, fut.get)
def test_client_failure_leave(self, mock_client_cls):
mock_client = mock.MagicMock()
mock_client_cls.return_value = mock_client
member_id = tests.get_random_uuid()
coord = coordination.get_coordinator(self.FAKE_URL, member_id)
coord.start()
mock_client.gets.side_effect = socket.timeout('timed-out')
fut = coord.leave_group(tests.get_random_uuid())
self.assertRaises(coordination.ToozConnectionError, fut.get)
def _do_test_get_lock_concurrency_locking_two_lock(self, executor,
use_same_coord):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
with lock1:
with executor(max_workers=1) as e:
coord = self._coord if use_same_coord else None
f = e.submit(try_to_lock_job, name, coord, self.url,
tests.get_random_uuid())
self.assertFalse(f.result())
def _do_test_get_lock_concurrency_locking_two_lock(self, executor,
use_same_coord):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
with lock1:
with executor(max_workers=1) as e:
coord = self._coord if use_same_coord else None
f = e.submit(try_to_lock_job, name, coord, self.url,
tests.get_random_uuid())
self.assertFalse(f.result())
def setUp(self):
super(TestAPI, self).setUp()
if self.url is None:
self.skipTest("No URL set for this driver")
self.useFixture(fixtures.NestedTempfile())
self.group_id = tests.get_random_uuid()
self.member_id = tests.get_random_uuid()
self._coord = tooz.coordination.get_coordinator(self.url,
self.member_id)
self._coord.start()
def _do_test_get_lock_serial_locking_two_lock(self, executor,
use_same_coord):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock1.acquire()
lock1.release()
with executor(max_workers=1) as e:
coord = self._coord if use_same_coord else None
f = e.submit(try_to_lock_job, name, coord, self.url,
tests.get_random_uuid())
self.assertTrue(f.result())
def test_get_members(self):
group_id_test2 = tests.get_random_uuid()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
self._coord.create_group(group_id_test2).get()
self._coord.join_group(group_id_test2).get()
client2.join_group(group_id_test2).get()
members_ids = self._coord.get_members(group_id_test2).get()
self.assertEqual({self.member_id, member_id_test2}, members_ids)
def _do_test_get_lock_serial_locking_two_lock(self, executor,
use_same_coord):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock1.acquire()
lock1.release()
with executor(max_workers=1) as e:
coord = self._coord if use_same_coord else None
f = e.submit(try_to_lock_job, name, coord, self.url,
tests.get_random_uuid())
self.assertTrue(f.result())
def test_get_members(self):
group_id_test2 = tests.get_random_uuid()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
self._coord.create_group(group_id_test2).get()
self._coord.join_group(group_id_test2).get()
client2.join_group(group_id_test2).get()
members_ids = self._coord.get_members(group_id_test2).get()
self.assertEqual({self.member_id, member_id_test2}, members_ids)
def test_get_lock_locking_exclusive_and_shared(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
coord = tooz.coordination.get_coordinator(
self.url, tests.get_random_uuid())
coord.start()
lock2 = coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(shared=True, blocking=False))
self.assertTrue(lock1.release())
self.assertFalse(lock2.release())
def test_get_lock_locking_exclusive_and_shared(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
coord = tooz.coordination.get_coordinator(self.url,
tests.get_random_uuid())
coord.start()
lock2 = coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(shared=True, blocking=False))
self.assertTrue(lock1.release())
self.assertFalse(lock2.release())
def test_get_lock_multiple_coords(self):
member_id2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id2)
client2.start()
lock_name = tests.get_random_uuid()
lock = self._coord.get_lock(lock_name)
self.assertTrue(lock.acquire())
lock2 = client2.get_lock(lock_name)
self.assertFalse(lock2.acquire(blocking=False))
self.assertTrue(lock.release())
self.assertTrue(lock2.acquire(blocking=True))
self.assertTrue(lock2.release())
def test_get_lock_multiple_coords(self):
member_id2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id2)
client2.start()
lock_name = tests.get_random_uuid()
lock = self._coord.get_lock(lock_name)
self.assertTrue(lock.acquire())
lock2 = client2.get_lock(lock_name)
self.assertFalse(lock2.acquire(blocking=False))
self.assertTrue(lock.release())
self.assertTrue(lock2.acquire(blocking=True))
self.assertTrue(lock2.release())
def test_get_member_info_nonjoined_member(self):
self._coord.create_group(self.group_id).get()
member_id = tests.get_random_uuid()
member_info = self._coord.get_member_info(self.group_id,
member_id)
self.assertRaises(tooz.coordination.MemberNotJoined,
member_info.get)
def test_watch_join_group_booted_out(self):
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
self._coord.watch_join_group(self.group_id, self._set_event)
self._coord.watch_leave_group(self.group_id, self._set_event)
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
client2.join_group(self.group_id).get()
while True:
if self._coord.run_watchers():
break
client3 = tooz.coordination.get_coordinator(self.url, self.member_id)
client3.start()
client3.leave_group(self.group_id).get()
# Only works for clients that have access to the groups they are part
# of, to ensure that after we got booted out by client3 that this
# client now no longer believes its part of the group.
if (hasattr(self._coord, '_joined_groups')
and (self._coord.run_watchers
== tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers)): # noqa
self.assertIn(self.group_id, self._coord._joined_groups)
self._coord.run_watchers()
self.assertNotIn(self.group_id, self._coord._joined_groups)
def test_watch_join_group_disappear(self):
if not hasattr(self._coord, '_destroy_group'):
self.skipTest("This test only works with coordinators"
" that have the ability to destroy groups.")
self._coord.create_group(self.group_id).get()
self._coord.watch_join_group(self.group_id, self._set_event)
self._coord.watch_leave_group(self.group_id, self._set_event)
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
client2.join_group(self.group_id).get()
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup)
self.events = []
# Force the group to disappear...
self._coord._destroy_group(self.group_id)
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberLeftGroup)
def test_watch_group_join(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
# Watch the group
self._coord.watch_join_group(self.group_id, self._set_event)
# Join the group
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(member_id_test2, members_ids)
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
# Stop watching
self._coord.unwatch_join_group(self.group_id, self._set_event)
self.events = []
# Leave and rejoin group
client2.leave_group(self.group_id).get()
client2.join_group(self.group_id).get()
self._coord.run_watchers()
self.assertEqual([], self.events)
def test_get_member_info_nonjoined_member(self):
self._coord.create_group(self.group_id).get()
member_id = tests.get_random_uuid()
member_info = self._coord.get_member_info(self.group_id,
member_id)
self.assertRaises(tooz.coordination.MemberNotJoined,
member_info.get)
def test_get_groups(self):
groups_ids = [tests.get_random_uuid() for _ in range(0, 5)]
for group_id in groups_ids:
self._coord.create_group(group_id).get()
created_groups = self._coord.get_groups().get()
for group_id in groups_ids:
self.assertIn(group_id, created_groups)
def test_watch_join_group_disappear(self):
if not hasattr(self._coord, '_destroy_group'):
self.skipTest("This test only works with coordinators"
" that have the ability to destroy groups.")
self._coord.create_group(self.group_id).get()
self._coord.watch_join_group(self.group_id, self._set_event)
self._coord.watch_leave_group(self.group_id, self._set_event)
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
client2.join_group(self.group_id).get()
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup)
self.events = []
# Force the group to disappear...
self._coord._destroy_group(self.group_id)
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberLeftGroup)
def test_watch_group_join(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
# Watch the group
self._coord.watch_join_group(self.group_id, self._set_event)
# Join the group
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(member_id_test2, members_ids)
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
# Stop watching
self._coord.unwatch_join_group(self.group_id, self._set_event)
self.events = []
# Leave and rejoin group
client2.leave_group(self.group_id).get()
client2.join_group(self.group_id).get()
self._coord.run_watchers()
self.assertEqual([], self.events)
def test_run_for_election_multiple_clients(self):
self._coord.create_group(self.group_id).get()
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
self._coord.run_watchers()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
client2.watch_elected_as_leader(self.group_id, self._set_event)
client2.run_watchers()
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.LeaderElected)
self.assertEqual(self.member_id, event.member_id)
self.assertEqual(self.group_id, event.group_id)
self.assertEqual(
self._coord.get_leader(self.group_id).get(), self.member_id)
self.events = []
self._coord.stop()
client2.run_watchers()
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.LeaderElected)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
self.assertEqual(
client2.get_leader(self.group_id).get(), member_id_test2)
# Restart the coord because tearDown stops it
self._coord.start()
def test_watch_join_group_booted_out(self):
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
self._coord.watch_join_group(self.group_id, self._set_event)
self._coord.watch_leave_group(self.group_id, self._set_event)
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
client2.join_group(self.group_id).get()
while True:
if self._coord.run_watchers():
break
client3 = tooz.coordination.get_coordinator(self.url, self.member_id)
client3.start()
client3.leave_group(self.group_id).get()
# Only works for clients that have access to the groups they are part
# of, to ensure that after we got booted out by client3 that this
# client now no longer believes its part of the group.
if (hasattr(self._coord, '_joined_groups') and
(self._coord.run_watchers == tooz.coordination.
CoordinationDriverCachedRunWatchers.run_watchers)): # noqa
self.assertIn(self.group_id, self._coord._joined_groups)
self._coord.run_watchers()
self.assertNotIn(self.group_id, self._coord._joined_groups)
def test_get_groups(self):
groups_ids = [tests.get_random_uuid() for _ in range(0, 5)]
for group_id in groups_ids:
self._coord.create_group(group_id).get()
created_groups = self._coord.get_groups().get()
for group_id in groups_ids:
self.assertIn(group_id, created_groups)
def test_lock_context_manager_acquire_argument(self):
name = tests.get_random_uuid()
blocking_value = 10.12
lock = self._coord.get_lock(name)
with mock.patch.object(lock, 'acquire', wraps=True, autospec=True) as \
mock_acquire:
with lock(blocking_value):
mock_acquire.assert_called_once_with(blocking_value)
def test_get_lock_context_fails(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
with mock.patch.object(lock2, 'acquire', return_value=False):
with lock1:
self.assertRaises(tooz.coordination.LockAcquireFailed,
lock2.__enter__)
def _add_members(self, number_of_members, weight=1):
for _ in six.moves.range(number_of_members):
m = tests.get_random_uuid()
coord = coordination.get_coordinator(self.url, m)
coord.start()
coord.join_partitioned_group(self.group_id, weight=weight)
self._extra_coords.append(coord)
self._coord.run_watchers()
def test_get_multiple_locks_with_same_coord(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(blocking=False))
self.assertFalse(self._coord.get_lock(name).acquire(blocking=False))
self.assertTrue(lock1.release())
def test_get_multiple_locks_with_same_coord(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(blocking=False))
self.assertFalse(self._coord.get_lock(name).acquire(blocking=False))
self.assertTrue(lock1.release())
def test_lock_context_manager_acquire_argument(self):
name = tests.get_random_uuid()
blocking_value = 10.12
lock = self._coord.get_lock(name)
with mock.patch.object(lock, 'acquire', wraps=True, autospec=True) as \
mock_acquire:
with lock(blocking_value):
mock_acquire.assert_called_once_with(blocking_value)
def test_client_run_watchers_mixin(self, mock_client_cls,
mock_run_watchers):
mock_client = mock.MagicMock()
mock_client_cls.return_value = mock_client
member_id = tests.get_random_uuid()
coord = coordination.get_coordinator(self.FAKE_URL, member_id)
coord.start()
coord.run_watchers()
self.assertTrue(mock_run_watchers.called)
def test_get_shared_lock_locking_same_lock_twice(self):
lock = self._coord.get_lock(tests.get_random_uuid())
self.assertTrue(lock.acquire(shared=True))
self.assertTrue(lock.acquire(shared=True))
self.assertTrue(lock.release())
self.assertTrue(lock.release())
self.assertFalse(lock.release())
with lock(shared=True):
pass
def test_get_lock_twice_locked_one_released_two(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(blocking=False))
self.assertFalse(lock2.release())
self.assertTrue(lock1.release())
self.assertFalse(lock2.release())
def test_get_lock_context_fails(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
with mock.patch.object(lock2, 'acquire', return_value=False):
with lock1:
self.assertRaises(
tooz.coordination.LockAcquireFailed,
lock2.__enter__)
def test_get_lock_twice_locked_one_released_two(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
self.assertTrue(lock1.acquire())
self.assertFalse(lock2.acquire(blocking=False))
self.assertFalse(lock2.release())
self.assertTrue(lock1.release())
self.assertFalse(lock2.release())
def test_client_run_watchers_mixin(self, mock_client_cls,
mock_run_watchers):
mock_client = mock.MagicMock()
mock_client_cls.return_value = mock_client
member_id = tests.get_random_uuid()
coord = coordination.get_coordinator(self.FAKE_URL, member_id)
coord.start()
coord.run_watchers()
self.assertTrue(mock_run_watchers.called)
def test_get_shared_lock_locking_same_lock_twice(self):
lock = self._coord.get_lock(tests.get_random_uuid())
self.assertTrue(lock.acquire(shared=True))
self.assertTrue(lock.acquire(shared=True))
self.assertTrue(lock.release())
self.assertTrue(lock.release())
self.assertFalse(lock.release())
with lock(shared=True):
pass
def test_get_lock_release_broken(self):
name = tests.get_random_uuid()
memberid2 = tests.get_random_uuid()
coord2 = tooz.coordination.get_coordinator(self.url, memberid2)
coord2.start()
lock1 = self._coord.get_lock(name)
lock2 = coord2.get_lock(name)
self.assertTrue(lock1.acquire(blocking=False))
self.assertFalse(lock2.acquire(blocking=False))
self.assertTrue(lock2.break_())
self.assertTrue(lock2.acquire(blocking=False))
self.assertFalse(lock1.release())
# Assert lock is not accidentally broken now
memberid3 = tests.get_random_uuid()
coord3 = tooz.coordination.get_coordinator(self.url, memberid3)
coord3.start()
lock3 = coord3.get_lock(name)
self.assertFalse(lock3.acquire(blocking=False))
def test_lock_context_manager_acquire_argument_timeout(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
with lock1:
try:
with lock2(False):
self.fail('Lock acquire should have failed')
except tooz.coordination.LockAcquireFailed:
pass
def test_lock_context_manager_acquire_argument_timeout(self):
name = tests.get_random_uuid()
lock1 = self._coord.get_lock(name)
lock2 = self._coord.get_lock(name)
with lock1:
try:
with lock2(False):
self.fail('Lock acquire should have failed')
except tooz.coordination.LockAcquireFailed:
pass
def test_get_lock_release_broken(self):
name = tests.get_random_uuid()
memberid2 = tests.get_random_uuid()
coord2 = tooz.coordination.get_coordinator(self.url,
memberid2)
coord2.start()
lock1 = self._coord.get_lock(name)
lock2 = coord2.get_lock(name)
self.assertTrue(lock1.acquire(blocking=False))
self.assertFalse(lock2.acquire(blocking=False))
self.assertTrue(lock2.break_())
self.assertTrue(lock2.acquire(blocking=False))
self.assertFalse(lock1.release())
# Assert lock is not accidentally broken now
memberid3 = tests.get_random_uuid()
coord3 = tooz.coordination.get_coordinator(self.url,
memberid3)
coord3.start()
lock3 = coord3.get_lock(name)
self.assertFalse(lock3.acquire(blocking=False))
def test_hashring_weight(self):
p = self._coord.join_partitioned_group(self.group_id, weight=5)
self.assertEqual([5], list(p.ring.nodes.values()))
coord = coordination.get_coordinator(self.url, tests.get_random_uuid())
coord.start()
p2 = coord.join_partitioned_group(self.group_id, weight=10)
self._extra_coords.append(coord)
self._coord.run_watchers()
self.assertEqual(set([5, 10]), set(p.ring.nodes.values()))
self.assertEqual(set([5, 10]), set(p2.ring.nodes.values()))
p.stop()
p2.stop()
def _create_coordinator(self, url):
def _safe_stop(coord):
try:
coord.stop()
except tooz.ToozError as e:
message = encodeutils.exception_to_unicode(e)
if (message != 'Can not stop a driver which has not'
' been started'):
raise
coord = coordination.get_coordinator(url, tests.get_random_uuid())
self.addCleanup(_safe_stop, coord)
return coord
def test_disconnect_leave_group(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url, member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertIn(member_id_test2, members_ids)
client2.stop()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertNotIn(member_id_test2, members_ids)
def test_timeout(self):
if (tooz.coordination.Characteristics.NON_TIMEOUT_BASED
in self._coord.CHARACTERISTICS):
self.skipTest("This driver is not based on timeout")
self._coord.stop()
if "?" in self.url:
sep = "&"
else:
sep = "?"
url = self.url + sep + "timeout=5"
self._coord = tooz.coordination.get_coordinator(url, self.member_id)
self._coord.start()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(url, member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertIn(member_id_test2, members_ids)
# Watch the group, we want to be sure that when client2 is kicked out
# we get an event.
self._coord.watch_leave_group(self.group_id, self._set_event)
# Run watchers to be sure we initialize the member cache and we *know*
# client2 is a member now
self._coord.run_watchers()
time.sleep(3)
self._coord.heartbeat()
time.sleep(3)
# Now client2 has timed out!
members_ids = self._coord.get_members(self.group_id).get()
while True:
if self._coord.run_watchers():
break
self.assertIn(self.member_id, members_ids)
self.assertNotIn(member_id_test2, members_ids)
# Check that the event has been triggered
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberLeftGroup)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
def test_timeout(self):
if (tooz.coordination.Characteristics.NON_TIMEOUT_BASED
in self._coord.CHARACTERISTICS):
self.skipTest("This driver is not based on timeout")
self._coord.stop()
if "?" in self.url:
sep = "&"
else:
sep = "?"
url = self.url + sep + "timeout=5"
self._coord = tooz.coordination.get_coordinator(url, self.member_id)
self._coord.start()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(url, member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertIn(member_id_test2, members_ids)
# Watch the group, we want to be sure that when client2 is kicked out
# we get an event.
self._coord.watch_leave_group(self.group_id, self._set_event)
# Run watchers to be sure we initialize the member cache and we *know*
# client2 is a member now
self._coord.run_watchers()
time.sleep(3)
self._coord.heartbeat()
time.sleep(3)
# Now client2 has timed out!
members_ids = self._coord.get_members(self.group_id).get()
while True:
if self._coord.run_watchers():
break
self.assertIn(self.member_id, members_ids)
self.assertNotIn(member_id_test2, members_ids)
# Check that the event has been triggered
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberLeftGroup)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
def _create_coordinator(self, url):
def _safe_stop(coord):
try:
coord.stop()
except coordination.ToozError as e:
message = encodeutils.exception_to_unicode(e)
if (message != 'Can not stop a driver which has not'
' been started'):
raise
coord = coordination.get_coordinator(url,
tests.get_random_uuid())
self.addCleanup(_safe_stop, coord)
return coord
def test_disconnect_leave_group(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()
client2.join_group(self.group_id).get()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertIn(member_id_test2, members_ids)
client2.stop()
members_ids = self._coord.get_members(self.group_id).get()
self.assertIn(self.member_id, members_ids)
self.assertNotIn(member_id_test2, members_ids)
def _create_coordinator(self):
def _safe_stop(coord):
try:
coord.stop()
except tooz.ToozError as e:
# TODO(harlowja): make this better, so that we don't have to
# do string checking...
message = encodeutils.exception_to_unicode(e)
if (message != 'Can not stop a driver which has not'
' been started'):
raise
coord = coordination.get_coordinator(self.FAKE_URL,
tests.get_random_uuid())
self.addCleanup(_safe_stop, coord)
return coord
def _create_coordinator(self):
def _safe_stop(coord):
try:
coord.stop()
except tooz.ToozError as e:
# TODO(harlowja): make this better, so that we don't have to
# do string checking...
message = encodeutils.exception_to_unicode(e)
if (message != 'Can not stop a driver which has not'
' been started'):
raise
coord = coordination.get_coordinator(self.FAKE_URL,
tests.get_random_uuid())
self.addCleanup(_safe_stop, coord)
return coord
class TestFileDriver(testcase.TestCase):
_FAKE_MEMBER_ID = tests.get_random_uuid()
def test_base_dir(self):
file_path = '/fake/file/path'
url = 'file://%s' % file_path
coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID)
self.assertEqual(file_path, coord._dir)
def test_leftover_file(self):
fixture = self.useFixture(fixtures.TempDir())
file_path = fixture.path
url = 'file://%s' % file_path
coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID)
coord.start()
self.addCleanup(coord.stop)
coord.create_group(b"my_group").get()
safe_group_id = coord._make_filesystem_safe(b"my_group")
with open(os.path.join(file_path, 'groups',
safe_group_id, "junk.txt"), "wb"):
pass
os.unlink(os.path.join(file_path, 'groups',
safe_group_id, '.metadata'))
self.assertRaises(tooz.ToozError,
coord.delete_group(b"my_group").get)
@mock.patch('os.path.normpath', lambda x: x.replace('/', '\\'))
@mock.patch('sys.platform', 'win32')
def test_base_dir_win32(self):
coord = coordination.get_coordinator(
'file:///C:/path/', self._FAKE_MEMBER_ID)
self.assertEqual('C:\\path\\', coord._dir)
coord = coordination.get_coordinator(
'file:////share_addr/share_path/', self._FAKE_MEMBER_ID)
self.assertEqual('\\\\share_addr\\share_path\\', coord._dir)
# Administrative shares should be handled properly.
coord = coordination.get_coordinator(
'file:////c$/path/', self._FAKE_MEMBER_ID)
self.assertEqual('\\\\c$\\path\\', coord._dir)
def test_get_lock_concurrency_locking_same_lock(self):
lock = self._coord.get_lock(tests.get_random_uuid())
graceful_ending = threading.Event()
def thread():
self.assertTrue(lock.acquire())
self.assertTrue(lock.release())
graceful_ending.set()
t = threading.Thread(target=thread)
t.daemon = True
with lock:
t.start()
# Ensure the thread try to get the lock
time.sleep(.1)
t.join()
graceful_ending.wait(.2)
self.assertTrue(graceful_ending.is_set())
def test_get_lock_concurrency_locking_same_lock(self):
lock = self._coord.get_lock(tests.get_random_uuid())
graceful_ending = threading.Event()
def thread():
self.assertTrue(lock.acquire())
self.assertTrue(lock.release())
graceful_ending.set()
t = threading.Thread(target=thread)
t.daemon = True
with lock:
t.start()
# Ensure the thread try to get the lock
time.sleep(.1)
t.join()
graceful_ending.wait(.2)
self.assertTrue(graceful_ending.is_set())
def test_watch_leave_group(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
# Watch the group: this can leads to race conditions in certain
# driver that are not able to see all events, so we join, wait for
# the join to be seen, and then we leave, and wait for the leave to
# be seen.
self._coord.watch_join_group(self.group_id, lambda children: True)
self._coord.watch_leave_group(self.group_id, self._set_event)
# Join and leave the group
client2.join_group(self.group_id).get()
# Consumes join event
while True:
if self._coord.run_watchers():
break
client2.leave_group(self.group_id).get()
# Consumes leave event
while True:
if self._coord.run_watchers():
break
self.assertIsInstance(self.event,
tooz.coordination.MemberLeftGroup)
self.assertEqual(member_id_test2,
self.event.member_id)
self.assertEqual(self.group_id,
self.event.group_id)
# Stop watching
self._coord.unwatch_leave_group(self.group_id, self._set_event)
self.event = None
# Rejoin and releave group
client2.join_group(self.group_id).get()
client2.leave_group(self.group_id).get()
self._coord.run_watchers()
self.assertIsNone(self.event)
def test_run_for_election_multiple_clients_stand_down(self):
self._coord.create_group(self.group_id).get()
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
self._coord.run_watchers()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
client2.watch_elected_as_leader(self.group_id, self._set_event)
client2.run_watchers()
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.LeaderElected)
self.assertEqual(self.member_id, event.member_id)
self.assertEqual(self.group_id, event.group_id)
self.events = []
self._coord.stand_down_group_leader(self.group_id)
client2.run_watchers()
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.LeaderElected)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
self.events = []
client2.stand_down_group_leader(self.group_id)
self._coord.run_watchers()
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.LeaderElected)
self.assertEqual(self.member_id, event.member_id)
self.assertEqual(self.group_id, event.group_id)
def test_watch_leave_group(self):
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
# Watch the group: this can leads to race conditions in certain
# driver that are not able to see all events, so we join, wait for
# the join to be seen, and then we leave, and wait for the leave to
# be seen.
self._coord.watch_join_group(self.group_id, lambda children: True)
self._coord.watch_leave_group(self.group_id, self._set_event)
# Join and leave the group
client2.join_group(self.group_id).get()
# Consumes join event
while True:
if self._coord.run_watchers():
break
client2.leave_group(self.group_id).get()
# Consumes leave event
while True:
if self._coord.run_watchers():
break
self.assertEqual(1, len(self.events))
event = self.events[0]
self.assertIsInstance(event, tooz.coordination.MemberLeftGroup)
self.assertEqual(member_id_test2, event.member_id)
self.assertEqual(self.group_id, event.group_id)
# Stop watching
self._coord.unwatch_leave_group(self.group_id, self._set_event)
self.events = []
# Rejoin and releave group
client2.join_group(self.group_id).get()
client2.leave_group(self.group_id).get()
self._coord.run_watchers()
self.assertEqual([], self.events)
def test_run_for_election_multiple_clients(self):
self._coord.create_group(self.group_id).get()
self._coord.watch_elected_as_leader(self.group_id, self._set_event)
self._coord.run_watchers()
member_id_test2 = tests.get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2.start()
client2.watch_elected_as_leader(self.group_id, self._set_event)
client2.run_watchers()
self.assertIsInstance(self.event,
tooz.coordination.LeaderElected)
self.assertEqual(self.member_id,
self.event.member_id)
self.assertEqual(self.group_id,
self.event.group_id)
self.assertEqual(self._coord.get_leader(self.group_id).get(),
self.member_id)
self.event = None
self._coord.stop()
client2.run_watchers()
self.assertIsInstance(self.event,
tooz.coordination.LeaderElected)
self.assertEqual(member_id_test2,
self.event.member_id)
self.assertEqual(self.group_id,
self.event.group_id)
self.assertEqual(client2.get_leader(self.group_id).get(),
member_id_test2)
# Restart the coord because tearDown stops it
self._coord.start()
def test_client_failure_start(self, mock_client_cls):
mock_client_cls.side_effect = socket.timeout('timed-out')
member_id = tests.get_random_uuid()
coord = coordination.get_coordinator(self.FAKE_URL, member_id)
self.assertRaises(coordination.ToozConnectionError, coord.start)
