def test_lock_one(self):
lock_name = uuid.uuid4().hex
lock = self.client.Lock(self.lockpath, lock_name)
event = threading.Event()
thread = threading.Thread(target=self._thread_lock_acquire_til_event,
args=(lock_name, lock, event))
thread.start()
lock2_name = uuid.uuid4().hex
anotherlock = self.client.Lock(self.lockpath, lock2_name)
# wait for any contender to show up on the lock
wait(anotherlock.contenders)
eq_(anotherlock.contenders(), [lock_name])
with self.condition:
while self.active_thread != lock_name:
self.condition.wait()
# release the lock
event.set()
with self.condition:
while self.active_thread:
self.condition.wait()
self.released.wait()
thread.join()
def test_lock_reconnect(self):
event = threading.Event()
other_lock = self.client.Lock(self.lockpath, 'contender')
thread = threading.Thread(target=self._thread_lock_acquire_til_event,
args=('contender', other_lock, event))
# acquire the lock ourselves first to make the contender line up
lock = self.client.Lock(self.lockpath, "test")
lock.acquire()
thread.start()
# wait for the contender to line up on the lock
wait(lambda: len(lock.contenders()) == 2)
eq_(lock.contenders(), ['test', 'contender'])
self.expire_session()
lock.release()
with self.condition:
while not self.active_thread:
self.condition.wait()
eq_(self.active_thread, 'contender')
event.set()
thread.join()
def test_lock_reconnect(self):
event = threading.Event()
other_lock = self.client.Lock(self.lockpath, 'contender')
thread = threading.Thread(target=self._thread_lock_acquire_til_event,
args=('contender', other_lock, event))
# acquire the lock ourselves first to make the contender line up
lock = self.client.Lock(self.lockpath, "test")
lock.acquire()
thread.start()
# wait for the contender to line up on the lock
wait(lambda: len(lock.contenders()) == 2)
eq_(lock.contenders(), ['test', 'contender'])
self.expire_session()
lock.release()
with self.condition:
while not self.active_thread:
self.condition.wait()
eq_(self.active_thread, 'contender')
event.set()
thread.join()
def test_lock_one(self):
lock_name = uuid.uuid4().hex
lock = self.client.Lock(self.lockpath, lock_name)
event = threading.Event()
thread = threading.Thread(target=self._thread_lock_acquire_til_event,
args=(lock_name, lock, event))
thread.start()
lock2_name = uuid.uuid4().hex
anotherlock = self.client.Lock(self.lockpath, lock2_name)
# wait for any contender to show up on the lock
wait(anotherlock.contenders)
eq_(anotherlock.contenders(), [lock_name])
with self.condition:
while self.active_thread != lock_name:
self.condition.wait()
# release the lock
event.set()
with self.condition:
while self.active_thread:
self.condition.wait()
self.released.wait()
def test_logging(self):
handler = InstalledHandler('ZooKeeper')
try:
handle = zookeeper.init('zookeeper.example.com:2181')
zookeeper.close(handle)
except Exception:
pass
wait(lambda: [r for r in handler.records
if 'environment' in r.getMessage()]
)
handler.clear()
kazoo.klog.setup_logging()
# Test that the filter for the "Exceeded deadline by" noise works.
# cheat and bypass zk by writing to the pipe directly.
os.write(kazoo.klog._logging_pipe[1],
'2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
wait(lambda: [r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.DEBUG)
]
)
self.assertFalse([r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.WARNING)
])
handler.uninstall()
def test_dirty_sock(self):
client = self.client
read_sock = client._connection._read_sock
write_sock = client._connection._write_sock
# add a stray byte to the socket and ensure that doesn't
# blow up client. simulates case where some error leaves
# a byte in the socket which doesn't correspond to the
# request queue.
write_sock.send(b'\0')
# eventually this byte should disappear from socket
wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
def test_dirty_sock(self):
client = self.client
read_sock = client._connection._read_sock
write_sock = client._connection._write_sock
# add a stray byte to the socket and ensure that doesn't
# blow up client. simulates case where some error leaves
# a byte in the socket which doesn't correspond to the
# request queue.
write_sock.send(b'\0')
# eventually this byte should disappear from socket
wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
def test_dirty_pipe(self):
client = self.client
read_pipe = client._connection._read_pipe
write_pipe = client._connection._write_pipe
# add a stray byte to the pipe and ensure that doesn't
# blow up client. simulates case where some error leaves
# a byte in the pipe which doesn't correspond to the
# request queue.
os.write(write_pipe, b'\0')
# eventually this byte should disappear from pipe
wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
def test_dirty_pipe(self):
client = self.client
read_pipe = client._connection._read_pipe
write_pipe = client._connection._write_pipe
# add a stray byte to the pipe and ensure that doesn't
# blow up client. simulates case where some error leaves
# a byte in the pipe which doesn't correspond to the
# request queue.
os.write(write_pipe, b'\0')
# eventually this byte should disappear from pipe
wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
def test_lock(self):
threads = []
names = ["contender" + str(i) for i in range(5)]
contender_bits = {}
for name in names:
e = threading.Event()
l = self.client.Lock(self.lockpath, name)
t = threading.Thread(target=self._thread_lock_acquire_til_event,
args=(name, l, e))
contender_bits[name] = (t, e)
threads.append(t)
# acquire the lock ourselves first to make the others line up
lock = self.client.Lock(self.lockpath, "test")
lock.acquire()
for t in threads:
t.start()
# wait for everyone to line up on the lock
wait(lambda: len(lock.contenders()) == 6)
contenders = lock.contenders()
eq_(contenders[0], "test")
contenders = contenders[1:]
remaining = list(contenders)
# release the lock and contenders should claim it in order
lock.release()
for contender in contenders:
thread, event = contender_bits[contender]
with self.condition:
while not self.active_thread:
self.condition.wait()
eq_(self.active_thread, contender)
eq_(lock.contenders(), remaining)
remaining = remaining[1:]
event.set()
with self.condition:
while self.active_thread:
self.condition.wait()
for thread in threads:
thread.join()
def test_lock(self):
threads = []
names = ["contender" + str(i) for i in range(5)]
contender_bits = {}
for name in names:
e = threading.Event()
l = self.client.Lock(self.lockpath, name)
t = threading.Thread(target=self._thread_lock_acquire_til_event,
args=(name, l, e))
contender_bits[name] = (t, e)
threads.append(t)
# acquire the lock ourselves first to make the others line up
lock = self.client.Lock(self.lockpath, "test")
lock.acquire()
for t in threads:
t.start()
# wait for everyone to line up on the lock
wait(lambda: len(lock.contenders()) == 6)
contenders = lock.contenders()
eq_(contenders[0], "test")
contenders = contenders[1:]
remaining = list(contenders)
# release the lock and contenders should claim it in order
lock.release()
for contender in contenders:
thread, event = contender_bits[contender]
with self.condition:
while not self.active_thread:
self.condition.wait()
eq_(self.active_thread, contender)
eq_(lock.contenders(), remaining)
remaining = remaining[1:]
event.set()
with self.condition:
while self.active_thread:
self.condition.wait()
for thread in threads:
thread.join()
def test_lock_cancel(self):
event1 = threading.Event()
lock1 = self.client.Lock(self.lockpath, "one")
thread1 = threading.Thread(target=self._thread_lock_acquire_til_event,
args=("one", lock1, event1))
thread1.start()
# wait for this thread to acquire the lock
with self.condition:
if not self.active_thread:
self.condition.wait(5)
eq_(self.active_thread, "one")
client2 = self._get_client()
client2.start()
event2 = threading.Event()
lock2 = client2.Lock(self.lockpath, "two")
thread2 = threading.Thread(target=self._thread_lock_acquire_til_event,
args=("two", lock2, event2))
thread2.start()
# this one should block in acquire. check that it is a contender
wait(lambda: len(lock2.contenders()) > 1)
eq_(lock2.contenders(), ["one", "two"])
lock2.cancel()
with self.condition:
if "two" not in self.cancelled_threads:
self.condition.wait()
assert "two" in self.cancelled_threads
eq_(lock2.contenders(), ["one"])
thread2.join()
event1.set()
thread1.join()
client2.stop()
def test_lock_cancel(self):
event1 = threading.Event()
lock1 = self.client.Lock(self.lockpath, "one")
thread1 = threading.Thread(target=self._thread_lock_acquire_til_event,
args=("one", lock1, event1))
thread1.start()
# wait for this thread to acquire the lock
with self.condition:
if not self.active_thread:
self.condition.wait(5)
eq_(self.active_thread, "one")
client2 = self._get_client()
client2.start()
event2 = threading.Event()
lock2 = client2.Lock(self.lockpath, "two")
thread2 = threading.Thread(target=self._thread_lock_acquire_til_event,
args=("two", lock2, event2))
thread2.start()
# this one should block in acquire. check that it is a contender
wait(lambda: len(lock2.contenders()) > 1)
eq_(lock2.contenders(), ["one", "two"])
lock2.cancel()
with self.condition:
if not "two" in self.cancelled_threads:
self.condition.wait()
assert "two" in self.cancelled_threads
eq_(lock2.contenders(), ["one"])
thread2.join()
event1.set()
thread1.join()
client2.stop()
def test_election(self):
elections = {}
threads = {}
for _ in range(3):
contender = "c" + uuid.uuid4().hex
elections[contender] = self.client.Election(self.path, contender)
threads[contender] = self._spawn_contender(contender,
elections[contender])
# wait for a leader to be elected
times = 0
with self.condition:
while not self.leader_id:
self.condition.wait(5)
times += 1
if times > 5:
raise Exception("Still not a leader: lid: %s",
self.leader_id)
election = self.client.Election(self.path)
# make sure all contenders are in the pool
wait(lambda: len(election.contenders()) == len(elections))
contenders = election.contenders()
eq_(set(contenders), set(elections.keys()))
# first one in list should be leader
first_leader = contenders[0]
eq_(first_leader, self.leader_id)
# tell second one to cancel election. should never get elected.
elections[contenders[1]].cancel()
# make leader exit. third contender should be elected.
self.exit_event.set()
with self.condition:
while self.leader_id == first_leader:
self.condition.wait(45)
eq_(self.leader_id, contenders[2])
self._check_thread_error()
# make first contender re-enter the race
threads[first_leader].join()
threads[first_leader] = self._spawn_contender(first_leader,
elections[first_leader])
# contender set should now be the current leader plus the first leader
wait(lambda: len(election.contenders()) == 2)
contenders = election.contenders()
eq_(set(contenders), set([self.leader_id, first_leader]))
# make current leader raise an exception. first should be reelected
self.raise_exception = True
self.exit_event.set()
with self.condition:
while self.leader_id != first_leader:
self.condition.wait(45)
eq_(self.leader_id, first_leader)
self._check_thread_error()
self.exit_event.set()
for thread in threads.values():
thread.join()
self._check_thread_error()
def test_election(self):
elections = {}
threads = {}
for _ in range(3):
contender = "c" + uuid.uuid4().hex
elections[contender] = self.client.Election(self.path, contender)
threads[contender] = self._spawn_contender(contender,
elections[contender])
# wait for a leader to be elected
times = 0
with self.condition:
while not self.leader_id:
self.condition.wait(5)
times += 1
if times > 5:
raise Exception("Still not a leader: lid: %s",
self.leader_id)
election = self.client.Election(self.path)
# make sure all contenders are in the pool
wait(lambda: len(election.contenders()) == len(elections))
contenders = election.contenders()
eq_(set(contenders), set(elections.keys()))
# first one in list should be leader
first_leader = contenders[0]
eq_(first_leader, self.leader_id)
# tell second one to cancel election. should never get elected.
elections[contenders[1]].cancel()
# make leader exit. third contender should be elected.
self.exit_event.set()
with self.condition:
while self.leader_id == first_leader:
self.condition.wait(45)
eq_(self.leader_id, contenders[2])
self._check_thread_error()
# make first contender re-enter the race
threads[first_leader].join()
threads[first_leader] = self._spawn_contender(first_leader,
elections[first_leader])
# contender set should now be the current leader plus the first leader
wait(lambda: len(election.contenders()) == 2)
contenders = election.contenders()
eq_(set(contenders), set([self.leader_id, first_leader]))
# make current leader raise an exception. first should be reelected
self.raise_exception = True
self.exit_event.set()
with self.condition:
while self.leader_id != first_leader:
self.condition.wait(45)
eq_(self.leader_id, first_leader)
self._check_thread_error()
self.exit_event.set()
for thread in threads.values():
thread.join()
self._check_thread_error()
def _do_test(self, contender_count, exclusive_mod=0,
revocation_mod=0, callback_mod=0,
callback_action_mod=0):
"""Runs a shared lock test based on the parameters given. Allows
testing different combinations of shared vs exclusive locks, locks that
request revocations, and locks that receive callbacks. This is useful
given the range of lock strategies offered by ``Lock``.
The method computes the expected groups of locks that should occur
given the parameters. It then tests the state of the lock system at
each of these phases, such as which locks have been acquired, which
are pending, whether revocations have been issued, and whether a single
revocation callback has been received for impacted locks.
The main parameter is ``contender_count``, which indicates how many
concurrent lock contenders with associated threads will be created. The
remaining parameters indicate the modulus interval (of the contender
identifier) that will attract a particular behavior. For example, a
``contender_count`` of 20 with an ``exclusive_mod`` of 6 will produce
lock contenders numbered 0 to 19, and contenders numbered 0, 6, 12 and
18 will request exclusive locks. Use modulus 0 to disable a behaviour.
:param contender_count: number of contenders to create
:param exclusive_mod: the modulus that will be used to produce
exclusive locks (use 1 to make all contenders use
exclusive locks, or 0 for all contenders to use
shared locks)
:param revocation_mod: the modulus that will request revocation of any
existing lock
:param callback_mod: the modulus that will request an unlock callback
"""
# record the lock contenders we're creating
def build_contenders(contender_count, modulus):
if modulus == 0:
return []
return ["contender" + str(i) for i in range(contender_count)
if i % modulus == 0]
names = build_contenders(contender_count, 1)
exclusives = build_contenders(contender_count, exclusive_mod)
revokers = build_contenders(contender_count, revocation_mod)
callbacks = build_contenders(contender_count, callback_mod)
# compute the phases (ie groups of concurrent locks) that should occur
phases = []
this_phase = []
for name in names:
if name in exclusives:
if len(this_phase) > 0:
phases.append(this_phase) # store last phase
phases.append([name]) # record a phase with just the exclusive
this_phase = [] # start new phase
else:
this_phase.append(name) # shared
phases.append(this_phase) # store the final phase
# setup our contenders
threads = []
contender_bits = {}
for name in names:
e = threading.Event()
exclusive = True if name in exclusives else False
l = self.client.Lock(self.lockpath, name, exclusive=exclusive)
revoke = True if name in revokers else False
callback = True if name in callbacks else False
revoke = True if name in revokers else False
t = threading.Thread(target=self._thread_lock_acquire_til_event,
args=(name, l, e, revoke, callback))
contender_bits[name] = (t, e)
threads.append(t)
# acquire the lock ourselves first to make the others line up
lock = self.client.Lock(self.lockpath, "test")
lock.acquire()
# start threads one-by-one to prevent thread scheduling inconsistencies
for contender in names:
thread, event = contender_bits[contender]
thread.start()
wait(lambda: contender in lock.contenders())
contenders = lock.contenders()
eq_(contenders[0], "test")
contenders = contenders[1:]
remaining = list(contenders)
# release the lock and contenders should claim it in order
lock.release()
# validate state at each phase
for phase in phases:
for contender in phase:
thread, event = contender_bits[contender]
with self.condition:
while not contender in self.active_threads:
self.condition.wait()
# all claim to have their locks, so check expected remainders
eq_(lock.contenders(), remaining)
remaining = [x for x in remaining if x not in phase]
unlocks = lock.contenders(unlocks_only=True)
if not set(remaining).isdisjoint(revokers):
# 1+ revoker is remaining, so revocations should have been made
# for at least all members of this phase
ok_(set(phase).issubset(set(unlocks)))
# in addition all members of this phase with callbacks should
# have fired
for expected_callback in set(phase).intersection(callbacks):
with self.condition:
while not expected_callback in self.callback_threads:
self.condition.wait()
# release their locks
for contender in phase:
thread, event = contender_bits[contender]
event.set()
with self.condition:
while contender in self.active_threads:
self.condition.wait()
# ensure all terminate
for thread in threads:
thread.join()
