def _inner_acquire(self, blocking, timeout=None):
"""Inner loop that runs from the top anytime a command hits a
retryable Zookeeper exception."""
self._session_expired = False
self.client.add_listener(self._watch_session)
if not self.assured_path:
self._ensure_path()
# Do we already have a lease?
if self.client.exists(self.create_path):
return True
with self.client.Lock(self.lock_path, self.data):
while True:
self.wake_event.clear()
# Attempt to grab our lease...
if self._get_lease():
return True
if blocking:
# If blocking, wait until self._watch_lease_change() is
# called before returning
self.wake_event.wait(timeout)
if not self.wake_event.isSet():
raise LockTimeout(
"Failed to acquire semaphore on %s "
"after %s seconds" % (self.path, timeout))
else:
# If not blocking, register another watch that will trigger
# self._get_lease() as soon as the children change again.
self.client.get_children(self.path, self._get_lease)
return False
def _inner_acquire(self, timeout, ephemeral=True):
# wait until it's our chance to get it..
if self.is_acquired:
raise ForceRetryError()
# make sure our election parent node exists
if not self.assured_path:
yield self._ensure_path()
node = None
if self.create_tried:
node = yield self._find_node()
else:
self.create_tried = True
if not node:
node = yield self.tornado_kazoo.create(
self.create_path, self.data, ephemeral=ephemeral, sequence=True)
# strip off path to node
node = node[len(self.path) + 1:]
self.node = node
while True:
self.wake_event.clear()
# bail out with an exception if cancellation has been requested
if self.cancelled:
raise CancelledError()
children = yield self._get_sorted_children()
try:
our_index = children.index(node)
except ValueError: # pragma: nocover
# somehow we aren't in the children -- probably we are
# recovering from a session failure and our ephemeral
# node was removed
raise ForceRetryError()
predecessor = self.predecessor(children, our_index)
if not predecessor:
raise gen.Return(True)
# otherwise we are in the mix. watch predecessor and bide our time
predecessor = self.path + "/" + predecessor
self.client.add_listener(self._watch_session_listener)
try:
yield self.tornado_kazoo.get(predecessor, self._watch_predecessor)
except NoNodeError:
pass # predecessor has already been deleted
else:
try:
yield self.wake_event.wait(timeout)
except gen.TimeoutError:
raise LockTimeout("Failed to acquire lock on %s after "
"%s seconds" % (self.path, timeout))
finally:
self.client.remove_listener(self._watch_session_listener)
def _inner_acquire(self, blocking, timeout):
# wait until it's our chance to get it..
if self.is_acquired:
if not blocking:
return False
raise ForceRetryError()
# make sure our election parent node exists
if not self.assured_path:
self._ensure_path()
node = None
if self.create_tried:
node = self._find_node()
else:
self.create_tried = True
if not node:
node = self.client.create(self.create_path, self.data,
ephemeral=True, sequence=True)
# strip off path to node
node = node[len(self.path) + 1:]
self.node = node
while True:
self.wake_event.clear()
# bail out with an exception if cancellation has been requested
if self.cancelled:
raise CancelledError()
children = self._get_sorted_children()
try:
our_index = children.index(node)
except ValueError: # pragma: nocover
# somehow we aren't in the children -- probably we are
# recovering from a session failure and our ephemeral
# node was removed
raise ForceRetryError()
if self.acquired_lock(children, our_index):
return True
if not blocking:
return False
# otherwise we are in the mix. watch predecessor and bide our time
predecessor = self.path + "/" + children[our_index - 1]
self.client.add_listener(self._watch_session)
try:
if self.client.exists(predecessor, self._watch_predecessor):
self.wake_event.wait(timeout)
if not self.wake_event.isSet():
raise LockTimeout("Failed to acquire lock on %s after "
"%s seconds" % (self.path, timeout))
finally:
self.client.remove_listener(self._watch_session)
def _inner_acquire(self, blocking, timeout, ephemeral=True):
# wait until it's our chance to get it..
if self.is_acquired:
if not blocking:
return False
raise ForceRetryError()
# make sure our election parent node exists
if not self.assured_path:
self._ensure_path()
node = None
if self.create_tried:
node = self._find_node()
else:
self.create_tried = True
if not node:
node = self.client.create(self.create_path,
self.data,
ephemeral=ephemeral,
sequence=True)
# strip off path to node
node = node[len(self.path) + 1:]
self.node = node
while True:
self.wake_event.clear()
# bail out with an exception if cancellation has been requested
if self.cancelled:
raise CancelledError()
predecessor = self._get_predecessor(node)
if predecessor is None:
return True
if not blocking:
return False
# otherwise we are in the mix. watch predecessor and bide our time
predecessor = self.path + "/" + predecessor
self.client.add_listener(self._watch_session)
try:
self.client.get(predecessor, self._watch_predecessor)
except NoNodeError:
pass # predecessor has already been deleted
else:
self.wake_event.wait(timeout)
if not self.wake_event.isSet():
raise LockTimeout(
"Failed to acquire lock on %s after %s seconds" %
(self.path, timeout))
finally:
self.client.remove_listener(self._watch_session)
def _acquire_loop(self, blocking, timeout):
acquired = yield self.is_acquired_eff()
if acquired or not blocking:
yield do_return(acquired)
start = yield Effect(Func(time.time))
while True:
yield Effect(Delay(self._interval))
if (yield self.is_acquired_eff()):
yield do_return(True)
if timeout is not None:
now = yield Effect(Func(time.time))
if now - start > timeout:
raise LockTimeout(
"Failed to acquire lock on {} in {} seconds".format(
self.path, now - start))
def acquire(self, timeout=None):
sleep = self._client.handler.sleep_func
sleep(self.delay_time)
if timeout is None:
return self._lock.acquire()
start_time = time.time()
while time.time() - start_time < timeout:
if self._lock.acquire(False):
return True
sleep(0.1)
raise LockTimeout("Mocked slow lock has timed out.")
def acquire(self, timeout=None, ephemeral=True):
""" Acquires the lock. By default, it blocks and waits forever.
Args:
timeout: A float specifying how long to wait to acquire the lock.
ephemeral: A boolean indicating that the lock should use an ephemeral
node.
Raises:
LockTimeout if the lock wasn't acquired within `timeout` seconds.
"""
retry = self._retry.copy()
retry.deadline = timeout
# Ensure we are locked so that we avoid multiple coroutines in
# this acquisition routine at the same time...
timeout_interval = None
if timeout is not None:
timeout_interval = datetime.timedelta(seconds=timeout)
try:
with (yield self._lock.acquire(timeout=timeout_interval)):
already_acquired = self.is_acquired
gotten = False
try:
gotten = yield retry(self._inner_acquire, timeout=timeout,
ephemeral=ephemeral)
except RetryFailedError:
pass
except KazooException:
# if we did ultimately fail, attempt to clean up
exc_info = sys.exc_info()
if not already_acquired:
yield self._best_effort_cleanup()
self.cancelled = False
six.reraise(exc_info[0], exc_info[1], exc_info[2])
if gotten:
self.is_acquired = gotten
if not gotten and not already_acquired:
yield self._best_effort_cleanup()
raise gen.Return(gotten)
except gen.TimeoutError:
raise LockTimeout("Failed to acquire lock on %s after "
"%s seconds" % (self.path, timeout))
def _inner_acquire(self, blocking, timeout=None):
"""Inner loop that runs from the top anytime a command hits a
retryable Zookeeper exception."""
self._session_expired = False
self.client.add_listener(self._watch_session)
if not self.assured_path:
self._ensure_path()
# Do we already have a lease?
if self.client.exists(self.create_path):
return True
w = _Watch(duration=timeout)
w.start()
lock = self.client.Lock(self.lock_path, self.data)
gotten = lock.acquire(blocking=blocking, timeout=w.leftover())
if not gotten:
return False
try:
while True:
self.wake_event.clear()
# Attempt to grab our lease...
if self._get_lease():
return True
if blocking:
# If blocking, wait until self._watch_lease_change() is
# called before returning
self.wake_event.wait(w.leftover())
if not self.wake_event.isSet():
raise LockTimeout("Failed to acquire semaphore on %s "
"after %s seconds" %
(self.path, timeout))
else:
return False
finally:
lock.release()
def acquire_loop(self, timeout=None):
if timeout is None:
timeout = self.timeout
expire_at = time.time() + timeout
while True:
# Even if timeout is smaller than 0, try-loop continue on until
# maybe_available is not ready.
#
# There is a chance that:
# - Failed to create lock node(lock is occupied by other)
# - Failed to get lock node(just deleted)
# - Failed to create lock node(lock is occupied by other)
# - Failed to get lock node(just deleted)
# - ...
if not self.maybe_available.wait(timeout=expire_at - time.time()):
logger.debug('lock is still held by others: ' + str(self))
if time.time() > expire_at:
raise LockTimeout('lock: ' + str(self.lock_path))
# Always proceed the "get" phase, in order to add a watch handler.
# To watch node change event.
self._create()
self._acquire_by_get()
if self.is_locked():
return
# If it is possible to acquire the lock in next retry, do not yield
if self.maybe_available.is_set():
continue
else:
yield self.lock_holder[0], self.lock_holder[1]
def bad_lock(self):
bad_lock = mock.Mock()
bad_lock.acquire = mock.Mock(side_effect=LockTimeout('Test exception'))
return bad_lock
def _inner_acquire(self):
""" Create contender node(s) and wait until the lock is acquired. """
# Make sure the group lock node exists.
self._ensure_path()
nodes = [None for _ in self.paths]
if self.create_tried:
nodes = self._find_nodes()
else:
self.create_tried = True
for index, node in enumerate(nodes):
if node is not None:
continue
# The entity group lock root may have been deleted, so try a few times.
try_num = 0
while True:
try:
node = self.client.create(self.create_paths[index],
self.data,
sequence=True)
break
except NoNodeError:
self.client.ensure_path(self.paths[index])
if try_num > 3:
raise ForceRetryError()
try_num += 1
# Strip off path to node.
node = node[len(self.paths[index]) + 1:]
nodes[index] = node
self.nodes = nodes
while True:
self.wake_event.clear()
# Bail out with an exception if cancellation has been requested.
if self.cancelled:
raise CancelledError()
children_list = self._get_sorted_children()
predecessors = []
for index, children in enumerate(children_list):
try:
our_index = children.index(nodes[index])
except ValueError:
raise ForceRetryError()
# If the lock for this group hasn't been acquired, get the predecessor.
if our_index != 0:
predecessors.append(self.paths[index] + "/" +
children[our_index - 1])
if not predecessors:
return True
if len(nodes) > 1:
self._resolve_deadlocks(children_list)
# Wait for predecessor to be removed.
# TODO: Listen for all at the same time.
for index, predecessor in enumerate(predecessors):
self.client.add_listener(self._watch_session)
try:
if self.client.exists(predecessor,
self._watch_predecessor):
self.wake_event.wait(LOCK_TIMEOUT)
if not self.wake_event.isSet():
error = 'Failed to acquire lock on {} after {} '\
'seconds'.format(self.paths, LOCK_TIMEOUT * (index + 1))
raise LockTimeout(error)
finally:
self.client.remove_listener(self._watch_session)
