def _RequestAndWait(self, request, timeout):
"""Request locks from WConfD and wait for them to be granted.
@type request: list
@param request: the lock request to be sent to WConfD
@type timeout: float
@param timeout: the time to wait for the request to be granted
@raise LockAcquireTimeout: In case locks couldn't be acquired in specified
amount of time; in this case, locks still might be acquired or a request
pending.
"""
logging.debug("Trying %ss to request %s for %s", timeout, request,
self._wconfdcontext)
if self._cbs:
priority = self._cbs.CurrentPriority() # pylint: disable=W0612
else:
priority = None
if priority is None:
priority = constants.OP_PRIO_DEFAULT
## Expect a signal
if sighupReceived[0]:
logging.warning("Ignoring unexpected SIGHUP")
sighupReceived[0] = False
# Request locks
self.wconfd.Client().UpdateLocksWaiting(self._wconfdcontext, priority,
request)
pending = self.wconfd.Client().HasPendingRequest(self._wconfdcontext)
if pending:
def _HasPending():
if sighupReceived[0]:
return self.wconfd.Client().HasPendingRequest(
self._wconfdcontext)
else:
return True
pending = utils.SimpleRetry(False, _HasPending, 0.05, timeout)
signal = sighupReceived[0]
if pending:
pending = self.wconfd.Client().HasPendingRequest(
self._wconfdcontext)
if pending and signal:
logging.warning("Ignoring unexpected SIGHUP")
sighupReceived[0] = False
logging.debug("Finished trying. Pending: %s", pending)
if pending:
raise LockAcquireTimeout()
def testSimpleRetry(self):
self.assertFalse(
utils.SimpleRetry(True,
lambda: False,
0.01,
0.02,
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertFalse(
utils.SimpleRetry(lambda x: x,
lambda: False,
0.01,
0.02,
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertTrue(
utils.SimpleRetry(True,
lambda: True,
0,
1,
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertTrue(
utils.SimpleRetry(lambda x: x,
lambda: True,
0,
1,
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertTrue(
utils.SimpleRetry(True,
self._SimpleRetryAndSucceed,
0,
1,
args=[1],
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertEqual(self.retries, 1)
self.assertEqual(self.called, 2)
self.called = self.retries = 0
self.assertTrue(
utils.SimpleRetry(True,
self._SimpleRetryAndSucceed,
0,
1,
args=[2],
wait_fn=self._wait_fn,
_time_fn=self._time_fn))
self.assertEqual(self.called, 3)
def _AcquireLocks(self,
level,
names,
shared,
opportunistic,
timeout,
opportunistic_count=1,
request_only=False):
"""Acquires locks via the Ganeti lock manager.
@type level: int
@param level: Lock level
@type names: list or string
@param names: Lock names
@type shared: bool
@param shared: Whether the locks should be acquired in shared mode
@type opportunistic: bool
@param opportunistic: Whether to acquire opportunistically
@type timeout: None or float
@param timeout: Timeout for acquiring the locks
@type request_only: bool
@param request_only: do not acquire the locks, just return the request
@raise LockAcquireTimeout: In case locks couldn't be acquired in specified
amount of time; in this case, locks still might be acquired or a request
pending.
"""
self._CheckLocksEnabled()
if self._cbs:
priority = self._cbs.CurrentPriority() # pylint: disable=W0612
else:
priority = None
if priority is None:
priority = constants.OP_PRIO_DEFAULT
if names == locking.ALL_SET:
if opportunistic:
expand_fns = {
locking.LEVEL_CLUSTER: (lambda: [locking.BGL]),
locking.LEVEL_INSTANCE: self.cfg.GetInstanceList,
locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList,
locking.LEVEL_NODE: self.cfg.GetNodeList,
locking.LEVEL_NODE_RES: self.cfg.GetNodeList,
locking.LEVEL_NETWORK: self.cfg.GetNetworkList,
}
names = expand_fns[level]()
else:
names = locking.LOCKSET_NAME
names = _LockList(names)
# For locks of the same level, the lock order is lexicographic
names.sort()
levelname = locking.LEVEL_NAMES[level]
locks = ["%s/%s" % (levelname, lock) for lock in list(names)]
if not names:
logging.debug("Acquiring no locks for (%s) at level %s",
self._wconfdcontext, levelname)
return []
if shared:
request = [[lock, "shared"] for lock in locks]
else:
request = [[lock, "exclusive"] for lock in locks]
if request_only:
logging.debug("Lock request for level %s is %s", level, request)
return request
self.cfg.OutDate()
if timeout is None:
## Note: once we are so desperate for locks to request them
## unconditionally, we no longer care about an original plan
## to acquire locks opportunistically.
logging.info("Definitely requesting %s for %s", request,
self._wconfdcontext)
## The only way to be sure of not getting starved is to sequentially
## acquire the locks one by one (in lock order).
for r in request:
logging.debug("Definite request %s for %s", r,
self._wconfdcontext)
self.wconfd.Client().UpdateLocksWaiting(
self._wconfdcontext, priority, [r])
while True:
pending = self.wconfd.Client().HasPendingRequest(
self._wconfdcontext)
if not pending:
break
time.sleep(10.0 * random.random())
elif opportunistic:
logging.debug(
"For %ss trying to opportunistically acquire"
" at least %d of %s for %s.", timeout, opportunistic_count,
locks, self._wconfdcontext)
locks = utils.SimpleRetry(
lambda l: l != [],
self.wconfd.Client().GuardedOpportunisticLockUnion,
2.0,
timeout,
args=[opportunistic_count, self._wconfdcontext, request])
logging.debug("Managed to get the following locks: %s", locks)
if locks == []:
raise LockAcquireTimeout()
else:
self._RequestAndWait(request, timeout)
return locks
def _AcquireLocks(self, level, names, shared, opportunistic, timeout):
"""Acquires locks via the Ganeti lock manager.
@type level: int
@param level: Lock level
@type names: list or string
@param names: Lock names
@type shared: bool
@param shared: Whether the locks should be acquired in shared mode
@type opportunistic: bool
@param opportunistic: Whether to acquire opportunistically
@type timeout: None or float
@param timeout: Timeout for acquiring the locks
@raise LockAcquireTimeout: In case locks couldn't be acquired in specified
amount of time
"""
self._CheckLocksEnabled()
# TODO: honor priority in lock allocation
if self._cbs:
priority = self._cbs.CurrentPriority() # pylint: disable=W0612
else:
priority = None
if names == locking.ALL_SET:
if opportunistic:
expand_fns = {
locking.LEVEL_CLUSTER: (lambda: [locking.BGL]),
locking.LEVEL_INSTANCE: self.cfg.GetInstanceList,
locking.LEVEL_NODE_ALLOC: (lambda: [locking.NAL]),
locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList,
locking.LEVEL_NODE: self.cfg.GetNodeList,
locking.LEVEL_NODE_RES: self.cfg.GetNodeList,
locking.LEVEL_NETWORK: self.cfg.GetNetworkList,
}
names = expand_fns[level]()
else:
names = locking.LOCKSET_NAME
names = _LockList(names)
levelname = locking.LEVEL_NAMES[level]
locks = ["%s/%s" % (levelname, lock) for lock in list(names)]
if not names:
logging.debug("Acquiring no locks for (%s) at level %s",
self._wconfdcontext, levelname)
return []
if shared:
request = [[lock, "shared"] for lock in locks]
else:
request = [[lock, "exclusive"] for lock in locks]
if opportunistic:
logging.debug("Opportunistically acquring some of %s for %s.",
locks, self._wconfdcontext)
locks = self.wconfd.Client().OpportunisticLockUnion(
self._wconfdcontext, request)
elif timeout is None:
while True:
## TODO: use asynchronous wait instead of polling
blockedon = self.wconfd.Client().TryUpdateLocks(
self._wconfdcontext, request)
logging.debug("Requesting %s for %s blocked on %s", request,
self._wconfdcontext, blockedon)
if not blockedon:
break
time.sleep(random.random())
else:
logging.debug("Trying %ss to request %s for %s", timeout, request,
self._wconfdcontext)
## TODO: use blocking wait instead of polling
blocked = utils.SimpleRetry([],
self.wconfd.Client().TryUpdateLocks,
0.1,
timeout,
args=[self._wconfdcontext, request])
if blocked:
raise LockAcquireTimeout()
return locks