def _RequestAndWait(self, request, timeout): """Request locks from WConfD and wait for them to be granted. @type request: list @param request: the lock request to be sent to WConfD @type timeout: float @param timeout: the time to wait for the request to be granted @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time; in this case, locks still might be acquired or a request pending. """ logging.debug("Trying %ss to request %s for %s", timeout, request, self._wconfdcontext) if self._cbs: priority = self._cbs.CurrentPriority() # pylint: disable=W0612 else: priority = None if priority is None: priority = constants.OP_PRIO_DEFAULT ## Expect a signal if sighupReceived[0]: logging.warning("Ignoring unexpected SIGHUP") sighupReceived[0] = False # Request locks self.wconfd.Client().UpdateLocksWaiting(self._wconfdcontext, priority, request) pending = self.wconfd.Client().HasPendingRequest(self._wconfdcontext) if pending: def _HasPending(): if sighupReceived[0]: return self.wconfd.Client().HasPendingRequest( self._wconfdcontext) else: return True pending = utils.SimpleRetry(False, _HasPending, 0.05, timeout) signal = sighupReceived[0] if pending: pending = self.wconfd.Client().HasPendingRequest( self._wconfdcontext) if pending and signal: logging.warning("Ignoring unexpected SIGHUP") sighupReceived[0] = False logging.debug("Finished trying. Pending: %s", pending) if pending: raise LockAcquireTimeout()
def testSimpleRetry(self): self.assertFalse( utils.SimpleRetry(True, lambda: False, 0.01, 0.02, wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertFalse( utils.SimpleRetry(lambda x: x, lambda: False, 0.01, 0.02, wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertTrue( utils.SimpleRetry(True, lambda: True, 0, 1, wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertTrue( utils.SimpleRetry(lambda x: x, lambda: True, 0, 1, wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertTrue( utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1, args=[1], wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertEqual(self.retries, 1) self.assertEqual(self.called, 2) self.called = self.retries = 0 self.assertTrue( utils.SimpleRetry(True, self._SimpleRetryAndSucceed, 0, 1, args=[2], wait_fn=self._wait_fn, _time_fn=self._time_fn)) self.assertEqual(self.called, 3)
def _AcquireLocks(self, level, names, shared, opportunistic, timeout, opportunistic_count=1, request_only=False): """Acquires locks via the Ganeti lock manager. @type level: int @param level: Lock level @type names: list or string @param names: Lock names @type shared: bool @param shared: Whether the locks should be acquired in shared mode @type opportunistic: bool @param opportunistic: Whether to acquire opportunistically @type timeout: None or float @param timeout: Timeout for acquiring the locks @type request_only: bool @param request_only: do not acquire the locks, just return the request @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time; in this case, locks still might be acquired or a request pending. """ self._CheckLocksEnabled() if self._cbs: priority = self._cbs.CurrentPriority() # pylint: disable=W0612 else: priority = None if priority is None: priority = constants.OP_PRIO_DEFAULT if names == locking.ALL_SET: if opportunistic: expand_fns = { locking.LEVEL_CLUSTER: (lambda: [locking.BGL]), locking.LEVEL_INSTANCE: self.cfg.GetInstanceList, locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList, locking.LEVEL_NODE: self.cfg.GetNodeList, locking.LEVEL_NODE_RES: self.cfg.GetNodeList, locking.LEVEL_NETWORK: self.cfg.GetNetworkList, } names = expand_fns[level]() else: names = locking.LOCKSET_NAME names = _LockList(names) # For locks of the same level, the lock order is lexicographic names.sort() levelname = locking.LEVEL_NAMES[level] locks = ["%s/%s" % (levelname, lock) for lock in list(names)] if not names: logging.debug("Acquiring no locks for (%s) at level %s", self._wconfdcontext, levelname) return [] if shared: request = [[lock, "shared"] for lock in locks] else: request = [[lock, "exclusive"] for lock in locks] if request_only: logging.debug("Lock request for level %s is %s", level, request) return request self.cfg.OutDate() if timeout is None: ## Note: once we are so desperate for locks to request them ## unconditionally, we no longer care about an original plan ## to acquire locks opportunistically. logging.info("Definitely requesting %s for %s", request, self._wconfdcontext) ## The only way to be sure of not getting starved is to sequentially ## acquire the locks one by one (in lock order). for r in request: logging.debug("Definite request %s for %s", r, self._wconfdcontext) self.wconfd.Client().UpdateLocksWaiting( self._wconfdcontext, priority, [r]) while True: pending = self.wconfd.Client().HasPendingRequest( self._wconfdcontext) if not pending: break time.sleep(10.0 * random.random()) elif opportunistic: logging.debug( "For %ss trying to opportunistically acquire" " at least %d of %s for %s.", timeout, opportunistic_count, locks, self._wconfdcontext) locks = utils.SimpleRetry( lambda l: l != [], self.wconfd.Client().GuardedOpportunisticLockUnion, 2.0, timeout, args=[opportunistic_count, self._wconfdcontext, request]) logging.debug("Managed to get the following locks: %s", locks) if locks == []: raise LockAcquireTimeout() else: self._RequestAndWait(request, timeout) return locks
def _AcquireLocks(self, level, names, shared, opportunistic, timeout): """Acquires locks via the Ganeti lock manager. @type level: int @param level: Lock level @type names: list or string @param names: Lock names @type shared: bool @param shared: Whether the locks should be acquired in shared mode @type opportunistic: bool @param opportunistic: Whether to acquire opportunistically @type timeout: None or float @param timeout: Timeout for acquiring the locks @raise LockAcquireTimeout: In case locks couldn't be acquired in specified amount of time """ self._CheckLocksEnabled() # TODO: honor priority in lock allocation if self._cbs: priority = self._cbs.CurrentPriority() # pylint: disable=W0612 else: priority = None if names == locking.ALL_SET: if opportunistic: expand_fns = { locking.LEVEL_CLUSTER: (lambda: [locking.BGL]), locking.LEVEL_INSTANCE: self.cfg.GetInstanceList, locking.LEVEL_NODE_ALLOC: (lambda: [locking.NAL]), locking.LEVEL_NODEGROUP: self.cfg.GetNodeGroupList, locking.LEVEL_NODE: self.cfg.GetNodeList, locking.LEVEL_NODE_RES: self.cfg.GetNodeList, locking.LEVEL_NETWORK: self.cfg.GetNetworkList, } names = expand_fns[level]() else: names = locking.LOCKSET_NAME names = _LockList(names) levelname = locking.LEVEL_NAMES[level] locks = ["%s/%s" % (levelname, lock) for lock in list(names)] if not names: logging.debug("Acquiring no locks for (%s) at level %s", self._wconfdcontext, levelname) return [] if shared: request = [[lock, "shared"] for lock in locks] else: request = [[lock, "exclusive"] for lock in locks] if opportunistic: logging.debug("Opportunistically acquring some of %s for %s.", locks, self._wconfdcontext) locks = self.wconfd.Client().OpportunisticLockUnion( self._wconfdcontext, request) elif timeout is None: while True: ## TODO: use asynchronous wait instead of polling blockedon = self.wconfd.Client().TryUpdateLocks( self._wconfdcontext, request) logging.debug("Requesting %s for %s blocked on %s", request, self._wconfdcontext, blockedon) if not blockedon: break time.sleep(random.random()) else: logging.debug("Trying %ss to request %s for %s", timeout, request, self._wconfdcontext) ## TODO: use blocking wait instead of polling blocked = utils.SimpleRetry([], self.wconfd.Client().TryUpdateLocks, 0.1, timeout, args=[self._wconfdcontext, request]) if blocked: raise LockAcquireTimeout() return locks