def routineTrylock(key): l = Lock(key, rc.scheduler) locked = l.trylock() result.append(locked) for m in rc.waitWithTimeout(0.5): yield m l.unlock()
def send_batch(self, connection, container, *cmds): ''' Send multiple commands to redis server at once :param connection: redis connection :param container: routine container :param *cmds: commands to send. Each command is a tuple/list of bytes/str. :returns: list of reply event matchers (from container.retvalue) ''' if not cmds: raise RedisProtocolException('No commands') l = Lock(connection.redis_locker, connection.scheduler) for m in l.lock(container): yield m with l: commands = [] matchers = [] for c in cmds: try: r, reply_matcher = self._prepare_command(connection, c) commands.append(r) matchers.append(reply_matcher) except: self._logger.warning('Error in one of the commands in a batch: %r. The command is ignored.', c, exc_info = True) if not commands: raise RedisProtocolException('Error for every command in a batch') for m in connection.write(ConnectionWriteEvent(connection, connection.connmark, data = b''.join(commands)), False): yield m container.retvalue = matchers
async def routineLock(key): l = Lock(key, rc.scheduler) await l.lock(rc) t = obj[0] await rc.do_events() obj[0] = t + 1 l.unlock()
def _send_batch(self, connection, container, *cmds): "Use delegate to ensure it always ends" if not cmds: raise RedisProtocolException('No commands') l = Lock(connection.redis_locker, connection.scheduler) for m in l.lock(container): yield m with l: commands = [] matchers = [] for c in cmds: try: r, reply_matcher = self._prepare_command(connection, c) commands.append(r) matchers.append(reply_matcher) except: self._logger.warning( 'Error in one of the commands in a batch: %r. The command is ignored.', c, exc_info=True) if not commands: raise RedisProtocolException( 'Error for every command in a batch') for m in connection.write( ConnectionWriteEvent(connection, connection.connmark, data=b''.join(commands)), False): yield m container.retvalue = matchers
def routineLock(key): l = Lock(key, rc.scheduler) for m in l.lock(rc): yield m t = obj[0] for m in rc.waitWithTimeout(0.5): yield m obj[0] = t + 1 l.unlock()
def _send_command(self, connection, container, *args): if not args: raise RedisProtocolException('No command name') l = Lock(connection.redis_locker, connection.scheduler) # The socket write sequence must be the same as the send sequence, add a lock to ensure that for m in l.lock(container): yield m with l: r, reply_matcher = self._prepare_command(connection, args) for m in connection.write(ConnectionWriteEvent(connection, connection.connmark, data = r), False): yield m container.retvalue = reply_matcher
async def routineLock(key): l = Lock(key, rc.scheduler) locked = l.beginlock(rc) if not locked: await rc.wait_with_timeout(0.2) await l.lock(rc) t = obj[0] await rc.do_events() obj[0] = t + 1 l.unlock() await rc.do_events() await l.lock(rc) t = obj[0] if t != 2: obj[0] = t - 1 l.unlock()
async def async_requests(self, connection, requests, container=None, priority=0): ''' :return: (matchers, sendall), where matchers are event matchers for the requests; sendall is an async function to send to requests. Use `await sendall()` to send the requests. ''' async with Lock((connection, 'async_requests', priority), connection.scheduler): matchers = [] for r in requests: xid = self._pre_assign_xid(connection, r) resp_matcher = ZooKeeperResponseEvent.createMatcher( connection, connection.connmark, None, xid) matchers.append(resp_matcher) alldata = [] for r in requests: if priority < ZooKeeperWriteEvent.HIGH: # Test if already limited by consuming 1 byte await connection._rate_limiter[priority].limit(1) data = r._tobytes() if len(data) >= 0xfffff: # This is the default limit of ZooKeeper, reject this request raise ZooKeeperRequestTooLargeException( 'The request is %d bytes which is too large for ZooKeeper' % len(data)) if priority < ZooKeeperWriteEvent.HIGH: await connection._rate_limiter[priority].limit( len(data) - 1) alldata.append(data) for r in requests: self._register_xid(connection, r) async def _sendall(): sent_requests = [] for data in alldata: try: sent_requests.append(await self._senddata( connection, data, container, priority)) except ZooKeeperRetryException: raise ZooKeeperRetryException(sent_requests) return sent_requests return (matchers, _sendall)
async def routineLock(key): l = Lock(key, rc.scheduler) locked = l.beginlock(rc) if not locked: await rc.wait_with_timeout(0.2) locked = l.trylock() if not locked: raise ValueError('Not locked') t = obj[0] await rc.do_events() obj[0] = t + 1 l.unlock() await rc.do_events() await l.lock(rc) t = obj[0] await rc.do_events() obj[0] = t + 1 l.unlock()
def send_command(self, connection, container, *args): ''' Send command to Redis server. :param connection: Redis connection :param container: routine container :param *args: command paramters, begin with command name, e.g. 'SET','key','value' :returns: Event matcher to wait for reply. The value is returned from container.retvalue ''' if not args: raise RedisProtocolException('No command name') l = Lock(connection.redis_locker, connection.scheduler) # The socket write sequence must be the same as the send sequence, add a lock to ensure that for m in l.lock(container): yield m with l: r, reply_matcher = self._prepare_command(connection, args) for m in connection.write(ConnectionWriteEvent(connection, connection.connmark, data = r), False): yield m container.retvalue = reply_matcher
def routineLock(key): l = Lock(key, rc.scheduler) locked = l.beginlock(rc) if not locked: for m in rc.waitWithTimeout(0.5): yield m for m in l.lock(rc): yield m t = obj[0] for m in rc.waitWithTimeout(1.0): yield m obj[0] = t + 1 l.unlock() for m in rc.doEvents(): yield m for m in l.lock(rc): yield m t = obj[0] if t != 2: obj[0] = t - 1 l.unlock()
def routineLock(key): l = Lock(key, rc.scheduler) locked = l.beginlock(rc) if not locked: for m in rc.waitWithTimeout(1.0): yield m locked = l.trylock() if not locked: raise ValueError('Not locked') t = obj[0] for m in rc.waitWithTimeout(0.5): yield m obj[0] = t + 1 l.unlock() for m in rc.doEvents(): yield m for m in l.lock(rc): yield m t = obj[0] for m in rc.waitWithTimeout(1.0): yield m obj[0] = t + 1 l.unlock()
def ipam_requestpool(self, env, params): if params['AddressSpace'] != _GLOBAL_SPACE: raise ValueError( 'Unsupported address space: must use this IPAM driver together with network driver' ) if params['V6']: raise ValueError('IPv6 is not supported') new_pool = IPAMReserve.create_instance(uuid1().hex) new_pool.pool = params['Pool'] if new_pool.pool: subnet, mask = parse_ip4_network(new_pool.pool) new_pool.pool = ip4_addr.formatter(subnet) + '/' + str(mask) new_pool.subpool = params['SubPool'] if new_pool.subpool: subnet, mask = parse_ip4_network(new_pool.subpool) new_pool.subpool = ip4_addr.formatter(subnet) + '/' + str(mask) new_pool.options = params['Options'] if new_pool.pool: l = Lock(('dockerplugin_ipam_request_pool', new_pool.pool), self.scheduler) for m in l.lock(self): yield m else: l = None try: while True: fail = 0 rets = [] def _updater(keys, values, timestamp): reservepool = values[0] reserve_new_pool = set_new(values[1], new_pool) remove_keys = self._remove_staled_pools( reservepool, timestamp) used_cidrs = set( cidr for _, (cidr, _) in reservepool.reserved_pools.items()) if not reserve_new_pool.pool: # pool is not specified for _ in range(0, self.cidrrange_end): reservepool.nextalloc += 1 if reservepool.nextalloc >= self.cidrrange_end: reservepool.nextalloc = 0 new_subnet = self.cidrrange_subnet | ( reservepool.nextalloc << 8) new_cidr = ip4_addr.formatter(new_subnet) + '/24' if new_cidr not in used_cidrs: break reserve_new_pool.pool = new_cidr reserve_new_pool.subpool = '' rets[:] = [reserve_new_pool.pool] if reserve_new_pool.pool in used_cidrs: # We must wait until this CIDR is released raise IPAMUsingException reservepool.reserved_pools[reserve_new_pool.id] = \ [reserve_new_pool.pool, timestamp + self.pooltimeout * 1000000] marker = IPAMReserveMarker.create_instance( reserve_new_pool.pool) if marker.getkey() in remove_keys: remove_keys.remove(marker.getkey()) return (tuple(keys[0:2]) + tuple(remove_keys), (reservepool, reserve_new_pool) + (None, ) * len(remove_keys)) else: return (tuple(keys[0:2]) + (marker.getkey(), ) + tuple(remove_keys), (reservepool, reserve_new_pool, marker) + (None, ) * len(remove_keys)) try: for m in callAPI( self, 'objectdb', 'transact', { 'keys': (IPAMPoolReserve.default_key(), new_pool.getkey()), 'updater': _updater, 'withtime': True }): yield m except IPAMUsingException: # Wait for the CIDR to be released self._reqid += 1 fail += 1 reqid = ('dockerplugin_ipam', self._reqid) marker_key = IPAMReserveMarker.default_key(rets[0]) for m in callAPI(self, 'objectdb', 'get', { 'key': marker_key, 'requestid': reqid, 'nostale': True }): yield m retvalue = self.retvalue with watch_context([marker_key], [retvalue], reqid, self): if retvalue is not None and not retvalue.isdeleted(): for m in self.executeWithTimeout( self.pooltimeout, retvalue.waitif(self, lambda x: x.isdeleted())): yield m else: env.outputjson({ 'PoolID': new_pool.id, 'Pool': rets[0], 'Data': {} }) break finally: if l is not None: l.unlock()
def __init__(self, sessionobj, container): self.sessionobj = sessionobj self.id = sessionobj.id self.vars = sessionobj.vars self._lock = Lock(sessionobj, container.scheduler) self.container = container
async def routineTrylock(key): l = Lock(key, rc.scheduler) locked = l.trylock() result.append(locked) await rc.do_events() l.unlock()
async def routineLock(key): l = Lock(key, rc.scheduler) async with l: t = obj[0] await rc.do_events() obj[0] = t + 1
def _retry_write(self, process, vhost): c = self._redis_clients.get(vhost) # Always try once first while True: for m in c.get_connection(self.apiroutine): yield m newconn = self.apiroutine.retvalue with newconn.context(self.apiroutine): try: for m in process(newconn): yield m except RedisConnectionDown: continue except RedisWriteConflictException: break else: return enterseq = self.enterseq enterlock = self.enterlock for i in range(0, self.maxretry): for m in c.get_connection(self.apiroutine): yield m newconn = self.apiroutine.retvalue with newconn.context(self.apiroutine): if self._sequencial: curr_time = time() if self._sequencialsince + 1.0 < curr_time: self._sequencial = False else: self._sequencialsince = curr_time if self._sequencial: # First make all the conflicted updates in this process retry in sequence l = Lock(self, self.scheduler) for m in l.lock(self.apiroutine): yield m with l: try: for m in process(newconn): yield m except RedisConnectionDown: continue except RedisWriteConflictException: if i > enterseq and not self._sequencial: self._sequencial = True self._sequencialsince = time() if i > enterlock: try: # Maybe too many VLCP processes are trying to write the same Redis server # Set a special flag in Redis with very short expire time # If multiple VLCP processes tries to set the flag at the same time, some # will be blocked for a short time to leave time for the winner for i in range(0, 100): for m in newconn.execute_command(self.apiroutine, 'SET', 'vlcp._reserved.redisdb.connwriteblock', '1', 'PX', '100', 'NX'): yield m if self.apiroutine.retvalue is not None: break else: for m in self.apiroutine.waitWithTimeout(0.02): yield m except Exception: self._logger.warning('Exception raised on waiting for a lock, will ignore and continue', exc_info = True) for m in self.apiroutine.waitWithTimeout(0.1): yield m else: return else: try: for m in process(newconn): yield m except RedisConnectionDown: continue except RedisWriteConflictException: if i > enterseq and not self._sequencial: self._sequencial = True self._sequencialsince = time() else: return raise RedisWriteConflictException('Transaction still fails after many retries')