def pop(self, uniqId=0, erase=True):
'''
@brief: 弹出item
@param uniqId: item的id，如果为0，按FIFO弹出
@type uniqId: int
@param erase: 弹出后是否从字典里删除item
@type erase: bool
@return: item
@rtype: any type
'''
ret = None
# self.__lock.acquire()
lock = LockGuard(self.__lock)
if uniqId < 0 and self.__data:
uniqId = self.__data.keys()[0]
if uniqId in self.__queue:
self.__queue.remove(uniqId)
if not uniqId and self.__queue:
uniqId = self.__queue.pop(0)
if uniqId:
if erase:
ret = self.__data.pop(uniqId, None)
else:
ret = self.__data.get(uniqId, None)
# self.__lock.release()
return ret[0] if ret else None
def __updateConHashProxyWeighted(self):
tarsLogger.debug('AdapterProxyManager:updateConHashProxyWeighted')
lock = LockGuard(self.__newLock)
if len(self.__regAdapterProxyDict) == 0:
raise TarsException("the adapter proxy is empty")
self.__lastConHashPrxList = sorted(self.__regAdapterProxyDict.items(),
key=lambda item: item[0])
nodes = []
for var in self.__lastConHashPrxList:
nodes.append(var[0])
if self.__consistentHashWeight is None:
self.__consistentHashWeight = ConsistentHashNew(nodes)
else:
theOldActiveNodes = [
var for var in nodes
if var in self.__consistentHashWeight.nodes
]
theOldInactiveNodes = [
var for var in self.__consistentHashWeight.nodes
if var not in theOldActiveNodes
]
for var in theOldInactiveNodes:
self.__consistentHashWeight.removeNode(var)
theNewActiveNodes = [
var for var in nodes if var not in theOldActiveNodes
]
for var in theNewActiveNodes:
self.__consistentHashWeight.addNode(var)
self.__consistentHashWeight.nodes = nodes
pass
def __getNextValidProxy(self):
'''
@brief: 刷新本地缓存列表，如果服务下线了，要求删除本地缓存
@return:
@rtype: EndPointInfo列表
@todo: 优化负载均衡算法
'''
tarsLogger.debug('AdapterProxyManager:getNextValidProxy')
lock = LockGuard(self.__newLock)
if len(self.__adps) == 0:
raise TarsException("the activate adapter proxy is empty")
sortedActivateAdp = sorted(self.__adps.items(),
key=lambda item: item[1][2])
# self.refreshEndpoints()
# self.__lock.acquire()
sortedActivateAdpSize = len(sortedActivateAdp)
while sortedActivateAdpSize != 0:
if sortedActivateAdp[0][1][1].checkActive():
self.__adps[sortedActivateAdp[0][0]][2] += 1
# 返回的是 adapterProxy
return self.__adps[sortedActivateAdp[0][0]][1]
sortedActivateAdp.pop(0)
sortedActivateAdpSize -= 1
# 随机重连一个可用节点
adpPrx = self.__adps.items()[random.randint(0, len(self.__adps))][1][1]
adpPrx.checkActive()
return None
def timeout(self):
'''
@brief: 检测是否有item超时，如果有就删除
@return: None
@rtype: None
'''
endtime = time.time()
# self.__lock.acquire()
lock = LockGuard(self.__lock)
# 处理异常情况，防止死锁
try:
new_data = {}
new_queue = self.__queue[:]
for uniqId, item in self.__data.iteritems():
if endtime - item[1] < self.__timeout:
new_data[uniqId] = item
else:
if uniqId in new_queue:
new_queue.remove(uniqId)
tarsLogger.debug('TimeoutQueue:timeout remove id : %d' %
uniqId)
self.__data = new_data
self.__queue = new_queue
finally:
# self.__lock.release()
pass
def setEndpoints(self, eplist, ieplist):
'''
@brief: 设置服务端信息
@para eplist: 活跃的被调节点列表
@para ieplist: 不活跃的被调节点列表
'''
tarsLogger.debug('AdapterProxyManager:setEndpoints')
adps = {}
iadps = {}
comm = self.__comm
isNeedNotify = False
# self.__lock.acquire()
lock = LockGuard(self.__newLock)
isStartStatic = True
for ep in eplist:
if ep.getWeightType() == 0:
isStartStatic = False
epstr = str(ep)
if epstr in self.__adps:
adps[epstr] = self.__adps[epstr]
continue
isNeedNotify = True
self.__update = True
adapter = AdapterProxy()
adapter.initialize(ep, self.__object, comm.getReactor(),
comm.getAsyncProc())
adapter.activatestateinreg = True
adps[epstr] = [ep, adapter, 0]
self.__adps, adps = adps, self.__adps
for iep in ieplist:
iepstr = str(iep)
if iepstr in self.__iadps:
iadps[iepstr] = self.__iadps[iepstr]
continue
isNeedNotify = True
adapter = AdapterProxy()
adapter.initialize(iep, self.__object, comm.getReactor(),
comm.getAsyncProc())
adapter.activatestateinreg = False
iadps[iepstr] = [iep, adapter, 0]
self.__iadps, iadps = iadps, self.__iadps
if isStartStatic:
self.__weightType = EndpointWeightType.E_STATIC_WEIGHT
else:
self.__weightType = EndpointWeightType.E_LOOP
# self.__lock.release()
if isNeedNotify:
self.__notifyEndpoints(self.__adps, self.__iadps)
# 关闭已经失效的连接
for ep in adps:
if ep not in self.__adps:
adps[ep][1].terminate()
def size(self):
'''
@brief: 获取队列长度
@return: 队列长度
@rtype: int
'''
# self.__lock.acquire()
lock = LockGuard(self.__lock)
ret = len(self.__data)
# self.__lock.release()
return ret
def terminate(self):
'''
@brief: 释放资源
'''
tarsLogger.debug('AdapterProxyManager:terminate')
# self.__lock.acquire()
lock = LockGuard(self.__newLock)
for ep, epinfo in self.__adps.iteritems():
epinfo[1].terminate()
self.__adps = {}
self.__lock.release()
def __checkConHashChange(self, lastConHashPrxList):
tarsLogger.debug('AdapterProxyManager:checkConHashChange')
lock = LockGuard(self.__newLock)
if len(lastConHashPrxList) != len(self.__regAdapterProxyDict):
return True
regAdapterProxyList = sorted(
self.__regAdapterProxyDict.items(), key=lambda item: item[0])
regAdapterProxyListSize = len(regAdapterProxyList)
for index in range(regAdapterProxyListSize):
if cmp(lastConHashPrxList[index][0], regAdapterProxyList[index][0]) != 0:
return True
return False
def getEndpoints(self):
'''
@brief: 获取可用服务列表 如果启用分组,只返回同分组的服务端ip
@return: 获取节点列表
@rtype: EndPointInfo列表
'''
tarsLogger.debug('AdapterProxyManager:getEndpoints')
# self.__lock.acquire()
lock = LockGuard(self.__newLock)
ret = [x[1][0] for x in self.__adps.items()]
# self.__lock.release()
return ret
def generateId(self):
'''
@brief: 生成唯一id，0 < id < 2 ** 32
@return: id
@rtype: int
'''
# self.__lock.acquire()
lock = LockGuard(self.__lock)
ret = self.__uniqId
ret = (ret + 1) % 0x7FFFFFFF
while ret <= 0:
ret = (ret + 1) % 0x7FFFFFFF
self.__uniqId = ret
# self.__lock.release()
return ret
def peek(self, uniqId):
'''
@brief: 根据uniqId获取item，不会删除item
@param uniqId: item的id
@type uniqId: int
@return: item
@rtype: any type
'''
# self.__lock.acquire()
lock = LockGuard(self.__lock)
ret = self.__data.get(uniqId, None)
# self.__lock.release()
if not ret:
return None
return ret[0]
def __getWeightedProxy(self):
tarsLogger.debug('AdapterProxyManager:getWeightedProxy')
lock = LockGuard(self.__newLock)
if len(self.__adps) == 0:
raise TarsException("the activate adapter proxy is empty")
if self.__update is True:
self.__lastWeightedProxyData.clear()
weightedProxyData = {}
minWeight = (self.__adps.items()[0][1][0]).getWeight()
for item in self.__adps.items():
weight = (item[1][0].getWeight())
weightedProxyData[item[0]] = (weight)
if minWeight > weight:
minWeight = weight
if minWeight <= 0:
addWeight = -minWeight + 1
for item in weightedProxyData.items():
item[1] += addWeight
self.__update = False
self.__lastWeightedProxyData = weightedProxyData
weightedProxyData = self.__lastWeightedProxyData
while len(weightedProxyData) > 0:
total = sum(weightedProxyData.values())
rand = random.randint(1, total)
temp = 0
for item in weightedProxyData.items():
temp += item[1]
if rand <= temp:
if self.__adps[item[0]][1].checkActive():
self.__adps[item[0]][2] += 1
return self.__adps[item[0]][1]
else:
weightedProxyData.pop(item[0])
break
# 没有一个活跃的节点
# 随机重连一个可用节点
adpPrx = self.__adps.items()[random.randint(
0, len(self.__adps))][1][1]
adpPrx.checkActive()
return None
def checkActive(self, forceConnect=False):
'''
@brief: 检测连接是否失效
@param forceConnect: 是否强制发起连接，为true时不对状态进行判断就发起连接
@type forceConnect: bool
@return: 连接是否有效
@rtype: bool
'''
tarsLogger.debug('AdapterProxy:checkActive')
# self.__lock.acquire()
lock = LockGuard(self.__lock)
tarsLogger.info('checkActive, %s, forceConnect: %s',
self.__trans.getEndPointInfo(), forceConnect)
if not self.__trans.isConnecting() and not self.__trans.hasConnected():
self.doReconnect()
# self.__lock.release()
return self.__trans.isConnecting() or self.__trans.hasConnected()
def push(self, item, uniqId):
'''
@brief: 数据入队列，如果队列已经有了uniqId，插入失败
@param item: 插入的数据
@type item: any type
@return: 插入是否成功
@rtype: bool
'''
begtime = time.time()
ret = True
# self.__lock.acquire()
lock = LockGuard(self.__lock)
if uniqId in self.__data:
ret = False
else:
self.__data[uniqId] = [item, begtime]
self.__queue.append(uniqId)
# self.__lock.release()
return ret
def __getConHashProxyForNormal(self, hashCode):
tarsLogger.debug('AdapterProxyManager:getConHashProxyForNormal')
lock = LockGuard(self.__newLock)
if len(self.__regAdapterProxyDict) == 0:
raise TarsException("the adapter proxy is empty")
if self.__consistentHashWeight is None or self.__checkConHashChange(
self.__lastConHashPrxList):
self.__updateConHashProxyWeighted()
if len(self.__consistentHashWeight.nodes) > 0:
conHashIndex = self.__consistentHashWeight.getNode(hashCode)
if conHashIndex in self.__regAdapterProxyDict and self.__regAdapterProxyDict[
conHashIndex][
1].activatestateinreg and self.__regAdapterProxyDict[
conHashIndex][1].checkActive():
self.__regAdapterProxyDict[conHashIndex][2] += 1
if conHashIndex in self.__adps:
self.__adps[conHashIndex][2] += 1
elif conHashIndex in self.__iadps:
self.__iadps[conHashIndex][2] += 1
return self.__regAdapterProxyDict[conHashIndex][1]
else:
if len(self.__adps) == 0:
raise TarsException("the activate adapter proxy is empty")
activeProxyList = self.__adps.items()
actPrxSize = len(activeProxyList)
while actPrxSize != 0:
hashNum = hashCode % actPrxSize
if activeProxyList[hashNum][1][1].checkActive():
self.__adps[activeProxyList[hashNum][0]][2] += 1
return self.__adps[activeProxyList[hashNum][0]][1]
activeProxyList.pop(hashNum)
actPrxSize -= 1
# 随机重连一个可用节点
adpPrx = self.__adps.items()[random.randint(
0, len(self.__adps))][1][1]
adpPrx.checkActive()
return None
pass
else:
return self.__getHashProxyForNormal(hashCode)
def __getHashProxyForNormal(self, hashCode):
tarsLogger.debug('AdapterProxyManager:getHashProxyForNormal')
# self.__lock.acquire()
lock = LockGuard(self.__newLock)
regAdapterProxyList = sorted(self.__regAdapterProxyDict.items(),
key=lambda item: item[0])
allPrxSize = len(regAdapterProxyList)
if allPrxSize == 0:
raise TarsException("the adapter proxy is empty")
hashNum = hashCode % allPrxSize
if regAdapterProxyList[hashNum][1][
1].activatestateinreg and regAdapterProxyList[hashNum][1][
1].checkActive():
epstr = regAdapterProxyList[hashNum][0]
self.__regAdapterProxyDict[epstr][2] += 1
if epstr in self.__adps:
self.__adps[epstr][2] += 1
elif epstr in self.__iadps:
self.__iadps[epstr][2] += 1
return self.__regAdapterProxyDict[epstr][1]
else:
if len(self.__adps) == 0:
raise TarsException("the activate adapter proxy is empty")
activeProxyList = self.__adps.items()
actPrxSize = len(activeProxyList)
while actPrxSize != 0:
hashNum = hashCode % actPrxSize
if activeProxyList[hashNum][1][1].checkActive():
self.__adps[activeProxyList[hashNum][0]][2] += 1
return self.__adps[activeProxyList[hashNum][0]][1]
activeProxyList.pop(hashNum)
actPrxSize -= 1
# 随机重连一个可用节点
adpPrx = self.__adps.items()[random.randint(0, len(
self.__adps))][1][1]
adpPrx.checkActive()
return None
def __notifyEndpoints(self, actives, inactives):
# self.__lock.acquire()
lock = LockGuard(self.__newLock)
self.__regAdapterProxyDict.clear()
self.__regAdapterProxyDict.update(actives)
self.__regAdapterProxyDict.update(inactives)