def subscribe():
try:
self.asyncView.subscribeToKeyspace(SharedStateNative.getClientInfoKeyspace(), 1, self.onNewClientInfo)
doneDeferred = self.asyncView.subscribeToKeyspace(self.workerStatusKeyspace, 0, self.onNewWorkerStatus)
doneDeferred.addCallbacks(subscribeCallback, lambda exception : None)
except UserWarning as ex:
logging.warn("Failed to subscribe to asyncView keyspace because we disconnected from shared state while reconnecting to shared state")
def connectedClientInfo(view):
"""given a view subscribed to the client info keyspace, computes all the connected
clients"""
clientInfoKeyspace = SharedStateNative.getClientInfoKeyspace()
k = view.nextKey(Key(clientInfoKeyspace, (NativeJson.lowestValue(), NativeJson.lowestValue())))
maxId = 0
tr = set()
while k is not None:
if k.keyspace != clientInfoKeyspace.name:
return tr, maxId
if view[k].value() != NativeJson.Json('disconnected'):
tr.add(k[1])
maxId = max(maxId, view[k].id())
k = view.nextKey(k)
return tr, maxId
def connectedClientInfo(view):
"""given a view subscribed to the client info keyspace, computes all the connected
clients"""
clientInfoKeyspace = SharedStateNative.getClientInfoKeyspace()
k = view.nextKey(
Key(clientInfoKeyspace,
(NativeJson.lowestValue(), NativeJson.lowestValue())))
maxId = 0
tr = set()
while k is not None:
if k.keyspace != clientInfoKeyspace.name:
return tr, maxId
if view[k].value() != NativeJson.Json('disconnected'):
tr.add(k[1])
maxId = max(maxId, view[k].id())
k = view.nextKey(k)
return tr, maxId
def updateActiveMachines(self):
if self.shouldStop():
return
if self._lastDisconnectTime is not None:
if self._lastReconnectTime is None:
return
if time.time(
) - self._lastReconnectTime < TIME_TO_SLEEP_AFTER_RECONNECT:
#we need to defer until later
return
logging.debug('%s attempting to get introspection information',
self.clientID)
try:
introspectionInfo = self.asyncView.keyspaceItems(
SharedStateNative.getClientInfoKeyspace())
except UserWarning:
logging.info('AsyncView received exception from View:\n%s',
traceback.format_exc())
self.setDisconnected()
raise
currentlyConnectedClientIds = set([
key[1].toSimple() for key, value in introspectionInfo
if value.toSimple() != 'disconnected'
])
currentlyConnectedWorkers = set()
for clientId in currentlyConnectedClientIds:
machineId = self.clientIdToMachineIdAsString(clientId)
if machineId:
currentlyConnectedWorkers.add(machineId)
logging.debug('%s currently connected list is %s',
self.ownMachineIdAsString, currentlyConnectedWorkers)
with self._lock:
for deadWorker in self.activeMachineIds - currentlyConnectedWorkers:
if deadWorker != self.ownMachineIdAsString():
logging.info("Worker dropped: %s which is not our ID (%s)",
deadWorker, self.ownMachineIdAsString())
self.activeMachineIds.remove(deadWorker)
if deadWorker not in self.machineIdToClientId:
logging.critical(
"Worker %s is dead, but I don't have a client id for it.",
deadWorker)
assert False
else:
deadClientID = self.machineIdToClientId[deadWorker]
self.onWorkerDrop(
deadClientID, *self.
clientIdToIpPortAndMachineIdAsString[deadClientID])
newlyAliveClientIds = []
with self._lock:
for newlyAliveClientId in set(
self.clientIdToIpPortAndMachineIdAsString.keys(
)).intersection(currentlyConnectedClientIds):
nowAliveMachineId = self.clientIdToMachineIdAsString(
newlyAliveClientId)
if nowAliveMachineId is not None and nowAliveMachineId not in self.activeMachineIds:
logging.debug(
"Worker clientId=%s added with IP %s and ports %s, machineIdAsString=%s",
nowAliveMachineId,
*self.clientIdToIpPortAndMachineIdAsString[
newlyAliveClientId])
self.activeMachineIds.add(nowAliveMachineId)
# Defer onWorkerAdd notifications.
newlyAliveClientIds.append(newlyAliveClientId)
logging.debug("Active workers: %s", self.activeMachineIds)
for newlyAliveClientId in newlyAliveClientIds:
self.onWorkerAdd(
newlyAliveClientId,
*self.clientIdToIpPortAndMachineIdAsString[newlyAliveClientId])
def updateActiveMachines(self):
if self.shouldStop():
return
if self._lastDisconnectTime is not None:
if self._lastReconnectTime is None:
return
if time.time() - self._lastReconnectTime < TIME_TO_SLEEP_AFTER_RECONNECT:
#we need to defer until later
return
logging.debug('%s attempting to get introspection information', self.clientID)
try:
introspectionInfo = self.asyncView.keyspaceItems(SharedStateNative.getClientInfoKeyspace())
except UserWarning:
logging.info('AsyncView received exception from View:\n%s', traceback.format_exc())
self.setDisconnected()
raise
currentlyConnectedClientIds = set([key[1].toSimple() for key, value in introspectionInfo if value.toSimple() != 'disconnected'])
currentlyConnectedWorkers = set()
for clientId in currentlyConnectedClientIds:
machineId = self.clientIdToMachineIdAsString(clientId)
if machineId:
currentlyConnectedWorkers.add(machineId)
logging.debug('%s currently connected list is %s', self.ownMachineIdAsString, currentlyConnectedWorkers)
with self._lock:
for deadWorker in self.activeMachineIds - currentlyConnectedWorkers:
if deadWorker != self.ownMachineIdAsString():
logging.info("Worker dropped: %s which is not our ID (%s)", deadWorker, self.ownMachineIdAsString())
self.activeMachineIds.remove(deadWorker)
if deadWorker not in self.machineIdToClientId:
logging.critical("Worker %s is dead, but I don't have a client id for it.", deadWorker)
assert False
else:
deadClientID = self.machineIdToClientId[deadWorker]
self.onWorkerDrop(deadClientID, *self.clientIdToIpPortAndMachineIdAsString[deadClientID])
newlyAliveClientIds = []
with self._lock:
for newlyAliveClientId in set(self.clientIdToIpPortAndMachineIdAsString.keys()).intersection(currentlyConnectedClientIds):
nowAliveMachineId = self.clientIdToMachineIdAsString(newlyAliveClientId)
if nowAliveMachineId is not None and nowAliveMachineId not in self.activeMachineIds:
logging.info(
"Worker clientId=%s added with IP %s and ports %s, machineIdAsString=%s",
nowAliveMachineId,
*self.clientIdToIpPortAndMachineIdAsString[newlyAliveClientId]
)
self.activeMachineIds.add(nowAliveMachineId)
# Defer onWorkerAdd notifications.
newlyAliveClientIds.append(newlyAliveClientId)
logging.debug("Active workers: %s", self.activeMachineIds)
for newlyAliveClientId in newlyAliveClientIds:
self.onWorkerAdd(newlyAliveClientId, *self.clientIdToIpPortAndMachineIdAsString[newlyAliveClientId])