def increaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition
)
with self.lock_:
if computationId not in self.computedValuesForComputations:
self.computedValuesForComputations[computationId] = set()
self.computedValuesForComputations[computationId].add(compValue)
self.refcountsForCompIds_[computationId] += 1
if self.refcountsForCompIds_[computationId] == 1:
self.cumulusGateway.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(self.allocNewPriority_())
)
if computationId in self.finishedResultsForComputations:
result, statistics = self.finishedResultsForComputations[computationId]
BackgroundUpdateQueue.push(
self.valueUpdater(
compValue,
result,
statistics
)
)
def resetStateEntirely(self):
"""Cancel all computations and clear the compute cache."""
self.cancelAllComputations(True)
with self.lock_:
computations = list(self.refcountsForCompIds_.keys())
for computationId in computations:
if computationId in self.finishedResultsForComputations:
del self.finishedResultsForComputations[computationId]
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
None,
None
)
)
for vecId in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[vecId]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
self.vectorDataIDRequestCount_ = {}
self.vectorDataIDToVectorSlices_ = {}
def waitForResults(graph, values, timeout=60.0):
"""wait for 'x' to have 'result' not be zero. return the maximum number
of CPUs ever observed to be allocated to 'x'"""
t0 = time.time()
def notLoadedCount():
notLoadedCount = 0
for x in values:
if x.valueIVC is None:
notLoadedCount += 1
return notLoadedCount
with IncreasedRequestCount(values):
maxCPUs = None
while notLoadedCount() > 0 and time.time() - t0 < timeout:
time.sleep(.01)
BackgroundUpdateQueue.pullOne(timeout=.10)
graph.flush()
for x in values:
if x.totalWorkerCount is not None:
if maxCPUs is None:
maxCPUs = x.totalWorkerCount
else:
maxCPUs = max(maxCPUs, x.totalWorkerCount)
assert notLoadedCount(
) == 0, "Timed out: %s of %s didn't finish:\n\t%s\n" % (
notLoadedCount(), len(values), "\n\t".join(
[str(x) for x in values if x.valueIVC is None]))
return maxCPUs
def increaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition
)
with self.lock_:
if computationId not in self.computedValuesForComputations:
self.computedValuesForComputations[computationId] = set()
self.computedValuesForComputations[computationId].add(compValue)
self.refcountsForCompIds_[computationId] += 1
if self.refcountsForCompIds_[computationId] == 1:
self.cumulusGateway.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(self.allocNewPriority_())
)
if computationId in self.finishedResultsForComputations:
result, statistics = self.finishedResultsForComputations[computationId]
BackgroundUpdateQueue.push(
self.valueUpdater(
compValue,
result,
statistics
)
)
def resetStateEntirely(self):
"""Cancel all computations and clear the compute cache."""
self.cancelAllComputations(True)
with self.lock_:
computations = list(self.refcountsForCompIds_.keys())
for computationId in computations:
if computationId in self.finishedResultsForComputations:
del self.finishedResultsForComputations[computationId]
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
None,
None
)
)
for vecId in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[vecId]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
self.vectorDataIDRequestCount_ = {}
self.vectorDataIDToVectorSlices_ = {}
def onComputationResult(self, computationId, result, statistics):
with self.lock_:
self.finishedResultsForComputations[computationId] = (result,
statistics)
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[
computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(compVal, result, statistics))
def onCPUCountChanged(self, computationSystemwideCpuAssignment):
with self.lock_:
computationId = computationSystemwideCpuAssignment.computation
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[
computationId]:
BackgroundUpdateQueue.push(
self.cpuCountSetter_(
compVal, computationSystemwideCpuAssignment))
def onNewGlobalUserFacingLogMessage(self, newMsg):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().pushNewGlobalUserFacingLogMessage(newMsg)
BackgroundUpdateQueue.push(updater)
def onNewGlobalUserFacingLogMessage(self, newMsg):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().pushNewGlobalUserFacingLogMessage(newMsg)
BackgroundUpdateQueue.push(updater)
def onCacheLoad(self, vectorDataID):
with self.lock_:
if vectorDataID in self.vectorDataIDToVectorSlices_:
cgLocations = self.vectorDataIDToVectorSlices_[vectorDataID]
else:
cgLocations = set()
self.collectOffloadedVectors_()
for cgLocation in cgLocations:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, True))
def onCacheLoad(self, vectorDataID):
with self.lock_:
if vectorDataID in self.vectorDataIDToVectorSlices_:
cgLocations = self.vectorDataIDToVectorSlices_[vectorDataID]
else:
cgLocations = set()
self.collectOffloadedVectors_()
for cgLocation in cgLocations:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, self.computeVectorSliceIsLoaded_(cgLocation)))
def onJsonViewOfSystemChanged(self, json):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().viewOfSystem_ = json.toSimple()
PersistentCacheIndex.PersistentCacheIndex().update()
BackgroundUpdateQueue.push(updater)
def onJsonViewOfSystemChanged(self, json):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().viewOfSystem_ = json.toSimple()
PersistentCacheIndex.PersistentCacheIndex().update()
BackgroundUpdateQueue.push(updater)
def onComputationResult(self, computationId, result, statistics):
with self.lock_:
self.finishedResultsForComputations[computationId] = (result, statistics)
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
result,
statistics
)
)
def onCPUCountChanged(self, computationSystemwideCpuAssignment):
with self.lock_:
computationId = computationSystemwideCpuAssignment.computation
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.cpuCountSetter_(
compVal,
computationSystemwideCpuAssignment
)
)
def waitUntilTrue(self, predicate, timeout = 10.0):
t0 = time.time()
interval = 0.001
while not predicate():
time.sleep(interval)
BackgroundUpdateQueue.pullOne(timeout = interval)
self.graph.flush()
if time.time() - t0 > timeout:
self.assertFalse(True, "timed out")
return
interval = min(interval * 2, 0.1)
def onExternalIoTaskCompleted(self, msg):
taskGuid = msg.taskId.guid
with self.lock_:
if taskGuid not in self.externalIoTaskCallbacks_:
logging.warn("TaskId %s was not found in the task guid list", taskGuid)
return
callback = self.externalIoTaskCallbacks_[taskGuid]
del self.externalIoTaskCallbacks_[taskGuid]
def executor():
callback(msg.result)
BackgroundUpdateQueue.push(executor)
def onExternalIoTaskCompleted(self, msg):
taskGuid = msg.taskId.guid
with self.lock_:
if taskGuid not in self.externalIoTaskCallbacks_:
logging.warn("TaskId %s was not found in the task guid list", taskGuid)
return
callback = self.externalIoTaskCallbacks_[taskGuid]
del self.externalIoTaskCallbacks_[taskGuid]
def executor():
callback(msg.result)
BackgroundUpdateQueue.push(executor)
def collectOffloadedVectors_(self):
offloaded = self.ramCacheOffloadRecorder.extractDropped()
if offloaded:
logging.info("ComputedValue RamCache dropped %s", offloaded)
for offloadedVecDataID in offloaded:
if offloadedVecDataID in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[offloadedVecDataID]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
if offloaded:
#check if there's anything we need to load
self.sendReloadRequests()
def collectOffloadedVectors_(self):
offloaded = self.ramCacheOffloadRecorder.extractDropped()
if offloaded:
logging.info("ComputedValue RamCache dropped %s", offloaded)
for offloadedVecDataID in offloaded:
if offloadedVecDataID in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[offloadedVecDataID]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
if offloaded:
#check if there's anything we need to load
self.sendReloadRequests()
def waitUntilTrue(self, predicate, timeout=10.0):
t0 = time.time()
interval = 0.001
while not predicate():
time.sleep(interval)
BackgroundUpdateQueue.pullOne(timeout=interval)
self.graph.flush()
if time.time() - t0 > timeout:
self.assertFalse(True, "timed out")
return
interval = min(interval * 2, 0.1)
def updateComputedGraph_(self):
self.sharedStateSynchronizer.update()
BackgroundUpdateQueue.moveNextFrameToCurFrame()
BackgroundUpdateQueue.pullAll()
self.computedGraph.flushOrphans()
self.computedGraph.flush()
self.controlRoot.pruneDirtyChildren()
self.controlRoot.update()
#self.controlRoot.display()
self.sharedStateSynchronizer.commitPendingWrites()
def updateComputedGraph_(self):
self.sharedStateSynchronizer.update()
BackgroundUpdateQueue.moveNextFrameToCurFrame()
BackgroundUpdateQueue.pullAll()
self.computedGraph.flushOrphans()
self.computedGraph.flush()
self.controlRoot.pruneDirtyChildren()
self.controlRoot.update()
#self.controlRoot.display()
self.sharedStateSynchronizer.commitPendingWrites()
def waitCacheItemsAreLoaded(graph, cacheItems, timeout = 60.0):
with IncreasedRequestCount(cacheItems):
def allAreLoaded():
for c in cacheItems:
if not c.isLoaded:
return False
return True
t0 = time.time()
while not allAreLoaded() and time.time() - t0 < timeout:
BackgroundUpdateQueue.pullOne(timeout = 1.0)
graph.flush()
assert allAreLoaded(), "Timed out"
def waitCacheItemsAreLoaded(graph, cacheItems, timeout=60.0):
with IncreasedRequestCount(cacheItems):
def allAreLoaded():
for c in cacheItems:
if not c.isLoaded:
return False
return True
t0 = time.time()
while not allAreLoaded() and time.time() - t0 < timeout:
BackgroundUpdateQueue.pullOne(timeout=1.0)
graph.flush()
assert allAreLoaded(), "Timed out"
def waitForResults(graph, values, timeout = 60.0):
"""wait for 'x' to have 'result' not be zero. return the maximum number
of CPUs ever observed to be allocated to 'x'"""
t0 = time.time()
def notLoadedCount():
notLoadedCount = 0
for x in values:
if x.valueIVC is None:
notLoadedCount += 1
return notLoadedCount
with IncreasedRequestCount(values):
maxCPUs = None
while notLoadedCount() > 0 and time.time() - t0 < timeout:
time.sleep(.01)
BackgroundUpdateQueue.pullOne(timeout = .10)
graph.flush()
for x in values:
if x.totalWorkerCount is not None:
if maxCPUs is None:
maxCPUs = x.totalWorkerCount
else:
maxCPUs = max(maxCPUs,x.totalWorkerCount)
assert notLoadedCount() == 0, "Timed out: %s of %s didn't finish:\n\t%s\n" % (
notLoadedCount(),
len(values),
"\n\t".join(
[str(x) for x in values if x.valueIVC is None]
)
)
return maxCPUs
def refreshGraph(self):
BackgroundUpdateQueue.pullAll()
self.graph.flush()
def waitCacheItemIsLoaded(graph, cacheItem):
with IncreasedRequestCount(cacheItem):
while not cacheItem.isLoaded:
BackgroundUpdateQueue.pullOne()
graph.flush()
def refreshGraph(graph):
BackgroundUpdateQueue.pullAll()
graph.flush()
def refreshGraph(graph):
BackgroundUpdateQueue.pullAll()
graph.flush()
def waitCacheItemIsLoaded(graph, cacheItem):
with IncreasedRequestCount(cacheItem):
while not cacheItem.isLoaded:
BackgroundUpdateQueue.pullOne()
graph.flush()
def refreshGraph(self):
BackgroundUpdateQueue.pullAll()
self.graph.flush()