def resetStateEntirely(self):
"""Cancel all computations and clear the compute cache."""
self.cancelAllComputations(True)
with self.lock_:
computations = list(self.refcountsForCompIds_.keys())
for computationId in computations:
if computationId in self.finishedResultsForComputations:
del self.finishedResultsForComputations[computationId]
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
None,
None
)
)
for vecId in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[vecId]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
self.vectorDataIDRequestCount_ = {}
self.vectorDataIDToVectorSlices_ = {}
def increaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition
)
with self.lock_:
if computationId not in self.computedValuesForComputations:
self.computedValuesForComputations[computationId] = set()
self.computedValuesForComputations[computationId].add(compValue)
self.refcountsForCompIds_[computationId] += 1
if self.refcountsForCompIds_[computationId] == 1:
self.cumulusGateway.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(self.allocNewPriority_())
)
if computationId in self.finishedResultsForComputations:
result, statistics = self.finishedResultsForComputations[computationId]
BackgroundUpdateQueue.push(
self.valueUpdater(
compValue,
result,
statistics
)
)
def increaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition
)
with self.lock_:
if computationId not in self.computedValuesForComputations:
self.computedValuesForComputations[computationId] = set()
self.computedValuesForComputations[computationId].add(compValue)
self.refcountsForCompIds_[computationId] += 1
if self.refcountsForCompIds_[computationId] == 1:
self.cumulusGateway.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(self.allocNewPriority_())
)
if computationId in self.finishedResultsForComputations:
result, statistics = self.finishedResultsForComputations[computationId]
BackgroundUpdateQueue.push(
self.valueUpdater(
compValue,
result,
statistics
)
)
def resetStateEntirely(self):
"""Cancel all computations and clear the compute cache."""
self.cancelAllComputations(True)
with self.lock_:
computations = list(self.refcountsForCompIds_.keys())
for computationId in computations:
if computationId in self.finishedResultsForComputations:
del self.finishedResultsForComputations[computationId]
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
None,
None
)
)
for vecId in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[vecId]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
self.vectorDataIDRequestCount_ = {}
self.vectorDataIDToVectorSlices_ = {}
def onComputationResult(self, computationId, result, statistics):
with self.lock_:
self.finishedResultsForComputations[computationId] = (result,
statistics)
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[
computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(compVal, result, statistics))
def onCPUCountChanged(self, computationSystemwideCpuAssignment):
with self.lock_:
computationId = computationSystemwideCpuAssignment.computation
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[
computationId]:
BackgroundUpdateQueue.push(
self.cpuCountSetter_(
compVal, computationSystemwideCpuAssignment))
def onCacheLoad(self, vectorDataID):
with self.lock_:
if vectorDataID in self.vectorDataIDToVectorSlices_:
cgLocations = self.vectorDataIDToVectorSlices_[vectorDataID]
else:
cgLocations = set()
self.collectOffloadedVectors_()
for cgLocation in cgLocations:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, True))
def onNewGlobalUserFacingLogMessage(self, newMsg):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().pushNewGlobalUserFacingLogMessage(newMsg)
BackgroundUpdateQueue.push(updater)
def onCacheLoad(self, vectorDataID):
with self.lock_:
if vectorDataID in self.vectorDataIDToVectorSlices_:
cgLocations = self.vectorDataIDToVectorSlices_[vectorDataID]
else:
cgLocations = set()
self.collectOffloadedVectors_()
for cgLocation in cgLocations:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, self.computeVectorSliceIsLoaded_(cgLocation)))
def onNewGlobalUserFacingLogMessage(self, newMsg):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().pushNewGlobalUserFacingLogMessage(newMsg)
BackgroundUpdateQueue.push(updater)
def onCPUCountChanged(self, computationSystemwideCpuAssignment):
with self.lock_:
computationId = computationSystemwideCpuAssignment.computation
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.cpuCountSetter_(
compVal,
computationSystemwideCpuAssignment
)
)
def onComputationResult(self, computationId, result, statistics):
with self.lock_:
self.finishedResultsForComputations[computationId] = (result, statistics)
if computationId in self.computedValuesForComputations:
for compVal in self.computedValuesForComputations[computationId]:
BackgroundUpdateQueue.push(
self.valueUpdater(
compVal,
result,
statistics
)
)
def onJsonViewOfSystemChanged(self, json):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().viewOfSystem_ = json.toSimple()
PersistentCacheIndex.PersistentCacheIndex().update()
BackgroundUpdateQueue.push(updater)
def onJsonViewOfSystemChanged(self, json):
def updater():
global ViewOfEntireCumulusSystem
global PersistentCacheIndex
if ViewOfEntireCumulusSystem is None:
import ufora.BackendGateway.ComputedValue.ViewOfEntireCumulusSystem as ViewOfEntireCumulusSystem
import ufora.BackendGateway.ComputedValue.PersistentCacheIndex as PersistentCacheIndex
ViewOfEntireCumulusSystem.ViewOfEntireCumulusSystem().viewOfSystem_ = json.toSimple()
PersistentCacheIndex.PersistentCacheIndex().update()
BackgroundUpdateQueue.push(updater)
def collectOffloadedVectors_(self):
offloaded = self.ramCacheOffloadRecorder.extractDropped()
if offloaded:
logging.info("ComputedValue RamCache dropped %s", offloaded)
for offloadedVecDataID in offloaded:
if offloadedVecDataID in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[offloadedVecDataID]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
if offloaded:
#check if there's anything we need to load
self.sendReloadRequests()
def onExternalIoTaskCompleted(self, msg):
taskGuid = msg.taskId.guid
with self.lock_:
if taskGuid not in self.externalIoTaskCallbacks_:
logging.warn("TaskId %s was not found in the task guid list", taskGuid)
return
callback = self.externalIoTaskCallbacks_[taskGuid]
del self.externalIoTaskCallbacks_[taskGuid]
def executor():
callback(msg.result)
BackgroundUpdateQueue.push(executor)
def onExternalIoTaskCompleted(self, msg):
taskGuid = msg.taskId.guid
with self.lock_:
if taskGuid not in self.externalIoTaskCallbacks_:
logging.warn("TaskId %s was not found in the task guid list", taskGuid)
return
callback = self.externalIoTaskCallbacks_[taskGuid]
del self.externalIoTaskCallbacks_[taskGuid]
def executor():
callback(msg.result)
BackgroundUpdateQueue.push(executor)
def collectOffloadedVectors_(self):
offloaded = self.ramCacheOffloadRecorder.extractDropped()
if offloaded:
logging.info("ComputedValue RamCache dropped %s", offloaded)
for offloadedVecDataID in offloaded:
if offloadedVecDataID in self.vectorDataIDToVectorSlices_:
for cgLocation in self.vectorDataIDToVectorSlices_[offloadedVecDataID]:
BackgroundUpdateQueue.push(self.createSetIsLoadedFun(cgLocation, False))
if offloaded:
#check if there's anything we need to load
self.sendReloadRequests()