def increaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition
)
with self.lock_:
if computationId not in self.computedValuesForComputations:
self.computedValuesForComputations[computationId] = set()
self.computedValuesForComputations[computationId].add(compValue)
self.refcountsForCompIds_[computationId] += 1
if self.refcountsForCompIds_[computationId] == 1:
self.cumulusGateway.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(self.allocNewPriority_())
)
if computationId in self.finishedResultsForComputations:
result, statistics = self.finishedResultsForComputations[computationId]
BackgroundUpdateQueue.push(
self.valueUpdater(
compValue,
result,
statistics
)
)
def submitComputationOnClient(self, clientIndex, expressionText, **freeVariables):
if isinstance(expressionText, ForaNative.ImplValContainer):
computationDefinition = CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(
[CumulusNative.ComputationDefinitionTerm.Value(x, None) for x in expressionText.getTuple()]
)
)
elif isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
varNames = list(freeVariables.keys())
expr = FORA.eval("fun(" + ",".join(varNames) + ") {" + expressionText + " } ")
computationDefinition = createComputationDefinition(
FORA.extractImplValContainer(
expr
),
ForaNative.makeSymbol("Call"),
*[freeVariables[v] for v in varNames]
)
computationId = self.getClient(clientIndex).createComputation(computationDefinition)
self.getClient(clientIndex).setComputationPriority(
computationId,
CumulusNative.ComputationPriority(1)
)
return computationId
def evaluate(self, *args):
self.verifyWorkersAvailable()
try:
args = self.expandIfListOrTuple(*args)
self.submitComputation(*args)
while True:
try:
result = self.results_.get_nowait()
break
except Queue.Empty:
pass
return result
except KeyboardInterrupt:
with self.lock_:
if self.currentComputationId is not None:
self.cumulusGateway.setComputationPriority(
self.currentComputationId,
CumulusNative.ComputationPriority())
self.currentComputationId = None
while not self.results_.empty():
self.results_.get(False)
raise
def requestComputation(self, computationDefinition):
with self.lock_:
oldId, id = self.getNewComputationIdForDefinition_(
computationDefinition)
self.setComputationPriority_(id,
CumulusNative.ComputationPriority(1))
return id
def submitComputation(self, *args):
self.currentComputationId = self.cumulusGateway.getComputationIdForDefinition(
self.cumulusComputationDefinition(*args))
with self.lock_:
self.cumulusGateway.setComputationPriority(
self.currentComputationId,
CumulusNative.ComputationPriority(self.allocNewPriority_()))
def cancelComputation(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition)
with self.lock_:
if computationId in self.refcountsForCompIds_ and self.refcountsForCompIds_[
computationId] > 0:
self.refcountsForCompIds_[computationId] = 0
self.cumulusGateway.setComputationPriority(
computationId, CumulusNative.ComputationPriority())
def cancelAllComputations(self, resetCompletely=False):
with self.lock_:
for compId, refcount in self.refcountsForCompIds_.iteritems():
if refcount > 0:
self.cumulusGateway.setComputationPriority(
compId, CumulusNative.ComputationPriority())
self.refcountsForCompIds_[compId] = 0
if resetCompletely:
self.cumulusGateway.resetStateCompletely()
def setComputationPriority_(self, computationId, computationPriority):
self.cumulusClient.setComputationPriority(computationId,
computationPriority)
if computationPriority == CumulusNative.ComputationPriority():
self.cpuAssignmentDependencyGraph.markNonrootComputation(
computationId)
else:
self.cpuAssignmentDependencyGraph.markRootComputation(
computationId)
def decreaseRequestCount(self, compValue, cumulusComputationDefinition):
computationId = self.cumulusGateway.getComputationIdForDefinition(
cumulusComputationDefinition)
with self.lock_:
if computationId in self.refcountsForCompIds_ and self.refcountsForCompIds_[
computationId] > 0:
self.refcountsForCompIds_[computationId] -= 1
if self.refcountsForCompIds_[computationId] == 0:
self.cumulusGateway.setComputationPriority(
computationId, CumulusNative.ComputationPriority())
def test_dependingOnCheckpointedFinishedCachecallWorks(self):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3)
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.waitForHandshake()
baseComp = simulation.createComputation("1+2")
baseCompId = simulation.getClient(0).createComputation(baseComp)
simulation.getClient(0).setComputationPriority(
baseCompId, CumulusNative.ComputationPriority(1))
result = simulation.waitForAnyResult()
self.assertTrue(result.isResult())
simulation.getGlobalScheduler(
).triggerFullCheckpointsOnOutstandingComputations()
self.waitForAllCheckpointsToClear(simulation)
for ix in range(100):
childComp = simulation.createComputation("x + %s" % ix,
x=baseComp)
childCompId = simulation.getClient(0).createComputation(
childComp)
simulation.getClient(0).setComputationPriority(
childCompId, CumulusNative.ComputationPriority(1))
result = simulation.waitForAnyResult()
finally:
simulation.teardown()
def submitAdditionalComputation(simulationDict,
expressionText,
timeout=10.0,
wantsStats=False):
client = simulationDict["client"]
computationId = client.createComputation(
createComputationDefinition(
FORA.extractImplValContainer(
FORA.eval("fun() {" + expressionText + " } ")),
ForaNative.makeSymbol("Call")))
client.setComputationPriority(computationId,
CumulusNative.ComputationPriority(1))
return computationId
def computeUsingSeveralWorkers(expressionText,
s3Service,
count,
objectStore=None,
wantsStats=False,
timeout=10,
returnEverything=False,
memoryLimitMb=100,
blockUntilConnected=False,
keepSimulationAlive=False,
sharedStateViewFactory=None,
threadCount=2):
if keepSimulationAlive:
assert returnEverything, \
"can't keep the simulation alive and not return it. how would you shut it down?"
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
if sharedStateViewFactory is None:
sharedStateViewFactory = createInMemorySharedStateViewFactory(
callbackSchedulerToUse=callbackSchedulerToUse)
workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = (
createWorkersAndClients(count,
1,
sharedStateViewFactory,
memoryLimitMb=memoryLimitMb,
threadCount=threadCount))
client = clientsAndVdms[0][0]
clientVdm = clientsAndVdms[0][1]
loadingServices = []
for ix in range(len(workersVdmsAndEventHandlers)):
worker = workersVdmsAndEventHandlers[ix][0]
workerVdm = workersVdmsAndEventHandlers[ix][1]
s3InterfaceFactory = s3Service.withMachine(ix)
if objectStore is None:
objectStore = S3ObjectStore.S3ObjectStore(
s3InterfaceFactory,
Setup.config().userDataS3Bucket,
prefix="test/")
loadingService = PythonIoTaskService.PythonIoTaskService(
s3InterfaceFactory, objectStore, workerVdm,
worker.getExternalDatasetRequestChannel(
callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse))
loadingService.startService()
loadingServices.append(loadingService)
if blockUntilConnected:
for worker, vdm, eventHandler in workersVdmsAndEventHandlers:
blockUntilWorkerIsConnected(worker, 2.0)
if isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
computationDefinition = (createComputationDefinition(
FORA.extractImplValContainer(
FORA.eval("fun() {" + expressionText + " } ")),
ForaNative.makeSymbol("Call")))
teardownGates = []
for client, vdm in clientsAndVdms:
teardownGates.append(vdm.getVdmmTeardownGate())
for worker, vdm, eventHandler in workersVdmsAndEventHandlers:
teardownGates.append(vdm.getVdmmTeardownGate())
simulationDict = {
"result": None,
"timedOut": None,
"stats": None,
"clientsAndVdms": clientsAndVdms,
"workersVdmsAndEventHandlers": workersVdmsAndEventHandlers,
"s3Service": s3Service,
"loadingServices": loadingServices,
"sharedStateViewFactory": sharedStateViewFactory,
"client": client,
"teardownGates": teardownGates
}
try:
listener = client.createListener()
computationSubmitTime = time.time()
computationId = client.createComputation(computationDefinition)
client.setComputationPriority(computationId,
CumulusNative.ComputationPriority(1))
if returnEverything:
valAndStatsOrNone = waitForResult(listener,
computationId,
clientVdm,
timeout=timeout,
wantsStats=True)
computationReturnTime = time.time()
if valAndStatsOrNone is None:
#we timed out
val = None
stats = None
timedOut = True
else:
val, stats = valAndStatsOrNone
timedOut = False
simulationDict.update({
"result":
val,
"stats":
stats,
"timedOut":
timedOut,
"computationId":
computationId,
"listener":
listener,
"totalTimeToReturnResult":
computationReturnTime - computationSubmitTime
})
return simulationDict
else:
return waitForResult(listener,
computationId,
clientVdm,
timeout=timeout,
wantsStats=wantsStats)
finally:
if not keepSimulationAlive:
teardownSimulation(simulationDict)
def deprioritizeComputation(self, computationId):
self.setComputationPriority(computationId,
CumulusNative.ComputationPriority())
