def increaseRequestCount(self, compValue, cumulusComputationDefinition): computationId = self.cumulusGateway.getComputationIdForDefinition( cumulusComputationDefinition ) with self.lock_: if computationId not in self.computedValuesForComputations: self.computedValuesForComputations[computationId] = set() self.computedValuesForComputations[computationId].add(compValue) self.refcountsForCompIds_[computationId] += 1 if self.refcountsForCompIds_[computationId] == 1: self.cumulusGateway.setComputationPriority( computationId, CumulusNative.ComputationPriority(self.allocNewPriority_()) ) if computationId in self.finishedResultsForComputations: result, statistics = self.finishedResultsForComputations[computationId] BackgroundUpdateQueue.push( self.valueUpdater( compValue, result, statistics ) )
def submitComputationOnClient(self, clientIndex, expressionText, **freeVariables): if isinstance(expressionText, ForaNative.ImplValContainer): computationDefinition = CumulusNative.ComputationDefinition.Root( CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm( [CumulusNative.ComputationDefinitionTerm.Value(x, None) for x in expressionText.getTuple()] ) ) elif isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: varNames = list(freeVariables.keys()) expr = FORA.eval("fun(" + ",".join(varNames) + ") {" + expressionText + " } ") computationDefinition = createComputationDefinition( FORA.extractImplValContainer( expr ), ForaNative.makeSymbol("Call"), *[freeVariables[v] for v in varNames] ) computationId = self.getClient(clientIndex).createComputation(computationDefinition) self.getClient(clientIndex).setComputationPriority( computationId, CumulusNative.ComputationPriority(1) ) return computationId
def evaluate(self, *args): self.verifyWorkersAvailable() try: args = self.expandIfListOrTuple(*args) self.submitComputation(*args) while True: try: result = self.results_.get_nowait() break except Queue.Empty: pass return result except KeyboardInterrupt: with self.lock_: if self.currentComputationId is not None: self.cumulusGateway.setComputationPriority( self.currentComputationId, CumulusNative.ComputationPriority()) self.currentComputationId = None while not self.results_.empty(): self.results_.get(False) raise
def requestComputation(self, computationDefinition): with self.lock_: oldId, id = self.getNewComputationIdForDefinition_( computationDefinition) self.setComputationPriority_(id, CumulusNative.ComputationPriority(1)) return id
def submitComputation(self, *args): self.currentComputationId = self.cumulusGateway.getComputationIdForDefinition( self.cumulusComputationDefinition(*args)) with self.lock_: self.cumulusGateway.setComputationPriority( self.currentComputationId, CumulusNative.ComputationPriority(self.allocNewPriority_()))
def cancelComputation(self, compValue, cumulusComputationDefinition): computationId = self.cumulusGateway.getComputationIdForDefinition( cumulusComputationDefinition) with self.lock_: if computationId in self.refcountsForCompIds_ and self.refcountsForCompIds_[ computationId] > 0: self.refcountsForCompIds_[computationId] = 0 self.cumulusGateway.setComputationPriority( computationId, CumulusNative.ComputationPriority())
def cancelAllComputations(self, resetCompletely=False): with self.lock_: for compId, refcount in self.refcountsForCompIds_.iteritems(): if refcount > 0: self.cumulusGateway.setComputationPriority( compId, CumulusNative.ComputationPriority()) self.refcountsForCompIds_[compId] = 0 if resetCompletely: self.cumulusGateway.resetStateCompletely()
def setComputationPriority_(self, computationId, computationPriority): self.cumulusClient.setComputationPriority(computationId, computationPriority) if computationPriority == CumulusNative.ComputationPriority(): self.cpuAssignmentDependencyGraph.markNonrootComputation( computationId) else: self.cpuAssignmentDependencyGraph.markRootComputation( computationId)
def decreaseRequestCount(self, compValue, cumulusComputationDefinition): computationId = self.cumulusGateway.getComputationIdForDefinition( cumulusComputationDefinition) with self.lock_: if computationId in self.refcountsForCompIds_ and self.refcountsForCompIds_[ computationId] > 0: self.refcountsForCompIds_[computationId] -= 1 if self.refcountsForCompIds_[computationId] == 0: self.cumulusGateway.setComputationPriority( computationId, CumulusNative.ComputationPriority())
def test_dependingOnCheckpointedFinishedCachecallWorks(self): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3) try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.waitForHandshake() baseComp = simulation.createComputation("1+2") baseCompId = simulation.getClient(0).createComputation(baseComp) simulation.getClient(0).setComputationPriority( baseCompId, CumulusNative.ComputationPriority(1)) result = simulation.waitForAnyResult() self.assertTrue(result.isResult()) simulation.getGlobalScheduler( ).triggerFullCheckpointsOnOutstandingComputations() self.waitForAllCheckpointsToClear(simulation) for ix in range(100): childComp = simulation.createComputation("x + %s" % ix, x=baseComp) childCompId = simulation.getClient(0).createComputation( childComp) simulation.getClient(0).setComputationPriority( childCompId, CumulusNative.ComputationPriority(1)) result = simulation.waitForAnyResult() finally: simulation.teardown()
def submitAdditionalComputation(simulationDict, expressionText, timeout=10.0, wantsStats=False): client = simulationDict["client"] computationId = client.createComputation( createComputationDefinition( FORA.extractImplValContainer( FORA.eval("fun() {" + expressionText + " } ")), ForaNative.makeSymbol("Call"))) client.setComputationPriority(computationId, CumulusNative.ComputationPriority(1)) return computationId
def computeUsingSeveralWorkers(expressionText, s3Service, count, objectStore=None, wantsStats=False, timeout=10, returnEverything=False, memoryLimitMb=100, blockUntilConnected=False, keepSimulationAlive=False, sharedStateViewFactory=None, threadCount=2): if keepSimulationAlive: assert returnEverything, \ "can't keep the simulation alive and not return it. how would you shut it down?" callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if sharedStateViewFactory is None: sharedStateViewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse=callbackSchedulerToUse) workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = ( createWorkersAndClients(count, 1, sharedStateViewFactory, memoryLimitMb=memoryLimitMb, threadCount=threadCount)) client = clientsAndVdms[0][0] clientVdm = clientsAndVdms[0][1] loadingServices = [] for ix in range(len(workersVdmsAndEventHandlers)): worker = workersVdmsAndEventHandlers[ix][0] workerVdm = workersVdmsAndEventHandlers[ix][1] s3InterfaceFactory = s3Service.withMachine(ix) if objectStore is None: objectStore = S3ObjectStore.S3ObjectStore( s3InterfaceFactory, Setup.config().userDataS3Bucket, prefix="test/") loadingService = PythonIoTaskService.PythonIoTaskService( s3InterfaceFactory, objectStore, workerVdm, worker.getExternalDatasetRequestChannel( callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse)) loadingService.startService() loadingServices.append(loadingService) if blockUntilConnected: for worker, vdm, eventHandler in workersVdmsAndEventHandlers: blockUntilWorkerIsConnected(worker, 2.0) if isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: computationDefinition = (createComputationDefinition( FORA.extractImplValContainer( FORA.eval("fun() {" + expressionText + " } ")), ForaNative.makeSymbol("Call"))) teardownGates = [] for client, vdm in clientsAndVdms: teardownGates.append(vdm.getVdmmTeardownGate()) for worker, vdm, eventHandler in workersVdmsAndEventHandlers: teardownGates.append(vdm.getVdmmTeardownGate()) simulationDict = { "result": None, "timedOut": None, "stats": None, "clientsAndVdms": clientsAndVdms, "workersVdmsAndEventHandlers": workersVdmsAndEventHandlers, "s3Service": s3Service, "loadingServices": loadingServices, "sharedStateViewFactory": sharedStateViewFactory, "client": client, "teardownGates": teardownGates } try: listener = client.createListener() computationSubmitTime = time.time() computationId = client.createComputation(computationDefinition) client.setComputationPriority(computationId, CumulusNative.ComputationPriority(1)) if returnEverything: valAndStatsOrNone = waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=True) computationReturnTime = time.time() if valAndStatsOrNone is None: #we timed out val = None stats = None timedOut = True else: val, stats = valAndStatsOrNone timedOut = False simulationDict.update({ "result": val, "stats": stats, "timedOut": timedOut, "computationId": computationId, "listener": listener, "totalTimeToReturnResult": computationReturnTime - computationSubmitTime }) return simulationDict else: return waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=wantsStats) finally: if not keepSimulationAlive: teardownSimulation(simulationDict)
def deprioritizeComputation(self, computationId): self.setComputationPriority(computationId, CumulusNative.ComputationPriority())