def test_verifyThatExtractingPausedComputationsDoesntDuplicateLargeStrings( self): text = """fun() { let s = ' ' while (size(s) < 1000000) s = s + s let f = fun(x) { if (x > 0) return f(x-1) + s[x]; `TriggerInterruptForTesting() } f(20) }""" vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) computation = context.extractPausedComputation() context2 = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context2.resumePausedComputation(computation) self.assertTrue(context2.totalBytesUsed < 2 * context.totalBytesUsed)
def test_resumingAfterCopyDataOutOfPages(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) context.interruptAfterCycleCount(100000) text = """ fun() { let v = Vector.range(1000).paged; let ix1 = 0 let res = 0 while (true) { res = res + v[ix1] ix1 = (ix1 + 1) % size(v) } res }""" context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) paused1 = context.extractPausedComputation() while not context.isVectorLoad(): context.copyValuesOutOfVectorPages() vdm.unloadAllPossible() context.interruptAfterCycleCount(100000) context.resume() paused2 = context.extractPausedComputation() self.assertTrue(len(paused1.frames) == len(paused2.frames))
def _test_vectorElementJOV_VectorElementJOVFromLiveValue_compatibility(self): vdmSimple = None # Required to pass lint check - represents a bug jovs = self.sampleJOVs() numpy.random.seed(42) context = ExecutionContext.ExecutionContext( 32 * 1024, False, False, vdmSimple ) randomJOVGenerator = FORANative.RandomJOVGenerator(0, context).symbolStrings(self.symbol_strings) for jov in jovs: for i in range(100): random_value = randomJOVGenerator.randomValue(jov) if random_value != None: vectorElementJOVFromLiveValue = \ FORANative.VectorElementJOVFromLiveValue(random_value) vectorElementJOVFromJOV = \ FORANative.JOVFromLiveValue(random_value).vectorElementJOV() if vectorElementJOVFromLiveValue != vectorElementJOVFromJOV: print "rand_val = %s" \ % random_value print "vectorElementJOVFromLiveValue = %s" \ % vectorElementJOVFromLiveValue print "vectorElementJOVFromJOV = %s" \ % vectorElementJOVFromJOV print "JOVFromLiveValue = %s" \ % FORANative.JOVFromLiveValue(random_value) self.assertEqual(vectorElementJOVFromLiveValue, vectorElementJOVFromJOV)
def test_teardown_simple(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) evaluate(context, FORA.extractImplValContainer(FORA.eval("fun(){nothing}")), FORANative.symbol_Call) context.getFinishedResult() toEval = FORA.extractImplValContainer( FORA.eval("""fun() { let f = fun() { }; let v = [1, [3]]; cached(f()) }""")) evaluate(context, toEval, FORANative.symbol_Call) while not context.isCacheRequest(): context.compute() context.teardown(True)
def test_extractPausedComputationDuringVectorLoad(self): self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context.evaluate( FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")), FORANative.ImplValContainer(FORANative.makeSymbol("Call"))) pagedVec = context.getFinishedResult().asResult.result context.placeInEvaluationState( FORANative.ImplValContainer( (pagedVec, FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")), FORANative.ImplValContainer(0)))) vdm.unloadAllPossible() context.resume() self.assertTrue(context.isVectorLoad()) computation = context.extractPausedComputation() self.assertEqual(len(computation.frames), 1)
def createWorker(machineId, viewFactory, callbackSchedulerToUse=None, threadCount=2, memoryLimitMb=100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024))) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex(viewFactory.createView(), callbackSchedulerToUse)) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return (CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None (), ExecutionContext.createContextConfiguration(), ""), vdm, cache, eventHandler), vdm, eventHandler)
def test_serialize_while_holding_interior_vector(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False, allowInternalSplitting=False) evaluate( context, FORA.extractImplValContainer( FORA.eval(""" fun() { let v = [[1].paged].paged; let v2 = v[0] `TriggerInterruptForTesting() 1+2+3+v+v2 }""")), FORANative.symbol_Call) self.assertTrue(context.isInterrupted()) serialized = context.serialize() context = None
def test_refcountsInCompiledCode(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=True, blockUntilTracesAreCompiled=True) text = """fun(){ let f = fun(v, depth) { if (depth > 100) //this will trigger an interrupt since the data cannot exist in the VDM datasets.s3('','') else f(v, depth+1) } f([1,2,3,4,5], 0) }""" context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) stacktraceText = context.extractCurrentTextStacktrace() self.assertTrue(stacktraceText.count("Vector") < 10)
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False, allowInternalSplitting=False) context.configuration.agressivelyValidateRefcountsAndPageReachability = True context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False context.placeInEvaluationState( FORANative.ImplValContainer( (FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call))) context.interruptAfterCycleCount(cycleCount) context.compute() if expectsToHaveCopies: self.assertTrue(context.copyValuesOutOfVectorPages()) self.assertFalse(context.copyValuesOutOfVectorPages()) else: self.assertFalse(context.copyValuesOutOfVectorPages())
def test_resumePausedComputationWithResult(self): self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False, allowInternalSplitting=False) text = """ let f = fun(v, ix) { if (ix > 0) { let (v2,res) = f(v,ix-1); return (v2, res + v2[0]) } `TriggerInterruptForTesting() return (v, 0) }; f([1], 10) """ evaluate( context, FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")), FORANative.ImplValContainer(FORANative.makeSymbol("Call"))) assert context.isInterrupted() pausedComp = context.extractPausedComputation() framesToUse = pausedComp.asThread.computation.frames[0:5] pausedComp2 = FORANative.PausedComputationTree( FORANative.PausedComputation( framesToUse, FORA.extractImplValContainer( FORA.eval("([2], 0)", keepAsForaValue=True)), False)) context.resumePausedComputation(pausedComp2) context.copyValuesOutOfVectorPages() context.pageLargeVectorHandles(0) context.resetInterruptState() context.compute() self.assertTrue(context.isFinished()) result = context.getFinishedResult() self.assertTrue(result.asResult.result[1].pyval == 6)
def grabContext(self): """get an execution context""" with self.lock_: if not self.contexts_: context = ExecutionContext.ExecutionContext( dataManager=self.vdm_) else: context = self.contexts_.pop() return context
def test_serialize_during_vector_load(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) context.evaluate( FORA.extractImplValContainer( FORA.eval("fun(){ datasets.s3('a','b')[0] }")), FORANative.symbol_Call) self.assertTrue(context.isVectorLoad()) serialized = context.serialize() context2 = ExecutionContext.ExecutionContext(dataManager=vdm) context2.deserialize(serialized) self.assertTrue(context2.isVectorLoad())
def createWorker_(machineId, viewFactory, callbackSchedulerToUse, threadCount, memoryLimitMb, cacheFunction, pageSizeOverride, disableEventHandler): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager( callbackSchedulerToUse, pageSizeOverride if pageSizeOverride is not None else 1 * 1024 * 1024 if memoryLimitMb < 1000 else 5 * 1024 * 1024 if memoryLimitMb < 5000 else 50 * 1024 * 1024 ) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = cacheFunction() if disableEventHandler: eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler() else: eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def test_covers(self): vdm = FORANative.VectorDataManager( callbackScheduler, Setup.config().maxPageSizeInBytes ) context = ExecutionContext.ExecutionContext(dataManager = vdm) randomJOVGenerator = FORANative.RandomJOVGenerator(0, context) self.treeElementCoveringTest(coverageTree, [], [], randomJOVGenerator)
def callAndGetResult(funImplVal): vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024) context = ExecutionContext.ExecutionContext(dataManager=vdm, allowInterpreterTracing=False) context.evaluate(funImplVal, ForaNative.symbol_Call) finishedResult = context.getFinishedResult() return finishedResult
def runSimpleEvaluation(self, inputType, outputType): mainRuntime = Runtime.getMainRuntime() foraCompiler = mainRuntime.getTypedForaCompiler() while foraCompiler.anyCompilingOrPending(): time.sleep(.01) aParticularStringValue = ForaNative.ImplValContainer(aBigString) callable = self.generateSimpleCallable(inputType, outputType) jumpTarget = foraCompiler.compile(callable) import gc gc.collect() for passIndex in range(PASS_COUNT): #type values are memoized, so we can't assume that the value has a refcount # of exactly one totalStringCount = ForaNative.totalStringCount() totalImplvalCount = ForaNative.totalImplvalCount() anExecutionContext = ExecutionContext.ExecutionContext( dataManager=VectorDataManager.constructVDM( self.callbackScheduler)) anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue) self.assertTrue(anExecutionContext.isFinished()) res = anExecutionContext.getFinishedResult() self.assertTrue(not res.isException()) self.assertEqual(res.asResult.result, aParticularStringValue) anExecutionContext.teardown() res = None #verify final refcounts self.assertEqual( aParticularStringValue.getStringObjectRefcount(), 1, "refcounts weren't maintained in %s->%s. %s != 1" % (inputType, outputType, aParticularStringValue.getStringObjectRefcount())) self.assertEqual((totalStringCount, totalImplvalCount), (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()), "refcounts weren't maintained in " + str(inputType) + "->" + str(outputType))
def test_extractPausedComputation(self): text = """fun() { let x = 0; while (x < 100000) x = x + 1 x }""" self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context.interruptAfterCycleCount(1010) context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) computation = context.extractPausedComputation() self.assertEqual(len(computation.frames), 2) self.assertEqual(computation.frames[1].values[0].pyval, 335) context2 = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context2.resumePausedComputation(computation) context2.resume() self.assertEqual(context2.getFinishedResult().asResult.result.pyval, 100000) context.teardown() context2.teardown()
def test_teardown_simple_2(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) context.placeInEvaluationState( FORANative.ImplValContainer((FORA.extractImplValContainer( FORA.eval( "fun(){ let f = fun() { throw 1 }; try { f() } catch(...) { throw 2 } }" )), FORANative.symbol_Call))) context.compute() self.assertTrue(context.getFinishedResult().isException())
def finishPausedComputation(pausedComputation): vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024) context2 = ExecutionContext.ExecutionContext(dataManager=vdm, allowInterpreterTracing=False) context2.resumePausedComputation(pausedComputation) context2.resume() if (not context2.isFinished()): raise CouldntFinishException(pausedComputation) finishedResult = context2.getFinishedResult() return finishedResult
def extractPagedUnloadedVector(self, vdm, count): context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) evaluate( context, FORA.extractImplValContainer( FORA.eval("fun() { Vector.range(%s).paged }" % count)), FORANative.ImplValContainer(FORANative.makeSymbol("Call"))) pagedVec = context.getFinishedResult().asResult.result vdm.unloadAllPossible() return pagedVec
def callAndExtractPausedCompuationAfterSteps(funToCall, steps): vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024) context = ExecutionContext.ExecutionContext(dataManager=vdm, allowInterpreterTracing=False) context.interruptAfterCycleCount(steps) context.evaluate(funToCall, ForaNative.symbol_Call) if (not context.isInterrupted()): raise NotInterruptedException(context) computation = context.extractPausedComputation() context.teardown() return computation
def test_extractStacktrace(self): text = """fun() { let x = 0; let v = [1,2,3] while (x < 100000) { x = x + 1 } (x,v) }""" self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context.interruptAfterCycleCount(1000) context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) trace = context.extractStacktrace(True) self.assertTrue(len(trace) == 1) stacktrace = trace[0][0] frameData = trace[0][1] codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0]) self.assertTrue(codeLocation is not None) self.assertTrue(frameData.wasCompiled is False) self.assertTrue(len(frameData.inScopeValues) == 2) self.assertTrue([str(x) for x in frameData.inScopeValues.keys] == ['x', 'v']) values = [ frameData.inScopeValues[x] for x in frameData.inScopeValues.keys ] self.assertTrue(values[0].isInt64()) self.assertTrue(values[1].isVector()) context.teardown()
def test_interrupt_works(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) triggerAfter(context.interrupt, .03) t0 = time.time() context.evaluate(self.loopFun, FORANative.symbol_Call) #make sure we actually looped! self.assertTrue(time.time() - t0 > .02) self.assertFalse(context.isEmpty()) self.assertFalse(context.isCacheRequest()) self.assertFalse(context.isVectorLoad()) self.assertFalse(context.isFinished()) self.assertTrue(context.isInterrupted())
def runSimpleEvaluation(self, callable, arguments, validator): mainRuntime = Runtime.getMainRuntime() foraCompiler = mainRuntime.getTypedForaCompiler() jumpTarget = foraCompiler.compile(callable) while foraCompiler.anyCompilingOrPending(): time.sleep(.01) gc.collect() for passIndex in range(PASS_COUNT): totalStringCount = ForaNative.totalStringCount() totalImplvalCount = ForaNative.totalImplvalCount() anExecutionContext = ExecutionContext.ExecutionContext( dataManager = VectorDataManager.constructVDM(self.callbackScheduler) ) anExecutionContext.evaluateFunctionPointer(jumpTarget, *arguments) self.assertTrue(anExecutionContext.isFinished()) res = anExecutionContext.getFinishedResult() self.assertTrue(validator(res.asResult.result), "invalid result in " + str(callable) + " with " + str(arguments) + ". got " + str(res) ) res = None anExecutionContext.teardown() curRefs = (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()) self.assertEqual( (totalStringCount, totalImplvalCount), curRefs, "refcounts weren't maintained in " + str(callable) + " with " + str(arguments) + ". %s != %s" % ( (totalStringCount, totalImplvalCount), curRefs ) )
def pageLargeVectorHandlesTest(self, text, cycleCount, expectsToHavePages): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False) context.configuration.agressivelyValidateRefcountsAndPageReachability = True context.interruptAfterCycleCount(cycleCount) context.evaluate(FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call) if expectsToHavePages: self.assertTrue(context.pageLargeVectorHandles(0)) self.assertFalse(context.pageLargeVectorHandles(0)) else: self.assertFalse(context.pageLargeVectorHandles(0)) return context
def test_axiom_consistency(self): vdm = FORANative.VectorDataManager( self.callbackScheduler, Setup.config().maxPageSizeInBytes ) self.context = ExecutionContext.ExecutionContext(dataManager = vdm) self.randomJOVGenerator = FORANative.RandomJOVGenerator( self.seed, self.context).symbolStrings(self.symbol_strings).setMaxStringLength(50) for jovt in self.axiom_signatures_to_test: axiom = self.axioms.getAxiomByJOVT(self.typed_fora_compiler, jovt) self.assertIsNotNone(axiom, "unable to find an axiom for the signature %s" %jovt) if axiom.isExpansion(): assert False, "We can only test native expansions." elif axiom.isNativeCall(): self.check_native_axiom_consistency(axiom) else: assert False, "Unexpected axiom type in signature %s" % jovt
def _test_jov_random_value(self): vdmSimple = None # Required to pass lint check - represents a bug context = ExecutionContext.ExecutionContext( 32 * 1024, False, False, vdmSimple ) jovs = self.sampleJOVs() randomJOVGenerator = FORANative.RandomJOVGenerator( 0, context).symbolStrings(self.symbol_strings) for jov in jovs: for i in range(100): random_value = randomJOVGenerator.randomValue(jov) if random_value != None: self.assertTrue(jov.covers(random_value), "jov %s doesn't cover %s, but generated it as a random value!" % (jov, random_value) )
def test_teardown_during_vector_load(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) context.evaluate( FORA.extractImplValContainer( FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")), FORANative.symbol_Call) vdm.unloadAllPossible() pagedVecAccessFun = context.getFinishedResult().asResult.result context.teardown() context.evaluate(pagedVecAccessFun, FORANative.symbol_Call) self.assertFalse(context.isInterrupted()) self.assertTrue(context.isVectorLoad()) context.teardown()
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def createCliqueFinderContext(self, text, *args): maxPageSizeInBytes = 32 * 1024 vdm = FORANative.VectorDataManager(callbackScheduler, maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) argumentImplvals = [] for a in args: context.evaluate( FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")), Symbol_Call) argumentImplvals.append( context.getFinishedResult().asResult.result) actualFunction = FORA.extractImplValContainer(FORA.eval(text)) context.placeInEvaluationState( FORANative.ImplValContainer((actualFunction, Symbol_Call) + tuple(argumentImplvals))) return context, argumentImplvals, vdm
def simpleEval(*args, **kwargs): return ExecutionContext.simpleEval(callbackScheduler, *args, **kwargs)
def __init__(self, ownAddress, channelListener, channelFactory, eventHandler, callbackScheduler, diagnosticsDir, config, viewFactory): Stoppable.Stoppable.__init__(self) #acquire a machineId randomly, using uuid self.machineId = CumulusNative.MachineId( Hash.Hash.sha1(str(uuid.uuid4())) ) self.ownAddress = ownAddress self.callbackScheduler = callbackScheduler self.viewFactory = viewFactory self.threadsStarted_ = False self.connectedMachines = set() self.connectingMachines = set() # machines we are in the process of connecting to self.droppedMachineIds = set() self.lock = threading.RLock() self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024 self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024 self.cumulusThreadCountOverride = config.cumulusServiceThreadCount self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc self.reconnectPersistentCacheIndexViewThreads = [] if config.cumulusDiskCacheStorageSubdirectory is not None: self.cumulusDiskCacheWantsDeletionOnTeardown = True self.cumulusDiskCacheStorageDir = os.path.join( config.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageSubdirectory ) else: self.cumulusDiskCacheWantsDeletionOnTeardown = False self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir logging.info( "Creating a CumulusService with ram cache of %s / %s MB and %s threads", self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0, self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0, self.cumulusThreadCountOverride ) self._stopEvent = threading.Event() self._channelListener = channelListener assert len(self._channelListener.ports) == 2 self._channelFactory = channelFactory Runtime.initialize() ModuleImporter.initialize() self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines( self.viewFactory ) self.cumulusChannelFactoryThread = ManagedThread.ManagedThread( target=self._channelListener.start ) self.vdm = VectorDataManager.constructVDM( callbackScheduler, self.cumulusVectorRamCacheSizeOverride, self.cumulusMaxRamCacheSizeOverride ) if self.cumulusTrackTcMalloc: logging.info( "CumulusService enabling track-tc-malloc memory with a max cache of %s MB", self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0 ) self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory() self.persistentCacheIndex = CumulusNative.PersistentCacheIndex( viewFactory.createView(retrySeconds=10.0, numRetries=10), callbackScheduler ) self.vdm.setPersistentCacheIndex(self.persistentCacheIndex) self.deleteCumulusDiskCacheIfNecessary() self.offlineCache = CumulusNative.DiskOfflineCache( callbackScheduler, self.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageMB * 1024 * 1024, config.cumulusDiskCacheStorageFileCount ) checkpointInterval = config.cumulusCheckpointIntervalSeconds if checkpointInterval == 0: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None() else: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic( checkpointInterval, 1024 * 1024 ) self.cumulusWorker = self.constructCumlusWorker( callbackScheduler, CumulusNative.CumulusWorkerConfiguration( self.machineId, self.cumulusThreadCountOverride, checkpointPolicy, ExecutionContext.createContextConfiguration(), diagnosticsDir or "" ), self.vdm, self.offlineCache, eventHandler ) #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel( #callbackScheduler #) #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService( #settings.s3InterfaceFactory, #settings.objectStore, #self.vdm, #externalDatasetChannel.makeQueuelike(callbackScheduler) #) self.cumulusWorker.startComputations()