def test_persistentCacheUnderLoad(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) t0 = time.time() #add 100k pages, which is enough for about 5 TB of data for index in range(100000): if index % 1000 == 0 and index > 0: print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000" cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1("")) print "took ", time.time() - t0, " to add 100k." t1 = time.time() bytes0 = TCMallocNative.getBytesUsed() cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) while cppView2.totalBytesInCache() < 100000: time.sleep(.1) print cppView2.totalBytesInCache() print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
def createWorker(machineId, viewFactory, callbackSchedulerToUse=None, threadCount=2, memoryLimitMb=100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024))) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex(viewFactory.createView(), callbackSchedulerToUse)) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return (CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None (), ExecutionContext.createContextConfiguration(), ""), vdm, cache, eventHandler), vdm, eventHandler)
def circularPageReferenceTest(self, shouldBeInvalid): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) computationId = CumulusNative.ComputationId.Root(sha1("computation")) checkpointRequest = CumulusNative.CheckpointRequest( 0.0, True, computationId) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage( sha1("page1"), (HashSet() + sha1("bigvec1")) if shouldBeInvalid else HashSet(), 1, sha1("")) cppView1.addCheckpointFile(checkpointRequest, sha1("file"), HashSet() + sha1("bigvec1"), 2, sha1("")) cppView1.addCheckpoint(checkpointRequest, HashSet() + sha1("file"), 2, sha1(""), True, 1.0, HashSet()) self.assertTrue( len(cppView1.computeInvalidObjects()) == (4 if shouldBeInvalid else 0), "%s != %s" % (len(cppView1.computeInvalidObjects()), (4 if shouldBeInvalid else 0)))
def test_writing_while_disconnected(self): currentView = [self.sharedState.newView()] cppView1 = CumulusNative.PersistentCacheIndex(currentView[0], callbackScheduler) def writeInLoop(): for ix in range(100): time.sleep(0.01) cppView1.addPage(sha1("page" + str(ix)), HashSet(), ix, sha1("")) thread1 = threading.Thread(target=writeInLoop) thread1.start() def disconnectAndReconnectInLoop(): ix = 0 while thread1.isAlive(): ix += 1 time.sleep(0.004) currentView[0].disconnect() currentView[0] = self.sharedState.newView() cppView1.resetView(currentView[0]) thread2 = threading.Thread(target=disconnectAndReconnectInLoop) thread2.start() thread1.join() thread2.join() self.assertTrue(cppView1.timesViewReconnected() > 10) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) time.sleep(2.0) count1 = 0 count2 = 0 for ix in range(100): if cppView1.pageExists(sha1("page" + str(ix))): count1 += 1 if cppView2.pageExists(sha1("page" + str(ix))): count2 += 1 self.assertTrue(count1 == 100 and count2 == 100, (count1, count2))
def createWorker_(machineId, viewFactory, callbackSchedulerToUse, threadCount, memoryLimitMb, cacheFunction, pageSizeOverride, disableEventHandler): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager( callbackSchedulerToUse, pageSizeOverride if pageSizeOverride is not None else 1 * 1024 * 1024 if memoryLimitMb < 1000 else 5 * 1024 * 1024 if memoryLimitMb < 5000 else 50 * 1024 * 1024 ) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = cacheFunction() if disableEventHandler: eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler() else: eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def __init__(self, callbackScheduler, vdm, viewFactory): Stoppable.Stoppable.__init__(self) self.lock_ = threading.Lock() self.callbackScheduler = callbackScheduler self.definitionToIdMap_ = {} self.idToDefinitionMap_ = {} self.vdm = vdm self.onJsonViewOfSystemChanged = None self.persistentCacheIndex = CumulusNative.PersistentCacheIndex( viewFactory.createView(retrySeconds=10.0, numRetries=10), callbackScheduler) self.vdm.setPersistentCacheIndex(self.persistentCacheIndex) self.cumulusClientId = CumulusNative.CumulusClientId( Hash.Hash.sha1(str(uuid.uuid4()))) logging.info("CumulusClient created with %s", self.cumulusClientId) self.cumulusClient = CumulusNative.CumulusClient( vdm, self.cumulusClientId, self.callbackScheduler) self.finalResponses = Queue.Queue() self.cumulusClientListener = self.cumulusClient.createListener() self.cpuAssignmentDependencyGraph = CumulusNative.CpuAssignmentDependencyGraph( self.callbackScheduler.getFactory().createScheduler( self.callbackScheduler.getMetadata() + "_cpuAssignmentGraph", 1), self.vdm) self.cpuAssignmentDependencyGraph.subscribeToCumulusClient( self.cumulusClient) self.pendingCallbacksByGuid = {} self.cpuAssignmentDependencyGraphListener = \ self.cpuAssignmentDependencyGraph.createListener() self.threads = [] self.threads.append( ManagedThread.ManagedThread(target=self.processClientMessages_, args=())) self.threads.append( ManagedThread.ManagedThread( target=self.processDependencyGraphMessages_, args=())) for t in self.threads: t.start() self.nextCpuUpdateTime = time.time() self.cpuMessagesSinceLastUpdate = 0 self.lastSystemwideUpdateTime = time.time()
def test_basicPersistentCache(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage(sha1("page2"), HashSet() + sha1("bigvec1"), 3, sha1("")) cppView1.addBigvec(sha1("bigvec2"), HashSet() + sha1("page2"), 4, sha1("")) self.assertEqual(cppView1.totalBytesInCache(), 10) def seesEverything(): if not cppView2.pageExists(sha1("page1")): return False if not cppView2.pageExists(sha1("page2")): return False if not cppView2.bigvecExists(sha1("bigvec1")): return False if not cppView2.bigvecExists(sha1("bigvec2")): return False return True self.waitForSync(seesEverything) for view in [cppView1, cppView2]: self.assertEqual(view.pageBytecount(sha1("page1")), 1) self.assertEqual(view.bigvecBytecount(sha1("bigvec1")), 2) self.assertEqual(view.pageBytecount(sha1("page2")), 3) self.assertEqual(view.bigvecBytecount(sha1("bigvec2")), 4) self.assertEqual(view.totalBytesInCache(), 10)
def __init__(self, ownAddress, channelListener, channelFactory, eventHandler, callbackScheduler, diagnosticsDir, config, viewFactory, s3InterfaceFactory=None, objectStore=None): Stoppable.Stoppable.__init__(self) #acquire a machineId randomly, using uuid self.machineId = CumulusNative.MachineId( Hash.Hash.sha1(str(uuid.uuid4())) ) self.ownAddress = ownAddress self.callbackScheduler = callbackScheduler self.viewFactory = viewFactory self.s3InterfaceFactory = s3InterfaceFactory self.objectStore = objectStore self.threadsStarted_ = False self.connectedMachines = set() self.connectingMachines = set() # machines we are in the process of connecting to self.droppedMachineIds = set() self.lock = threading.RLock() self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024 self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024 self.cumulusThreadCountOverride = config.cumulusServiceThreadCount self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc self.eventHandler = eventHandler self.reconnectPersistentCacheIndexViewThreads = [] if config.cumulusDiskCacheStorageSubdirectory is not None: self.cumulusDiskCacheWantsDeletionOnTeardown = True self.cumulusDiskCacheStorageDir = os.path.join( config.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageSubdirectory ) else: self.cumulusDiskCacheWantsDeletionOnTeardown = False self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir self._stopEvent = threading.Event() self._channelListener = channelListener assert len(self._channelListener.ports) == 2 self._channelFactory = channelFactory Runtime.initialize() ModuleImporter.initialize() self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines( self.viewFactory ) self.cumulusChannelFactoryThread = ManagedThread.ManagedThread( target=self._channelListener.start ) self.vdm = VectorDataManager.constructVDM( callbackScheduler, self.cumulusVectorRamCacheSizeOverride, self.cumulusMaxRamCacheSizeOverride ) if self.cumulusTrackTcmalloc: self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory() self.persistentCacheIndex = CumulusNative.PersistentCacheIndex( viewFactory.createView(retrySeconds=10.0, numRetries=10), callbackScheduler ) self.vdm.setPersistentCacheIndex(self.persistentCacheIndex) self.deleteCumulusDiskCacheIfNecessary() self.offlineCache = CumulusNative.DiskOfflineCache( callbackScheduler, self.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageMB * 1024 * 1024, config.cumulusDiskCacheStorageFileCount ) #If the "s3InterfaceFactory" is not in-memory, we use real out of process python. #it would be better if this were more explicit outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess) self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks) checkpointInterval = config.cumulusCheckpointIntervalSeconds if checkpointInterval == 0: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None() else: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic( checkpointInterval, 1024 * 1024 ) self.cumulusWorker = self.constructCumlusWorker( callbackScheduler, CumulusNative.CumulusWorkerConfiguration( self.machineId, self.cumulusThreadCountOverride, checkpointPolicy, ExecutionContext.createContextConfiguration(), diagnosticsDir or "" ), self.vdm, self.offlineCache, eventHandler ) self.datasetLoadService = None if self.s3InterfaceFactory: externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel( callbackScheduler ) self.datasetLoadService = PythonIoTaskService.PythonIoTaskService( self.s3InterfaceFactory, self.objectStore, self.vdm, externalDatasetChannel.makeQueuelike(callbackScheduler) ) self.cumulusWorker.startComputations() if self.datasetLoadService: self.datasetLoadService.startService()
def __init__(self, ownAddress, channelListener, channelFactory, eventHandler, callbackScheduler, diagnosticsDir, config, viewFactory): Stoppable.Stoppable.__init__(self) #acquire a machineId randomly, using uuid self.machineId = CumulusNative.MachineId( Hash.Hash.sha1(str(uuid.uuid4()))) self.ownAddress = ownAddress self.callbackScheduler = callbackScheduler self.viewFactory = viewFactory self.threadsStarted_ = False self.connectedMachines = set() self.connectingMachines = set( ) # machines we are in the process of connecting to self.droppedMachineIds = set() self.lock = threading.RLock() self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024 self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024 self.cumulusThreadCountOverride = config.cumulusServiceThreadCount self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc self.reconnectPersistentCacheIndexViewThreads = [] if config.cumulusDiskCacheStorageSubdirectory is not None: self.cumulusDiskCacheWantsDeletionOnTeardown = True self.cumulusDiskCacheStorageDir = os.path.join( config.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageSubdirectory) else: self.cumulusDiskCacheWantsDeletionOnTeardown = False self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir logging.info( "Creating a CumulusService with ram cache of %s / %s MB and %s threads", self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0, self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0, self.cumulusThreadCountOverride) self._stopEvent = threading.Event() self._channelListener = channelListener assert len(self._channelListener.ports) == 2 self._channelFactory = channelFactory Runtime.initialize() ModuleImporter.initialize() self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines( self.viewFactory) self.cumulusChannelFactoryThread = ManagedThread.ManagedThread( target=self._channelListener.start) self.vdm = VectorDataManager.constructVDM( callbackScheduler, self.cumulusVectorRamCacheSizeOverride, self.cumulusMaxRamCacheSizeOverride) if self.cumulusTrackTcMalloc: logging.info( "CumulusService enabling track-tc-malloc memory with a max cache of %s MB", self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0) self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory() self.persistentCacheIndex = CumulusNative.PersistentCacheIndex( viewFactory.createView(retrySeconds=10.0, numRetries=10), callbackScheduler) self.vdm.setPersistentCacheIndex(self.persistentCacheIndex) self.deleteCumulusDiskCacheIfNecessary() self.offlineCache = CumulusNative.DiskOfflineCache( callbackScheduler, self.cumulusDiskCacheStorageDir, config.cumulusDiskCacheStorageMB * 1024 * 1024, config.cumulusDiskCacheStorageFileCount) checkpointInterval = config.cumulusCheckpointIntervalSeconds if checkpointInterval == 0: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None () else: checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic( checkpointInterval, 1024 * 1024) self.cumulusWorker = self.constructCumlusWorker( callbackScheduler, CumulusNative.CumulusWorkerConfiguration( self.machineId, self.cumulusThreadCountOverride, checkpointPolicy, ExecutionContext.createContextConfiguration(), diagnosticsDir or ""), self.vdm, self.offlineCache, eventHandler) #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel( #callbackScheduler #) #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService( #settings.s3InterfaceFactory, #settings.objectStore, #self.vdm, #externalDatasetChannel.makeQueuelike(callbackScheduler) #) self.cumulusWorker.startComputations()
def test_orphanedPageIsCollected(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) self.assertTrue(len(cppView1.computeInvalidObjects()) == 1)