def createWorker(machineId, viewFactory, callbackSchedulerToUse=None, threadCount=2, memoryLimitMb=100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024))) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex(viewFactory.createView(), callbackSchedulerToUse)) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return (CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None (), ExecutionContext.createContextConfiguration(), ""), vdm, cache, eventHandler), vdm, eventHandler)
def createServiceAndServiceThread(self): config = Setup.config() config.cumulusMaxRamCacheMB = self.cumulusMaxRamCacheSizeOverride / 1024 / 1024 config.cumulusVectorRamCacheMB = self.cumulusVectorRamCacheSizeOverride / 1024 / 1024 config.cumulusTrackTcmalloc = False config.cumulusServiceThreadCount = self.cumulusThreadCountOverride config.cumulusDiskCacheStorageSubdirectory = str(uuid.uuid4()) ownAddress = str(uuid.uuid4()) callbackScheduler = self.callbackSchedulerFactory.createScheduler( "InMemoryClusterChild", 1) channelListener = self.createMultiChannelListener( callbackScheduler, [Setup.config().cumulusControlPort, Setup.config().cumulusDataPort], ownAddress) service = CumulusService.CumulusService( ownAddress=ownAddress, channelListener=channelListener, channelFactory=self.channelManager.createChannelFactory(), eventHandler=CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler(), callbackScheduler=callbackScheduler, diagnosticsDir=None, config=config, viewFactory=self.sharedStateViewFactory ) service.startService(lambda: None) return service
def createWorker_(machineId, viewFactory, callbackSchedulerToUse, threadCount, memoryLimitMb, cacheFunction, pageSizeOverride, disableEventHandler): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager( callbackSchedulerToUse, pageSizeOverride if pageSizeOverride is not None else 1 * 1024 * 1024 if memoryLimitMb < 1000 else 5 * 1024 * 1024 if memoryLimitMb < 5000 else 50 * 1024 * 1024 ) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = cacheFunction() if disableEventHandler: eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler() else: eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )