def createWorker(machineId,
viewFactory,
callbackSchedulerToUse=None,
threadCount=2,
memoryLimitMb=100):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024)))
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(viewFactory.createView(),
callbackSchedulerToUse))
cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse,
1000 * 1024 * 1024)
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId, threadCount,
CumulusNative.CumulusCheckpointPolicy.None (),
ExecutionContext.createContextConfiguration(), ""), vdm, cache,
eventHandler), vdm, eventHandler)
def createServiceAndServiceThread(self):
config = Setup.config()
config.cumulusMaxRamCacheMB = self.cumulusMaxRamCacheSizeOverride / 1024 / 1024
config.cumulusVectorRamCacheMB = self.cumulusVectorRamCacheSizeOverride / 1024 / 1024
config.cumulusTrackTcmalloc = False
config.cumulusServiceThreadCount = self.cumulusThreadCountOverride
config.cumulusDiskCacheStorageSubdirectory = str(uuid.uuid4())
ownAddress = str(uuid.uuid4())
callbackScheduler = self.callbackSchedulerFactory.createScheduler(
"InMemoryClusterChild",
1)
channelListener = self.createMultiChannelListener(
callbackScheduler,
[Setup.config().cumulusControlPort, Setup.config().cumulusDataPort],
ownAddress)
service = CumulusService.CumulusService(
ownAddress=ownAddress,
channelListener=channelListener,
channelFactory=self.channelManager.createChannelFactory(),
eventHandler=CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler(),
callbackScheduler=callbackScheduler,
diagnosticsDir=None,
config=config,
viewFactory=self.sharedStateViewFactory
)
service.startService(lambda: None)
return service
def createWorker_(machineId,
viewFactory,
callbackSchedulerToUse,
threadCount,
memoryLimitMb,
cacheFunction,
pageSizeOverride,
disableEventHandler):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(
callbackSchedulerToUse,
pageSizeOverride if pageSizeOverride is not None else
1 * 1024 * 1024 if memoryLimitMb < 1000 else
5 * 1024 * 1024 if memoryLimitMb < 5000 else
50 * 1024 * 1024
)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
)
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(
viewFactory.createView(),
callbackSchedulerToUse
)
)
cache = cacheFunction()
if disableEventHandler:
eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler()
else:
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (
CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId,
threadCount,
CumulusNative.CumulusCheckpointPolicy.None(),
ExecutionContext.createContextConfiguration(),
""
),
vdm,
cache,
eventHandler
),
vdm,
eventHandler
)