def __init__(self, channelManager):
self.channelManager = channelManager
self.rateLimitedChannelGroup = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.channelManager.callbackScheduler,
self.channelManager.perMachineThroughput
)
)
def __init__(self, channelManager):
self.channelManager = channelManager
self.rateLimitedChannelGroup = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.channelManager.callbackScheduler,
self.channelManager.perMachineThroughput
)
)
def addWorker(self):
self.workerCount += 1
newMachineId = self.allocateMachineId()
self.machineIds.append(newMachineId)
self.workersVdmsAndEventHandlers.append(
createWorker_(
newMachineId,
self.sharedStateViewFactory,
memoryLimitMb = self.memoryPerWorkerMB,
threadCount=self.threadsPerWorker,
callbackSchedulerToUse = self.callbackScheduler,
cacheFunction = self.cacheFunction,
pageSizeOverride = self.pageSizeOverride,
disableEventHandler = self.disableEventHandler
)
)
if self.channelThroughputMBPerSecond is not None:
self.rateLimitedChannelGroupsForEachListener.append(
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler,
int(self.channelThroughputMBPerSecond * 1024 * 1024 * (.5 + random.random()))
)
)
self.workersVdmsAndEventHandlers[-1][0].startComputations()
for ix1 in range(len(self.workersVdmsAndEventHandlers)-1):
ix2 = len(self.workersVdmsAndEventHandlers)-1
self.wireWorkersTogether_(ix1, ix2)
for ix1 in range(len(self.clientsAndVdms)):
ix2 = len(self.workersVdmsAndEventHandlers)-1
self.wireWorkerToClient_(ix2, ix1)
s3InterfaceFactory = self.s3Service.withMachine(self.machineIdsEverAllocated - 1)
worker, workerVdm, _ = self.workersVdmsAndEventHandlers[-1]
loadingService = PythonIoTaskService.PythonIoTaskService(
s3InterfaceFactory,
self.objectStore,
workerVdm,
worker.getExternalDatasetRequestChannel(self.callbackScheduler)
.makeQueuelike(self.callbackScheduler),
threadCount=self.ioTaskThreadOverride,
maxObjectStoreAttempts=1,
objectStoreFailureInterval=0
)
loadingService.startService()
self.loadingServices.append(loadingService)
self.workerTeardownGates.append(workerVdm.getVdmmTeardownGate())
def registerChannelListener(self, address, port, connectionCallback):
with self.lock:
assert (address, port) not in self.listeners
key = (str(address), int(port))
self.listeners[key] = connectionCallback
self.rateLimitedChannelGroupsForEachListener[key] = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler, self.perMachineThroughput))
def registerChannelListener(self, address, port, connectionCallback):
with self.lock:
assert (address, port) not in self.listeners
key = (str(address), int(port))
self.listeners[key] = connectionCallback
self.rateLimitedChannelGroupsForEachListener[key] = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler,
self.perMachineThroughput
)
)
