def createWorkersAndClients(workerCount,
clientCount,
viewFactory=None,
memoryLimitMb=100,
threadCount=2,
callbackSchedulerToUse=None):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
if viewFactory is None:
viewFactory = createInMemorySharedStateViewFactory(
callbackSchedulerToUse)
workersVdmsAndEventHandlers = [
createWorker(machineId(ix),
viewFactory,
memoryLimitMb=memoryLimitMb,
threadCount=threadCount,
callbackSchedulerToUse=callbackSchedulerToUse)
for ix in range(workerCount)
]
clientsAndVdms = [
createClient(clientId(ix),
callbackSchedulerToUse=callbackSchedulerToUse)
for ix in range(clientCount)
]
for ix1 in range(len(workersVdmsAndEventHandlers)):
workersVdmsAndEventHandlers[ix1][0].startComputations()
for ix1 in range(len(workersVdmsAndEventHandlers) - 1):
for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)):
worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addMachine(
machineId(ix2), [worker1Channel1, worker1Channel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix2][0].addMachine(
machineId(ix1), [worker2Channel1, worker2Channel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
for ix1 in range(len(workersVdmsAndEventHandlers)):
for ix2 in range(len(clientsAndVdms)):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addCumulusClient(
clientId(ix2), [workerChannel1, workerChannel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
clientsAndVdms[ix2][0].addMachine(machineId(ix1),
[clientChannel1, clientChannel2],
ForaNative.ImplValContainer(),
callbackSchedulerToUse)
return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def wireWorkerToClient_(self, ix1, ix2):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient(
clientId(ix2), [workerChannel1, workerChannel2],
ForaNative.ImplValContainer(), self.callbackScheduler)
self.clientsAndVdms[ix2][0].addMachine(
self.machineIds[ix1], [clientChannel1, clientChannel2],
ForaNative.ImplValContainer(), self.callbackScheduler)
def wireWorkersTogether_(self, ix1, ix2):
worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
if self.channelThroughputMBPerSecond is not None:
worker1Channel1 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel1)
worker1Channel2 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel2)
worker2Channel1 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel1)
worker2Channel2 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel2)
self.workersVdmsAndEventHandlers[ix1][0].addMachine(self.machineIds[ix2], [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
self.workersVdmsAndEventHandlers[ix2][0].addMachine(self.machineIds[ix1], [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def test_disconnect_channel(self):
channel1, channel2 = StringChannelNative.InMemoryStringChannel(callbackScheduler)
channel1Queue = channel1.makeQueuelike(callbackScheduler)
channel2Queue = channel2.makeQueuelike(callbackScheduler)
successes = [False, False]
def get1():
try:
channel1Queue.get()
except:
successes[0] = True
def get2():
try:
channel2Queue.get()
except:
successes[1] = True
thread1 = threading.Thread(target=get1)
thread2 = threading.Thread(target=get2)
thread1.start()
thread2.start()
time.sleep(.05)
self.assertEqual(successes, [False, False])
channel1.disconnect()
thread1.join()
thread2.join()
self.assertEqual(successes, [True, True])
def __init__(self, channelManager):
self.channelManager = channelManager
self.rateLimitedChannelGroup = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.channelManager.callbackScheduler,
self.channelManager.perMachineThroughput
)
)
def __init__(self, channelManager):
self.channelManager = channelManager
self.rateLimitedChannelGroup = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.channelManager.callbackScheduler,
self.channelManager.perMachineThroughput
)
)
def addWorker(self):
self.workerCount += 1
newMachineId = self.allocateMachineId()
self.machineIds.append(newMachineId)
self.workersVdmsAndEventHandlers.append(
createWorker_(
newMachineId,
self.sharedStateViewFactory,
memoryLimitMb = self.memoryPerWorkerMB,
threadCount=self.threadsPerWorker,
callbackSchedulerToUse = self.callbackScheduler,
cacheFunction = self.cacheFunction,
pageSizeOverride = self.pageSizeOverride,
disableEventHandler = self.disableEventHandler
)
)
if self.channelThroughputMBPerSecond is not None:
self.rateLimitedChannelGroupsForEachListener.append(
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler,
int(self.channelThroughputMBPerSecond * 1024 * 1024 * (.5 + random.random()))
)
)
self.workersVdmsAndEventHandlers[-1][0].startComputations()
for ix1 in range(len(self.workersVdmsAndEventHandlers)-1):
ix2 = len(self.workersVdmsAndEventHandlers)-1
self.wireWorkersTogether_(ix1, ix2)
for ix1 in range(len(self.clientsAndVdms)):
ix2 = len(self.workersVdmsAndEventHandlers)-1
self.wireWorkerToClient_(ix2, ix1)
s3InterfaceFactory = self.s3Service.withMachine(self.machineIdsEverAllocated - 1)
worker, workerVdm, _ = self.workersVdmsAndEventHandlers[-1]
loadingService = PythonIoTaskService.PythonIoTaskService(
s3InterfaceFactory,
self.objectStore,
workerVdm,
worker.getExternalDatasetRequestChannel(self.callbackScheduler)
.makeQueuelike(self.callbackScheduler),
threadCount=self.ioTaskThreadOverride,
maxObjectStoreAttempts=1,
objectStoreFailureInterval=0
)
loadingService.startService()
self.loadingServices.append(loadingService)
self.workerTeardownGates.append(workerVdm.getVdmmTeardownGate())
def registerChannelListener(self, address, port, connectionCallback):
with self.lock:
assert (address, port) not in self.listeners
key = (str(address), int(port))
self.listeners[key] = connectionCallback
self.rateLimitedChannelGroupsForEachListener[key] = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler, self.perMachineThroughput))
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
1,
0,
s3Service=s3Service,
memoryPerWorkerMB=400,
callbackScheduler=callbackScheduler,
threadsPerWorker=Setup.config().cumulusServiceThreadCount,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = self.workerVdm.getMachineId()
self.cumulusClient.addMachine(machineId,
[channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.loadingService = self.simulation.loadingServices[0]
def test_delayed_connection_workd(self):
channel1, channel2 = StringChannelNative.InMemoryStringChannel(callbackScheduler)
channel1Queue = channel1.makeQueuelike(callbackScheduler)
channel1Queue.write("a")
channel1Queue.write("b")
channel1Queue.disconnect()
channel2Queue = channel2.makeQueuelike(callbackScheduler)
self.assertTrue(channel2Queue.getTimeout(1.0),"a")
self.assertTrue(channel2Queue.getTimeout(1.0),"b")
self.assertRaises(channel2Queue.getNonblocking)
def registerChannelListener(self, address, port, connectionCallback):
with self.lock:
assert (address, port) not in self.listeners
key = (str(address), int(port))
self.listeners[key] = connectionCallback
self.rateLimitedChannelGroupsForEachListener[key] = (
StringChannelNative.createRateLimitedStringChannelGroup(
self.callbackScheduler,
self.perMachineThroughput
)
)
def createChannelToEndpoint(self, ipEndpoint):
with self.lock:
assert len(ipEndpoint) == 2
# we expect addresses to be strings and ports to be ints
ipEndpoint = (str(ipEndpoint[0]), int(ipEndpoint[1]))
assert ipEndpoint in self.listeners, (
"No listeners registered on endpoint: %s. Registered listeners: %s"
% (ipEndpoint, self.listeners))
clientChannel, serverChannel = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
rateLimitedServerChannel = (
self.rateLimitedChannelGroupsForEachListener[ipEndpoint].wrap(
serverChannel))
self.listeners[ipEndpoint](rateLimitedServerChannel.makeQueuelike(
self.callbackScheduler))
return clientChannel
def setUp(self):
self.serverChannel, self.clientChannel = StringChannelNative.QueuelikeInMemoryStringChannel(
callbackScheduler)
self.transport = InMemoryTransport.InMemoryTransport(
self.clientChannel)
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
#don't modify callers directly
kwds = dict(kwds)
if 's3Service' in kwds:
s3Service = kwds['s3Service']
del kwds['s3Service']
else:
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
if 'threadsPerWorker' in kwds:
threadsPerWorker = kwds['threadsPerWorker']
del kwds['threadsPerWorker']
else:
threadsPerWorker = Setup.config().cumulusServiceThreadCount
if 'memoryPerWorkerMB' in kwds:
memoryPerWorkerMB = kwds['memoryPerWorkerMB']
del kwds['memoryPerWorkerMB']
else:
memoryPerWorkerMB = 400
if 'maxMBPerOutOfProcessPythonTask' in kwds:
maxBytesPerOutOfProcessPythonTask = kwds[
'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024
del kwds['maxMBPerOutOfProcessPythonTask']
else:
maxBytesPerOutOfProcessPythonTask = None
workerCount = 1
if 'workerCount' in kwds:
workerCount = kwds['workerCount']
del kwds['workerCount']
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
workerCount,
0,
s3Service=s3Service,
memoryPerWorkerMB=memoryPerWorkerMB,
callbackScheduler=callbackScheduler,
threadsPerWorker=threadsPerWorker,
maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
for workerIx in xrange(workerCount):
worker = self.simulation.getWorker(workerIx)
workerVdm = self.simulation.getWorkerVdm(workerIx)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = workerVdm.getMachineId()
self.cumulusClient.addMachine(
machineId, [channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(), self.callbackScheduler)
worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
self.loadingService = self.simulation.loadingServices[0]