def createWorkersAndClients(workerCount, clientCount, viewFactory=None, memoryLimitMb=100, threadCount=2, callbackSchedulerToUse=None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if viewFactory is None: viewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse) workersVdmsAndEventHandlers = [ createWorker(machineId(ix), viewFactory, memoryLimitMb=memoryLimitMb, threadCount=threadCount, callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(workerCount) ] clientsAndVdms = [ createClient(clientId(ix), callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(clientCount) ] for ix1 in range(len(workersVdmsAndEventHandlers)): workersVdmsAndEventHandlers[ix1][0].startComputations() for ix1 in range(len(workersVdmsAndEventHandlers) - 1): for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addMachine( machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) workersVdmsAndEventHandlers[ix2][0].addMachine( machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) for ix1 in range(len(workersVdmsAndEventHandlers)): for ix2 in range(len(clientsAndVdms)): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addCumulusClient( clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def wireWorkerToClient_(self, ix1, ix2): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient( clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), self.callbackScheduler) self.clientsAndVdms[ix2][0].addMachine( self.machineIds[ix1], [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def wireWorkersTogether_(self, ix1, ix2): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) if self.channelThroughputMBPerSecond is not None: worker1Channel1 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel1) worker1Channel2 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel2) worker2Channel1 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel1) worker2Channel2 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel2) self.workersVdmsAndEventHandlers[ix1][0].addMachine(self.machineIds[ix2], [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), self.callbackScheduler) self.workersVdmsAndEventHandlers[ix2][0].addMachine(self.machineIds[ix1], [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def test_disconnect_channel(self): channel1, channel2 = StringChannelNative.InMemoryStringChannel(callbackScheduler) channel1Queue = channel1.makeQueuelike(callbackScheduler) channel2Queue = channel2.makeQueuelike(callbackScheduler) successes = [False, False] def get1(): try: channel1Queue.get() except: successes[0] = True def get2(): try: channel2Queue.get() except: successes[1] = True thread1 = threading.Thread(target=get1) thread2 = threading.Thread(target=get2) thread1.start() thread2.start() time.sleep(.05) self.assertEqual(successes, [False, False]) channel1.disconnect() thread1.join() thread2.join() self.assertEqual(successes, [True, True])
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds): s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 1, 0, s3Service=s3Service, memoryPerWorkerMB=400, callbackScheduler=callbackScheduler, threadsPerWorker=Setup.config().cumulusServiceThreadCount, **kwds) CumulusGateway.CumulusGateway.__init__( self, callbackScheduler, vdm, simulation.sharedStateViewFactory) self.s3Service = s3Service self.simulation = simulation self.viewFactory = self.simulation.sharedStateViewFactory self.worker = self.simulation.getWorker(0) self.workerVdm = self.simulation.getWorkerVdm(0) channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) machineId = self.workerVdm.getMachineId() self.cumulusClient.addMachine(machineId, [channel1Client, channel2Client], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.worker.addCumulusClient(self.cumulusClientId, [channel1Worker, channel2Worker], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.loadingService = self.simulation.loadingServices[0]
def test_delayed_connection_workd(self): channel1, channel2 = StringChannelNative.InMemoryStringChannel(callbackScheduler) channel1Queue = channel1.makeQueuelike(callbackScheduler) channel1Queue.write("a") channel1Queue.write("b") channel1Queue.disconnect() channel2Queue = channel2.makeQueuelike(callbackScheduler) self.assertTrue(channel2Queue.getTimeout(1.0),"a") self.assertTrue(channel2Queue.getTimeout(1.0),"b") self.assertRaises(channel2Queue.getNonblocking)
def createChannelToEndpoint(self, ipEndpoint): with self.lock: assert len(ipEndpoint) == 2 # we expect addresses to be strings and ports to be ints ipEndpoint = (str(ipEndpoint[0]), int(ipEndpoint[1])) assert ipEndpoint in self.listeners, ( "No listeners registered on endpoint: %s. Registered listeners: %s" % (ipEndpoint, self.listeners)) clientChannel, serverChannel = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) rateLimitedServerChannel = ( self.rateLimitedChannelGroupsForEachListener[ipEndpoint].wrap( serverChannel)) self.listeners[ipEndpoint](rateLimitedServerChannel.makeQueuelike( self.callbackScheduler)) return clientChannel
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds): #don't modify callers directly kwds = dict(kwds) if 's3Service' in kwds: s3Service = kwds['s3Service'] del kwds['s3Service'] else: s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory() if 'threadsPerWorker' in kwds: threadsPerWorker = kwds['threadsPerWorker'] del kwds['threadsPerWorker'] else: threadsPerWorker = Setup.config().cumulusServiceThreadCount if 'memoryPerWorkerMB' in kwds: memoryPerWorkerMB = kwds['memoryPerWorkerMB'] del kwds['memoryPerWorkerMB'] else: memoryPerWorkerMB = 400 if 'maxMBPerOutOfProcessPythonTask' in kwds: maxBytesPerOutOfProcessPythonTask = kwds[ 'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024 del kwds['maxMBPerOutOfProcessPythonTask'] else: maxBytesPerOutOfProcessPythonTask = None workerCount = 1 if 'workerCount' in kwds: workerCount = kwds['workerCount'] del kwds['workerCount'] simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( workerCount, 0, s3Service=s3Service, memoryPerWorkerMB=memoryPerWorkerMB, callbackScheduler=callbackScheduler, threadsPerWorker=threadsPerWorker, maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask, **kwds) CumulusGateway.CumulusGateway.__init__( self, callbackScheduler, vdm, simulation.sharedStateViewFactory) self.s3Service = s3Service self.simulation = simulation self.viewFactory = self.simulation.sharedStateViewFactory for workerIx in xrange(workerCount): worker = self.simulation.getWorker(workerIx) workerVdm = self.simulation.getWorkerVdm(workerIx) channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) machineId = workerVdm.getMachineId() self.cumulusClient.addMachine( machineId, [channel1Client, channel2Client], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) worker.addCumulusClient(self.cumulusClientId, [channel1Worker, channel2Worker], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.worker = self.simulation.getWorker(0) self.workerVdm = self.simulation.getWorkerVdm(0) self.loadingService = self.simulation.loadingServices[0]