def test_expansionWithVecOfVec(self): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3) try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.getGlobalScheduler().setCheckpointStatusInterval(0.0001) simulation.submitComputation( "Vector.range(20, fun(ix) { Vector.range(100000+ix).paged }).paged" ) simulation.waitForAnyResult() simulation.addWorker() self.assertTrue(simulation.waitForHandshake()) finally: simulation.teardown()
def test_cumulusCanTriggerNewRegimes(self): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3) try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.waitForHandshake() regime = simulation.getWorker(0).getRegimeHash() self.assertTrue(regime is not None) simulation.getWorker(0).triggerRegimeChange() time.sleep(1.0) simulation.waitForHandshake() regime2 = simulation.getWorker(0).getRegimeHash() self.assertTrue(regime2 is not None) self.assertTrue(regime2 != regime) finally: simulation.teardown()
def loadCheckpointFromFreshSimulationTest(self, calculationText, timestampsPerPassList, clientCount=1, timestep=1.0): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() statuses = [] viewFactory = None for timestampsThisPass in timestampsPerPassList: simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count clientCount, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3, sharedStateViewFactory=viewFactory) viewFactory = simulation.sharedStateViewFactory statusesThisPass = [] try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.getGlobalScheduler().setCheckpointStatusInterval( 0.1) for ix in range(clientCount): simulation.submitComputationOnClient(ix, calculationText) for subPass in range(timestampsThisPass): time.sleep(timestep) statusesThisPass.append( self.timeElapsedOfMostRecentCheckpoints(simulation)) simulation.getGlobalScheduler( ).triggerFullCheckpointsOnOutstandingComputations() self.waitForFullCheckpoint(simulation) statusesThisPass.append( self.timeElapsedOfMostRecentCheckpoints(simulation)) finally: for ix in range(4): simulation.getWorker(ix).dumpStateToLog() simulation.teardown() statuses.append(statusesThisPass) return statuses
def createSimulation(self, useHdfsObjectStore=False, objectStore=None, sharedStateViewFactory=None, workerCount=4, machineIdHashSeed=None, s3Service=None): s3 = s3Service or InMemoryS3Interface.InMemoryS3InterfaceFactory() return InMemoryCumulusSimulation.InMemoryCumulusSimulation( workerCount, 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3, objectStore=objectStore, sharedStateViewFactory=sharedStateViewFactory, machineIdHashSeed=machineIdHashSeed)
def DISABLEDtest_canTriggerCheckpointOfCompleted(self): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3) try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.waitForHandshake() simulation.submitComputation("1+2") result = simulation.waitForAnyResult() self.assertTrue(result.isResult()) simulation.getGlobalScheduler( ).triggerFullCheckpointsOnOutstandingComputations() ts = self.waitForFullCheckpoint(simulation, onlyUnfinished=False) self.assertTrue(ts is not None) statuses = simulation.getGlobalScheduler( ).currentOutstandingCheckpointStatuses(False, False) status = statuses[0] compId = status[0] checkpointStatus = status[1][0] checkpointRequest = status[1][1] #verify that it's a storage checkpoint self.assertTrue(checkpointRequest.writeToStorage) self.assertTrue(checkpointStatus.checkpointSuccessful) self.assertTrue(checkpointStatus.isRootComputationFinished) finally: simulation.teardown()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds): s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 1, 0, s3Service=s3Service, memoryPerWorkerMB=400, callbackScheduler=callbackScheduler, threadsPerWorker=Setup.config().cumulusServiceThreadCount, **kwds) CumulusGateway.CumulusGateway.__init__( self, callbackScheduler, vdm, simulation.sharedStateViewFactory) self.s3Service = s3Service self.simulation = simulation self.viewFactory = self.simulation.sharedStateViewFactory self.worker = self.simulation.getWorker(0) self.workerVdm = self.simulation.getWorkerVdm(0) channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) machineId = self.workerVdm.getMachineId() self.cumulusClient.addMachine(machineId, [channel1Client, channel2Client], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.worker.addCumulusClient(self.cumulusClientId, [channel1Worker, channel2Worker], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.loadingService = self.simulation.loadingServices[0]
def test_dependingOnCheckpointedFinishedCachecallWorks(self): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( 4, #worker count 1, memoryPerWorkerMB=100, threadsPerWorker=2, s3Service=s3) try: self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0)) simulation.waitForHandshake() baseComp = simulation.createComputation("1+2") baseCompId = simulation.getClient(0).createComputation(baseComp) simulation.getClient(0).setComputationPriority( baseCompId, CumulusNative.ComputationPriority(1)) result = simulation.waitForAnyResult() self.assertTrue(result.isResult()) simulation.getGlobalScheduler( ).triggerFullCheckpointsOnOutstandingComputations() self.waitForAllCheckpointsToClear(simulation) for ix in range(100): childComp = simulation.createComputation("x + %s" % ix, x=baseComp) childCompId = simulation.getClient(0).createComputation( childComp) simulation.getClient(0).setComputationPriority( childCompId, CumulusNative.ComputationPriority(1)) result = simulation.waitForAnyResult() finally: simulation.teardown()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds): #don't modify callers directly kwds = dict(kwds) if 's3Service' in kwds: s3Service = kwds['s3Service'] del kwds['s3Service'] else: s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory() if 'threadsPerWorker' in kwds: threadsPerWorker = kwds['threadsPerWorker'] del kwds['threadsPerWorker'] else: threadsPerWorker = Setup.config().cumulusServiceThreadCount if 'memoryPerWorkerMB' in kwds: memoryPerWorkerMB = kwds['memoryPerWorkerMB'] del kwds['memoryPerWorkerMB'] else: memoryPerWorkerMB = 400 if 'maxMBPerOutOfProcessPythonTask' in kwds: maxBytesPerOutOfProcessPythonTask = kwds[ 'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024 del kwds['maxMBPerOutOfProcessPythonTask'] else: maxBytesPerOutOfProcessPythonTask = None workerCount = 1 if 'workerCount' in kwds: workerCount = kwds['workerCount'] del kwds['workerCount'] simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation( workerCount, 0, s3Service=s3Service, memoryPerWorkerMB=memoryPerWorkerMB, callbackScheduler=callbackScheduler, threadsPerWorker=threadsPerWorker, maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask, **kwds) CumulusGateway.CumulusGateway.__init__( self, callbackScheduler, vdm, simulation.sharedStateViewFactory) self.s3Service = s3Service self.simulation = simulation self.viewFactory = self.simulation.sharedStateViewFactory for workerIx in xrange(workerCount): worker = self.simulation.getWorker(workerIx) workerVdm = self.simulation.getWorkerVdm(workerIx) channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) machineId = workerVdm.getMachineId() self.cumulusClient.addMachine( machineId, [channel1Client, channel2Client], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) worker.addCumulusClient(self.cumulusClientId, [channel1Worker, channel2Worker], ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) self.worker = self.simulation.getWorker(0) self.workerVdm = self.simulation.getWorkerVdm(0) self.loadingService = self.simulation.loadingServices[0]