def createWorker(machineId, viewFactory, callbackSchedulerToUse=None, threadCount=2, memoryLimitMb=100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024))) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex(viewFactory.createView(), callbackSchedulerToUse)) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return (CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None (), ExecutionContext.createContextConfiguration(), ""), vdm, cache, eventHandler), vdm, eventHandler)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.compiler = self.runtime.getTypedForaCompiler() self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant( FORA.builtin().implVal_)
def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed): object.__init__(self) self.callbackScheduler = CallbackScheduler.singletonForTesting() self.callbackSchedulerFactory = self.callbackScheduler.getFactory() self.numRandVals = numRandVals self.numRelaxations = numRelaxations self.maxForRelax = maxForRelax self.maxForRand = maxForRand self.seed = seed self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.typed_fora_compiler = self.runtime.getTypedForaCompiler() if testAxiomsPath is not None: pathToUse = testAxiomsPath else: pathToUse = UNIT_TEST_AXIOMS_PATH self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse) self.axiom_groups = [] for i in range(self.axioms.axiomCount): self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i)) self.symbol_strings = self.loadSymbolStrings() numpy.random.seed(seed)
def createWorkersAndClients(workerCount, clientCount, viewFactory=None, memoryLimitMb=100, threadCount=2, callbackSchedulerToUse=None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if viewFactory is None: viewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse) workersVdmsAndEventHandlers = [ createWorker(machineId(ix), viewFactory, memoryLimitMb=memoryLimitMb, threadCount=threadCount, callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(workerCount) ] clientsAndVdms = [ createClient(clientId(ix), callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(clientCount) ] for ix1 in range(len(workersVdmsAndEventHandlers)): workersVdmsAndEventHandlers[ix1][0].startComputations() for ix1 in range(len(workersVdmsAndEventHandlers) - 1): for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addMachine( machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) workersVdmsAndEventHandlers[ix2][0].addMachine( machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) for ix1 in range(len(workersVdmsAndEventHandlers)): for ix2 in range(len(clientsAndVdms)): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addCumulusClient( clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def createClient(clientId, callbackSchedulerToUse=None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit(100 * 1024 * 1024, 125 * 1024 * 1024) return (CumulusNative.CumulusClient(vdm, clientId, callbackSchedulerToUse), vdm)
def createWorker_(machineId, viewFactory, callbackSchedulerToUse, threadCount, memoryLimitMb, cacheFunction, pageSizeOverride, disableEventHandler): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager( callbackSchedulerToUse, pageSizeOverride if pageSizeOverride is not None else 1 * 1024 * 1024 if memoryLimitMb < 1000 else 5 * 1024 * 1024 if memoryLimitMb < 5000 else 50 * 1024 * 1024 ) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = cacheFunction() if disableEventHandler: eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler() else: eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def createClient_(clientId, callbackSchedulerToUse = None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit(100 * 1024 * 1024, 125 * 1024 * 1024) return ( CumulusNative.CumulusClient(vdm, clientId, callbackSchedulerToUse), vdm )
def test_roundtrip_conversion_simple(self): vdm = FORANative.VectorDataManager( CallbackScheduler.singletonForTesting(), 10000000) for obj in [ 10, 10.0, "asdf", None, False, True, [], (), [1, 2], [1, [1]], (1, 2), (1, 2, []), { 1: 2 } ]: self.assertEqual(roundtripConvert(obj, vdm)[0], obj, obj)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() def createStorage(vdm): self.simpleOfflineCache = CumulusNative.SimpleOfflineCache( self.callbackScheduler, 1000000000) return self.simpleOfflineCache self.evaluator = LocalEvaluator.LocalEvaluator( createStorage, 2000000, maxPageSizeInBytes=100000) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.compiler = self.runtime.getTypedForaCompiler() self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_) pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython") self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_) self.instructionGraph = self.runtime.getInstructionGraph() self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
def __init__(self, inMemory, port=None, cachePathOverride='', maxOpenFiles=256, inMemChannelFactoryFactory=None, maxLogFileSizeMb=10, pingInterval=None): self.inMemory = inMemory self.manager = None self.callbackScheduler = CallbackScheduler.singletonForTesting() if self.inMemory: self.manager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride=cachePathOverride, pingInterval=IN_MEMORY_HARNESS_PING_INTERVAL if pingInterval is None else pingInterval, maxOpenFiles=maxOpenFiles, maxLogFileSizeMb=maxLogFileSizeMb) #although named otherwise InMemoryChannelFactory is actually a factory for a channelFactory # or a channelFactoryFactory channelFactoryFactory = inMemChannelFactoryFactory if inMemChannelFactoryFactory is not None \ else InMemoryChannelFactory.InMemoryChannelFactory logging.info(channelFactoryFactory) self.channelFactory = channelFactoryFactory( self.callbackScheduler, self.manager) self.viewFactory = ViewFactory.ViewFactory(self.channelFactory) else: class Settings(object): callbackScheduler = self.callbackScheduler assert port is not None self.service = SharedStateService.SharedStateService( self.callbackScheduler, cachePathOverride=cachePathOverride, port=port) self.service.startService() self.service.blockUntilListening() self.viewFactory = ViewFactory.ViewFactory.TcpViewFactory( self.callbackScheduler, "localhost", port)
def createWorkersAndClients( workerCount, clientCount, viewFactory = None, memoryLimitMb = 100, threadCount = 2, callbackSchedulerToUse = None ): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if viewFactory is None: viewFactory = createInMemorySharedStateViewFactory(callbackSchedulerToUse) workersVdmsAndEventHandlers = [ createWorker( machineId(ix), viewFactory, memoryLimitMb = memoryLimitMb, threadCount=threadCount, callbackSchedulerToUse = callbackSchedulerToUse ) for ix in range(workerCount) ] clientsAndVdms = [ createClient( clientId(ix), callbackSchedulerToUse = callbackSchedulerToUse ) for ix in range(clientCount) ] for ix1 in range(len(workersVdmsAndEventHandlers)): workersVdmsAndEventHandlers[ix1][0].startComputations() for ix1 in range(len(workersVdmsAndEventHandlers)-1): for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addMachine(machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) workersVdmsAndEventHandlers[ix2][0].addMachine(machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) for ix1 in range(len(workersVdmsAndEventHandlers)): for ix2 in range(len(clientsAndVdms)): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() def createStorage(vdm): self.simpleOfflineCache = CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000) return self.simpleOfflineCache self.evaluator = LocalEvaluator.LocalEvaluator( createStorage, 2000000, maxPageSizeInBytes = 100000 ) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
def __init__(self, inMemory, port = None, cachePathOverride = '', maxOpenFiles = 256, inMemChannelFactoryFactory = None, maxLogFileSizeMb = 10, pingInterval = None): self.inMemory = inMemory self.manager = None self.callbackScheduler = CallbackScheduler.singletonForTesting() if self.inMemory: self.manager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride=cachePathOverride, pingInterval = IN_MEMORY_HARNESS_PING_INTERVAL if pingInterval is None else pingInterval, maxOpenFiles=maxOpenFiles, maxLogFileSizeMb=maxLogFileSizeMb ) #although named otherwise InMemoryChannelFactory is actually a factory for a channelFactory # or a channelFactoryFactory channelFactoryFactory = inMemChannelFactoryFactory if inMemChannelFactoryFactory is not None \ else InMemoryChannelFactory.InMemoryChannelFactory logging.info(channelFactoryFactory) self.channelFactory = channelFactoryFactory(self.callbackScheduler, self.manager) self.viewFactory = ViewFactory.ViewFactory(self.channelFactory) else: class Settings(object): callbackScheduler = self.callbackScheduler assert port is not None self.service = SharedStateService.SharedStateService( self.callbackScheduler, cachePathOverride=cachePathOverride, port=port ) self.service.startService() self.service.blockUntilListening() self.viewFactory = ViewFactory.ViewFactory.TcpViewFactory(self.callbackScheduler, "localhost", port)
def createInMemorySharedStateViewFactory(callbackSchedulerToUse=None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() sharedStateManager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride="", pingInterval=IN_MEMORY_CLUSTER_SS_PING_INTERVAL, maxOpenFiles=100) sharedStateChannelFactory = ( InMemorySharedStateChannelFactory.InMemoryChannelFactory( callbackSchedulerToUse.getFactory().createScheduler( "SharedState", 1), sharedStateManager)) return ViewFactory.ViewFactory(sharedStateChannelFactory)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vals_to_test = self.loadValuesFromFile( os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt")) self.evaluator = LocalEvaluator.LocalEvaluator( lambda vdm: CumulusNative.SimpleOfflineCache( self.callbackScheduler, 1000000000), 10000000, maxPageSizeInBytes=100000) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator) self.knownModulesAsConstantJOVs = dict() self.knownModulesAsConstantJOVs["builtin"] = \ FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt")) self.evaluator = LocalEvaluator.LocalEvaluator( lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000), 10000000, maxPageSizeInBytes = 100000 ) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator) self.knownModulesAsConstantJOVs = dict() self.knownModulesAsConstantJOVs["builtin"] = \ FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def test_roundtrip_convert_function(self): vdm = FORANative.VectorDataManager( CallbackScheduler.singletonForTesting(), 10000000) self.assertTrue( roundtripConvert( ThisIsAFunction, vdm, allowUserCodeModuleLevelLookups=True)[0] is ThisIsAFunction) self.assertTrue( roundtripConvert( ThisIsAClass, vdm, allowUserCodeModuleLevelLookups=True)[0] is ThisIsAClass) self.assertTrue( isinstance( roundtripConvert(ThisIsAClass(), vdm, allowUserCodeModuleLevelLookups=True)[0], ThisIsAClass))
def createInMemorySharedStateViewFactory(callbackSchedulerToUse = None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() sharedStateManager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride="", pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL, maxOpenFiles=100 ) sharedStateChannelFactory = ( InMemorySharedStateChannelFactory.InMemoryChannelFactory( callbackSchedulerToUse.getFactory().createScheduler("SharedState", 1), sharedStateManager ) ) return ViewFactory.ViewFactory(sharedStateChannelFactory)
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time import ufora.FORA.python.FORA as FORA import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface import ufora.native.CallbackScheduler as CallbackScheduler import ufora.test.PerformanceTestReporter as PerformanceTestReporter import ufora.FORA.python.Runtime as Runtime callbackScheduler = CallbackScheduler.singletonForTesting() class BigboxStringPerformanceTest(unittest.TestCase): def computeUsingSeveralWorkers(self, *args, **kwds): return InMemoryCumulusSimulation.computeUsingSeveralWorkers(*args, **kwds) def stringCreationAndSumTest(self, totalStrings, workers, threadsPerWorker, testName): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() #we wish we could actually test that we achieve saturation here but we can't yet. text = """Vector.range(%s, String).sum(size)""" % totalStrings t0 = time.time() _, simulation = \ self.computeUsingSeveralWorkers(
import logging import traceback import ufora.native.Cumulus as CumulusNative import ufora.FORA.python.FORA as FORA import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface import ufora.distributed.S3.ActualS3Interface as ActualS3Interface import ufora.native.CallbackScheduler as CallbackScheduler import ufora.test.PerformanceTestReporter as PerformanceTestReporter import ufora.FORA.python.Runtime as Runtime import uuid import os import ufora.distributed.Storage.S3ObjectStore as S3ObjectStore import ufora.config.Setup as Setup callbackScheduler = CallbackScheduler.singletonForTesting() class BigboxPerformanceTest(unittest.TestCase): def getTestDataBucket(self): aws_az_key = 'AWS_AVAILABILITY_ZONE' bucketName = 'ufora-test-data' if aws_az_key in os.environ: az = os.environ[aws_az_key] if az is not '': region = az[:-1] bucketName += '-' + region logging.info("Resolved az: %s, region: %s", az, region) else: logging.info("No availability zone resolved")
def computeUsingSeveralWorkers(expressionText, s3Service, count, objectStore=None, wantsStats=False, timeout=10, returnEverything=False, memoryLimitMb=100, blockUntilConnected=False, keepSimulationAlive=False, sharedStateViewFactory=None, threadCount=2): if keepSimulationAlive: assert returnEverything, \ "can't keep the simulation alive and not return it. how would you shut it down?" callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if sharedStateViewFactory is None: sharedStateViewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse = callbackSchedulerToUse ) workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = ( createWorkersAndClients( count, 1, sharedStateViewFactory, memoryLimitMb = memoryLimitMb, threadCount = threadCount ) ) client = clientsAndVdms[0][0] clientVdm = clientsAndVdms[0][1] loadingServices = [] for ix in range(len(workersVdmsAndEventHandlers)): worker = workersVdmsAndEventHandlers[ix][0] workerVdm = workersVdmsAndEventHandlers[ix][1] s3InterfaceFactory = s3Service.withMachine(ix) if objectStore is None: objectStore = S3ObjectStore.S3ObjectStore( s3InterfaceFactory, Setup.config().userDataS3Bucket, prefix="test/") loadingService = PythonIoTaskService.PythonIoTaskService( s3InterfaceFactory, objectStore, workerVdm, worker.getExternalDatasetRequestChannel(callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse) ) loadingService.startService() loadingServices.append(loadingService) if blockUntilConnected: for worker,vdm,eventHandler in workersVdmsAndEventHandlers: blockUntilWorkerIsConnected(worker, 2.0) if isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: computationDefinition = ( createComputationDefinition( FORA.extractImplValContainer( FORA.eval("fun() {" + expressionText + " } ") ), ForaNative.makeSymbol("Call") ) ) teardownGates = [] for client, vdm in clientsAndVdms: teardownGates.append(vdm.getVdmmTeardownGate()) for worker, vdm, eventHandler in workersVdmsAndEventHandlers: teardownGates.append(vdm.getVdmmTeardownGate()) simulationDict = { "result": None, "timedOut": None, "stats": None, "clientsAndVdms": clientsAndVdms, "workersVdmsAndEventHandlers": workersVdmsAndEventHandlers, "s3Service": s3Service, "loadingServices": loadingServices, "sharedStateViewFactory": sharedStateViewFactory, "client": client, "teardownGates": teardownGates } try: listener = client.createListener() computationSubmitTime = time.time() computationId = client.createComputation(computationDefinition) client.setComputationPriority( computationId, CumulusNative.ComputationPriority(1) ) if returnEverything: valAndStatsOrNone = waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=True) computationReturnTime = time.time() if valAndStatsOrNone is None: #we timed out val = None stats = None timedOut = True else: val, stats = valAndStatsOrNone timedOut = False simulationDict.update({ "result": val, "stats": stats, "timedOut": timedOut, "computationId": computationId, "listener": listener, "totalTimeToReturnResult": computationReturnTime - computationSubmitTime }) return simulationDict else: return waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=wantsStats) finally: if not keepSimulationAlive: teardownSimulation(simulationDict)
def __init__(self, workerCount, clientCount, memoryPerWorkerMB, threadsPerWorker, s3Service, objectStore=None, callbackScheduler=None, sharedStateViewFactory=None, ioTaskThreadOverride=None, useInMemoryCache=True, channelThroughputMBPerSecond=None, pageSizeOverride=None, disableEventHandler=False, machineIdHashSeed=None ): self.useInMemoryCache = useInMemoryCache self.machineIdHashSeed = machineIdHashSeed if not self.useInMemoryCache: self.diskCacheCount = 0 if os.getenv("CUMULUS_DATA_DIR") is None: self.diskCacheStorageDir = tempfile.mkdtemp() else: self.diskCacheStorageDir = os.path.join( os.getenv("CUMULUS_DATA_DIR"), str(uuid.uuid4()) ) self.ioTaskThreadOverride = ioTaskThreadOverride self.workerCount = 0 self.disableEventHandler = disableEventHandler self.clientCount = 0 self.memoryPerWorkerMB = memoryPerWorkerMB self.threadsPerWorker = threadsPerWorker self.s3Service = s3Service self.objectStore = objectStore if self.objectStore is None: s3 = s3Service() if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface): objectStoreBucket = "object_store_bucket" s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue') s3.deleteKey(objectStoreBucket, 'dummyKey') else: objectStoreBucket = Setup.config().userDataS3Bucket self.objectStore = S3ObjectStore.S3ObjectStore( s3Service, objectStoreBucket, prefix="test/") self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting() self.sharedStateViewFactory = ( sharedStateViewFactory or createInMemorySharedStateViewFactory(self.callbackScheduler) ) self.channelThroughputMBPerSecond = channelThroughputMBPerSecond self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler, 5 * 1024 * 1024) self.pageSizeOverride = pageSizeOverride self.rateLimitedChannelGroupsForEachListener = [] self.workersVdmsAndEventHandlers = [] self.machineIds = [] self.machineIdsEverAllocated = 0 self.clientsAndVdms = [] self.loadingServices = [] self.clientTeardownGates = [] self.workerTeardownGates = [] for ix in range(workerCount): self.addWorker() for ix in range(clientCount): self.addClient() if clientCount: self.listener = self.getClient(0).createListener() else: self.listener = None
def setUp(self): self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.compiler = self.runtime.getTypedForaCompiler() self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def setUp(self): self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vdm = FORANative.VectorDataManager( CallbackScheduler.singletonForTesting(), 10000000)
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting()
def computeUsingSeveralWorkers(expressionText, s3Service, count, objectStore=None, wantsStats=False, timeout=10, returnEverything=False, memoryLimitMb=100, blockUntilConnected=False, keepSimulationAlive=False, sharedStateViewFactory=None, threadCount=2): if keepSimulationAlive: assert returnEverything, \ "can't keep the simulation alive and not return it. how would you shut it down?" callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if sharedStateViewFactory is None: sharedStateViewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse=callbackSchedulerToUse) workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = ( createWorkersAndClients(count, 1, sharedStateViewFactory, memoryLimitMb=memoryLimitMb, threadCount=threadCount)) client = clientsAndVdms[0][0] clientVdm = clientsAndVdms[0][1] loadingServices = [] for ix in range(len(workersVdmsAndEventHandlers)): worker = workersVdmsAndEventHandlers[ix][0] workerVdm = workersVdmsAndEventHandlers[ix][1] s3InterfaceFactory = s3Service.withMachine(ix) if objectStore is None: objectStore = S3ObjectStore.S3ObjectStore( s3InterfaceFactory, Setup.config().userDataS3Bucket, prefix="test/") loadingService = PythonIoTaskService.PythonIoTaskService( s3InterfaceFactory, objectStore, workerVdm, worker.getExternalDatasetRequestChannel( callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse)) loadingService.startService() loadingServices.append(loadingService) if blockUntilConnected: for worker, vdm, eventHandler in workersVdmsAndEventHandlers: blockUntilWorkerIsConnected(worker, 2.0) if isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: computationDefinition = (createComputationDefinition( FORA.extractImplValContainer( FORA.eval("fun() {" + expressionText + " } ")), ForaNative.makeSymbol("Call"))) teardownGates = [] for client, vdm in clientsAndVdms: teardownGates.append(vdm.getVdmmTeardownGate()) for worker, vdm, eventHandler in workersVdmsAndEventHandlers: teardownGates.append(vdm.getVdmmTeardownGate()) simulationDict = { "result": None, "timedOut": None, "stats": None, "clientsAndVdms": clientsAndVdms, "workersVdmsAndEventHandlers": workersVdmsAndEventHandlers, "s3Service": s3Service, "loadingServices": loadingServices, "sharedStateViewFactory": sharedStateViewFactory, "client": client, "teardownGates": teardownGates } try: listener = client.createListener() computationSubmitTime = time.time() computationId = client.createComputation(computationDefinition) client.setComputationPriority(computationId, CumulusNative.ComputationPriority(1)) if returnEverything: valAndStatsOrNone = waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=True) computationReturnTime = time.time() if valAndStatsOrNone is None: #we timed out val = None stats = None timedOut = True else: val, stats = valAndStatsOrNone timedOut = False simulationDict.update({ "result": val, "stats": stats, "timedOut": timedOut, "computationId": computationId, "listener": listener, "totalTimeToReturnResult": computationReturnTime - computationSubmitTime }) return simulationDict else: return waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=wantsStats) finally: if not keepSimulationAlive: teardownSimulation(simulationDict)
def setUpClass(cls): cls.vdm = ForaNative.VectorDataManager( CallbackScheduler.singletonForTesting(), 10000000)