def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1) channelListener = MultiChannelListener(callbackScheduler, [args.base_port, args.base_port + 1]) sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('SharedState', 1), args.manager_address, int(args.manager_port) ) channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler) diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR") eventHandler = diagnostics_dir and createEventHandler( diagnostics_dir, callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1) ) own_address = args.own_address or get_own_ip() print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1 return CumulusService.CumulusService( own_address, channelListener, channelFactory, eventHandler, callbackScheduler, diagnostics_dir, Setup.config(), viewFactory=sharedStateViewFactory )
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-gateway', 1) channelListener = ChannelListener.SocketListener(args.port) return BackendGatewayService.BackendGatewayService(callbackScheduler, channelListener, args.store_address)
def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1) self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort #create an OutOfProcessDownloader so we can execute commands like 'forever' #from there, instead of forking from the main process (which can run out of memory) self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1) self.desirePublisher = None self._connectionManager = None
def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed): object.__init__(self) self.callbackScheduler = CallbackScheduler.singletonForTesting() self.callbackSchedulerFactory = self.callbackScheduler.getFactory() self.numRandVals = numRandVals self.numRelaxations = numRelaxations self.maxForRelax = maxForRelax self.maxForRand = maxForRand self.seed = seed self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.typed_fora_compiler = self.runtime.getTypedForaCompiler() if testAxiomsPath is not None: pathToUse = testAxiomsPath else: pathToUse = UNIT_TEST_AXIOMS_PATH self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse) self.axiom_groups = [] for i in range(self.axioms.axiomCount): self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i)) self.symbol_strings = self.loadSymbolStrings() numpy.random.seed(seed)
def createViewFactory(): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() return ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('fora-interpreter', 1), 'localhost', Setup.config().sharedStatePort )
def createWorker_(machineId, viewFactory, callbackSchedulerToUse, threadCount, memoryLimitMb, cacheFunction, pageSizeOverride, disableEventHandler): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager( callbackSchedulerToUse, pageSizeOverride if pageSizeOverride is not None else 1 * 1024 * 1024 if memoryLimitMb < 1000 else 5 * 1024 * 1024 if memoryLimitMb < 5000 else 50 * 1024 * 1024 ) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = cacheFunction() if disableEventHandler: eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler() else: eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def createClient_(clientId, callbackSchedulerToUse = None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit(100 * 1024 * 1024, 125 * 1024 * 1024) return ( CumulusNative.CumulusClient(vdm, clientId, callbackSchedulerToUse), vdm )
def handle(self): logging.info("all file descriptors closed") callbackScheduler = CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler( "BackendGatewayService", 1 ) channel = SocketStringChannel.SocketStringChannel(callbackScheduler, self.sock).makeQueuelike(callbackScheduler) logging.info("channel connected") self.serviceRequest(channel) logging.info("Killing self!")
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.compiler = self.runtime.getTypedForaCompiler() self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_) pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython") self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_) self.instructionGraph = self.runtime.getInstructionGraph() self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1) channelListener = MultiChannelListener(callbackScheduler, [args.base_port, args.base_port + 1]) sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('SharedState', 1), args.manager_address, int(args.manager_port) ) channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler) diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR") eventHandler = diagnostics_dir and createEventHandler( diagnostics_dir, callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1) ) own_address = args.own_address or get_own_ip() print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1 config = Setup.config() print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % ( config.cumulusVectorRamCacheMB, config.cumulusMaxRamCacheMB, config.cumulusServiceThreadCount, config.cumulusTrackTcmalloc ) print "Ufora store at %s:%s" % (args.manager_address, args.manager_port) s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory() print "PythonIoTasks threads: %d. Out of process: %s" % ( config.externalDatasetLoaderServiceThreads, s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool ) return CumulusService.CumulusService( own_address, channelListener, channelFactory, eventHandler, callbackScheduler, diagnostics_dir, Setup.config(), viewFactory=sharedStateViewFactory, s3InterfaceFactory=s3InterfaceFactory, objectStore=NullObjectStore.NullObjectStore() )
def createWorkersAndClients( workerCount, clientCount, viewFactory = None, memoryLimitMb = 100, threadCount = 2, callbackSchedulerToUse = None ): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if viewFactory is None: viewFactory = createInMemorySharedStateViewFactory(callbackSchedulerToUse) workersVdmsAndEventHandlers = [ createWorker( machineId(ix), viewFactory, memoryLimitMb = memoryLimitMb, threadCount=threadCount, callbackSchedulerToUse = callbackSchedulerToUse ) for ix in range(workerCount) ] clientsAndVdms = [ createClient( clientId(ix), callbackSchedulerToUse = callbackSchedulerToUse ) for ix in range(clientCount) ] for ix1 in range(len(workersVdmsAndEventHandlers)): workersVdmsAndEventHandlers[ix1][0].startComputations() for ix1 in range(len(workersVdmsAndEventHandlers)-1): for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addMachine(machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) workersVdmsAndEventHandlers[ix2][0].addMachine(machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) for ix1 in range(len(workersVdmsAndEventHandlers)): for ix2 in range(len(clientsAndVdms)): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() def createStorage(vdm): self.simpleOfflineCache = CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000) return self.simpleOfflineCache self.evaluator = LocalEvaluator.LocalEvaluator( createStorage, 2000000, maxPageSizeInBytes = 100000 ) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
def __init__(self, socketFd, sharedStateAddress): self.socketFd = socketFd self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.scheduler = self.callbackSchedulerFactory.createScheduler("BackendGatewayRequestHandler", 1) sharedStateHost, sharedStatePort = sharedStateAddress.split(":") sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( self.callbackSchedulerFactory.createScheduler("SharedState", 1), sharedStateHost, int(sharedStatePort) ) self.subscribableHandler = ConnectionHandler.ConnectionHandler( self.scheduler, sharedStateViewFactory, lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler) ) self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def __init__(self, inMemory, port = None, cachePathOverride = '', maxOpenFiles = 256, inMemChannelFactoryFactory = None, maxLogFileSizeMb = 10, pingInterval = None): self.inMemory = inMemory self.manager = None self.callbackScheduler = CallbackScheduler.singletonForTesting() if self.inMemory: self.manager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride=cachePathOverride, pingInterval = IN_MEMORY_HARNESS_PING_INTERVAL if pingInterval is None else pingInterval, maxOpenFiles=maxOpenFiles, maxLogFileSizeMb=maxLogFileSizeMb ) #although named otherwise InMemoryChannelFactory is actually a factory for a channelFactory # or a channelFactoryFactory channelFactoryFactory = inMemChannelFactoryFactory if inMemChannelFactoryFactory is not None \ else InMemoryChannelFactory.InMemoryChannelFactory logging.info(channelFactoryFactory) self.channelFactory = channelFactoryFactory(self.callbackScheduler, self.manager) self.viewFactory = ViewFactory.ViewFactory(self.channelFactory) else: class Settings(object): callbackScheduler = self.callbackScheduler assert port is not None self.service = SharedStateService.SharedStateService( self.callbackScheduler, cachePathOverride=cachePathOverride, port=port ) self.service.startService() self.service.blockUntilListening() self.viewFactory = ViewFactory.ViewFactory.TcpViewFactory(self.callbackScheduler, "localhost", port)
def __init__(self, cumulusVectorRamCacheSizeOverride = DEFAULT_VECTOR_RAM_CACHE_SIZE, cumulusMaxRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE, cumulusThreadCountOverride = DEFAULT_THREAD_COUNT, remoteGatewayRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE, perMachineThroughput = DEFAULT_PER_MACHINE_THROUGHPUT ): self.cumulusMaxRamCacheSizeOverride = cumulusMaxRamCacheSizeOverride self.cumulusVectorRamCacheSizeOverride = cumulusVectorRamCacheSizeOverride self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = self.callbackSchedulerFactory.createScheduler("InMemoryCluster", 1) self.cumulusThreadCountOverride = cumulusThreadCountOverride self.remoteGatewayCacheSize = remoteGatewayRamCacheSizeOverride self.sharedStateManager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride="", pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL, maxOpenFiles=100 ) self.sharedStateChannelFactory = ( InMemorySharedStateChannelFactory.InMemoryChannelFactory( self.callbackScheduler, self.sharedStateManager ) ) self.sharedStateViewFactory = ViewFactory.ViewFactory(self.sharedStateChannelFactory) self.client = InMemoryClient(self) self.cumuli = [] self.nextCumulusAddress = 0 self.channelManager = InMemoryChannelManager(self.callbackScheduler, perMachineThroughput) self.inMemoryDemuxingChannel = \ InMemorySharedStateChannelFactory.SerializedToManagerChannelFactory( self.callbackScheduler, self.sharedStateManager, "SharedState" )
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt")) self.evaluator = LocalEvaluator.LocalEvaluator( lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000), 10000000, maxPageSizeInBytes = 100000 ) self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator) self.knownModulesAsConstantJOVs = dict() self.knownModulesAsConstantJOVs["builtin"] = \ FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def createInMemorySharedStateViewFactory(callbackSchedulerToUse = None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() sharedStateManager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride="", pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL, maxOpenFiles=100 ) sharedStateChannelFactory = ( InMemorySharedStateChannelFactory.InMemoryChannelFactory( callbackSchedulerToUse.getFactory().createScheduler("SharedState", 1), sharedStateManager ) ) return ViewFactory.ViewFactory(sharedStateChannelFactory)
def initialize(setupObjectToUse=None, useLocalEvaluator=True, vdmOverride=None): global _evaluator if _evaluator is not None: return import ufora.FORA.python.Evaluator.LocalEvaluator as LocalEvaluator import ufora.FORA.python.Evaluator.CumulusEvaluator as CumulusEvaluator if setupObjectToUse is None: configToUse = Setup.config() else: configToUse = setupObjectToUse.config if useLocalEvaluator: _evaluator = LocalEvaluator.defaultLocalEvaluator(vdmOverride=vdmOverride) else: import ufora.native.CallbackScheduler as CallbackSchedulerNative schedulerFactory = CallbackSchedulerNative.createSimpleCallbackSchedulerFactory() _evaluator = CumulusEvaluator.CumulusEvaluator( schedulerFactory.createScheduler("CumulusEvaluator", 1) )
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024) vdm.setMemoryLimit( int(memoryLimitMb * 1024 * 1024), min(int(memoryLimitMb * 1.25 * 1024 * 1024), int((memoryLimitMb + 1024 * 2) * 1024 * 1024)) ) vdm.setPersistentCacheIndex( CumulusNative.PersistentCacheIndex( viewFactory.createView(), callbackSchedulerToUse ) ) cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024) eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler() return ( CumulusNative.CumulusWorker( callbackSchedulerToUse, CumulusNative.CumulusWorkerConfiguration( machineId, threadCount, CumulusNative.CumulusCheckpointPolicy.None(), ExecutionContext.createContextConfiguration(), "" ), vdm, cache, eventHandler ), vdm, eventHandler )
def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1) self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort self.desirePublisher = None self._connectionManager = None
def __init__(self, offlineCacheFunction, newMemLimit, remoteEvaluator=None, newLoadRatio=.5, maxPageSizeInBytes=None, vdmOverride=None): if maxPageSizeInBytes is None: maxPageSizeInBytes = Setup.config().maxPageSizeInBytes if vdmOverride is not None: self.vdm_ = vdmOverride self.offlineCache_ = None else: self.vdm_ = FORANative.VectorDataManager( CallbackSchedulerNative.createSimpleCallbackSchedulerFactory() .createScheduler("LocalEvaluator", 1), maxPageSizeInBytes ) self.vdm_.setDropUnreferencedPagesWhenFull(True) self.offlineCache_ = offlineCacheFunction(self.vdm_) if self.offlineCache_ is not None: self.vdm_.setOfflineCache(self.offlineCache_) logging.info("LocalEvaluator Creating VDMC with %s MB", newMemLimit / 1024.0 / 1024.0) self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25)) self.vdm_.setLoadRatio(newLoadRatio) self.remoteEvaluator_ = remoteEvaluator self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
def __init__(self, workerCount, clientCount, memoryPerWorkerMB, threadsPerWorker, s3Service, objectStore=None, callbackScheduler=None, sharedStateViewFactory=None, ioTaskThreadOverride=None, useInMemoryCache=True, channelThroughputMBPerSecond=None, pageSizeOverride=None, disableEventHandler=False, machineIdHashSeed=None ): self.useInMemoryCache = useInMemoryCache self.machineIdHashSeed = machineIdHashSeed if not self.useInMemoryCache: self.diskCacheCount = 0 if os.getenv("CUMULUS_DATA_DIR") is None: self.diskCacheStorageDir = tempfile.mkdtemp() else: self.diskCacheStorageDir = os.path.join( os.getenv("CUMULUS_DATA_DIR"), str(uuid.uuid4()) ) self.ioTaskThreadOverride = ioTaskThreadOverride self.workerCount = 0 self.disableEventHandler = disableEventHandler self.clientCount = 0 self.memoryPerWorkerMB = memoryPerWorkerMB self.threadsPerWorker = threadsPerWorker self.s3Service = s3Service self.objectStore = objectStore if self.objectStore is None: s3 = s3Service() if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface): objectStoreBucket = "object_store_bucket" s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue') s3.deleteKey(objectStoreBucket, 'dummyKey') else: objectStoreBucket = Setup.config().userDataS3Bucket self.objectStore = S3ObjectStore.S3ObjectStore( s3Service, objectStoreBucket, prefix="test/") self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting() self.sharedStateViewFactory = ( sharedStateViewFactory or createInMemorySharedStateViewFactory(self.callbackScheduler) ) self.channelThroughputMBPerSecond = channelThroughputMBPerSecond self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler, 5 * 1024 * 1024) self.pageSizeOverride = pageSizeOverride self.rateLimitedChannelGroupsForEachListener = [] self.workersVdmsAndEventHandlers = [] self.machineIds = [] self.machineIdsEverAllocated = 0 self.clientsAndVdms = [] self.loadingServices = [] self.clientTeardownGates = [] self.workerTeardownGates = [] for ix in range(workerCount): self.addWorker() for ix in range(clientCount): self.addClient() if clientCount: self.listener = self.getClient(0).createListener() else: self.listener = None
def setUp(self): self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.native_runtime = self.runtime.getTypedForaCompiler() self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
def __init__(self, workerCount, clientCount, memoryPerWorkerMB, threadsPerWorker, s3Service, objectStore=None, callbackScheduler=None, sharedStateViewFactory=None, ioTaskThreadOverride=None, useInMemoryCache=True, channelThroughputMBPerSecond=None, pageSizeOverride=None, disableEventHandler=False, machineIdHashSeed=None): self.useInMemoryCache = useInMemoryCache self.machineIdHashSeed = machineIdHashSeed if not self.useInMemoryCache: self.diskCacheCount = 0 if os.getenv("CUMULUS_DATA_DIR") is None: self.diskCacheStorageDir = tempfile.mkdtemp() else: self.diskCacheStorageDir = os.path.join( os.getenv("CUMULUS_DATA_DIR"), str(uuid.uuid4())) self.ioTaskThreadOverride = ioTaskThreadOverride self.workerCount = 0 self.disableEventHandler = disableEventHandler self.clientCount = 0 self.memoryPerWorkerMB = memoryPerWorkerMB self.threadsPerWorker = threadsPerWorker self.s3Service = s3Service self.objectStore = objectStore if self.objectStore is None: s3 = s3Service() if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface): objectStoreBucket = "object_store_bucket" s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue') s3.deleteKey(objectStoreBucket, 'dummyKey') else: objectStoreBucket = Setup.config().userDataS3Bucket self.objectStore = S3ObjectStore.S3ObjectStore(s3Service, objectStoreBucket, prefix="test/") self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting( ) self.sharedStateViewFactory = (sharedStateViewFactory or createInMemorySharedStateViewFactory( self.callbackScheduler)) self.channelThroughputMBPerSecond = channelThroughputMBPerSecond self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler, 5 * 1024 * 1024) self.pageSizeOverride = pageSizeOverride self.rateLimitedChannelGroupsForEachListener = [] self.workersVdmsAndEventHandlers = [] self.machineIds = [] self.machineIdsEverAllocated = 0 self.clientsAndVdms = [] self.loadingServices = [] self.clientTeardownGates = [] self.workerTeardownGates = [] for ix in range(workerCount): self.addWorker() for ix in range(clientCount): self.addClient() if clientCount: self.listener = self.getClient(0).createListener() else: self.listener = None
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting()
def setUp(self): self.callbackScheduler = CallbackScheduler.singletonForTesting() self.runtime = Runtime.getMainRuntime() self.axioms = self.runtime.getAxioms() self.compiler = self.runtime.getTypedForaCompiler() self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
import sys import argparse import signal import logging import ufora.config.Setup as Setup import ufora.distributed.SharedState.SharedStateService as SharedStateService import ufora.native.CallbackScheduler as CallbackScheduler import ufora.config.Mainline as Mainline if __name__ == "__main__": callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler("sharedStateScheduler", 1) parser = argparse.ArgumentParser() parser.add_argument('--cacheDir') parsed, remaining = parser.parse_known_args(sys.argv[1:]) def startSharedState(args): if parsed.cacheDir: logging.info("Shared state cache directory: %s", parsed.cacheDir) Setup.config().sharedStateCache = parsed.cacheDir service = SharedStateService.SharedStateService(callbackScheduler)
def setUpClass(cls): cls.vdm = ForaNative.VectorDataManager( CallbackScheduler.singletonForTesting(), 10000000)
def getCallbackSchedulerFactory(): global _callbackSchedulerFactory if _callbackSchedulerFactory is None: _callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() return _callbackSchedulerFactory
def computeUsingSeveralWorkers(expressionText, s3Service, count, objectStore=None, wantsStats=False, timeout=10, returnEverything=False, memoryLimitMb=100, blockUntilConnected=False, keepSimulationAlive=False, sharedStateViewFactory=None, threadCount=2): if keepSimulationAlive: assert returnEverything, \ "can't keep the simulation alive and not return it. how would you shut it down?" callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if sharedStateViewFactory is None: sharedStateViewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse = callbackSchedulerToUse ) workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = ( createWorkersAndClients( count, 1, sharedStateViewFactory, memoryLimitMb = memoryLimitMb, threadCount = threadCount ) ) client = clientsAndVdms[0][0] clientVdm = clientsAndVdms[0][1] loadingServices = [] for ix in range(len(workersVdmsAndEventHandlers)): worker = workersVdmsAndEventHandlers[ix][0] workerVdm = workersVdmsAndEventHandlers[ix][1] s3InterfaceFactory = s3Service.withMachine(ix) if objectStore is None: objectStore = S3ObjectStore.S3ObjectStore( s3InterfaceFactory, Setup.config().userDataS3Bucket, prefix="test/") loadingService = PythonIoTaskService.PythonIoTaskService( s3InterfaceFactory, objectStore, workerVdm, worker.getExternalDatasetRequestChannel(callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse) ) loadingService.startService() loadingServices.append(loadingService) if blockUntilConnected: for worker,vdm,eventHandler in workersVdmsAndEventHandlers: blockUntilWorkerIsConnected(worker, 2.0) if isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: computationDefinition = ( createComputationDefinition( FORA.extractImplValContainer( FORA.eval("fun() {" + expressionText + " } ") ), ForaNative.makeSymbol("Call") ) ) teardownGates = [] for client, vdm in clientsAndVdms: teardownGates.append(vdm.getVdmmTeardownGate()) for worker, vdm, eventHandler in workersVdmsAndEventHandlers: teardownGates.append(vdm.getVdmmTeardownGate()) simulationDict = { "result": None, "timedOut": None, "stats": None, "clientsAndVdms": clientsAndVdms, "workersVdmsAndEventHandlers": workersVdmsAndEventHandlers, "s3Service": s3Service, "loadingServices": loadingServices, "sharedStateViewFactory": sharedStateViewFactory, "client": client, "teardownGates": teardownGates } try: listener = client.createListener() computationSubmitTime = time.time() computationId = client.createComputation(computationDefinition) client.setComputationPriority( computationId, CumulusNative.ComputationPriority(1) ) if returnEverything: valAndStatsOrNone = waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=True) computationReturnTime = time.time() if valAndStatsOrNone is None: #we timed out val = None stats = None timedOut = True else: val, stats = valAndStatsOrNone timedOut = False simulationDict.update({ "result": val, "stats": stats, "timedOut": timedOut, "computationId": computationId, "listener": listener, "totalTimeToReturnResult": computationReturnTime - computationSubmitTime }) return simulationDict else: return waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=wantsStats) finally: if not keepSimulationAlive: teardownSimulation(simulationDict)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time import ufora.FORA.python.FORA as FORA import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface import ufora.native.CallbackScheduler as CallbackScheduler import ufora.test.PerformanceTestReporter as PerformanceTestReporter import ufora.FORA.python.Runtime as Runtime callbackScheduler = CallbackScheduler.singletonForTesting() class BigboxStringPerformanceTest(unittest.TestCase): def computeUsingSeveralWorkers(self, *args, **kwds): return InMemoryCumulusSimulation.computeUsingSeveralWorkers(*args, **kwds) def stringCreationAndSumTest(self, totalStrings, workers, threadsPerWorker, testName): s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory() #we wish we could actually test that we achieve saturation here but we can't yet. text = """Vector.range(%s, String).sum(size)""" % totalStrings t0 = time.time() _, simulation = \ self.computeUsingSeveralWorkers(
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface import ufora.native.CallbackScheduler as CallbackScheduler import ufora.test.PerformanceTestReporter as PerformanceTestReporter callbackScheduler = CallbackScheduler.singletonForTesting() class MultimachineLinearRegressionSimulationTest(unittest.TestCase): def dataGenerationScript(self, mbOfData, columns): valueCount = mbOfData * 1024 * 1024 / 8 rowCount = valueCount / columns return (""" let data = Vector.range(__columns__) ~~ fun(c) { Vector.range(__rows__, fun(r) { if (c == 0) 1.0 else Float64(r % (c+2)) }) }; let dfResponse = dataframe.DataFrame(data[-1,]); let dfPredictors = dataframe.DataFrame(data[,-1]);