def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) self.callbackScheduler = callbackSchedulerFactory.createScheduler( "Simulator", 1) self.uforaPath = os.path.abspath( os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join( self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config( ).subscribableWebObjectsPort #create an OutOfProcessDownloader so we can execute commands like 'forever' #from there, instead of forking from the main process (which can run out of memory) self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1) self.desirePublisher = None self._connectionManager = None
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1) channelListener = MultiChannelListener(callbackScheduler, [args.base_port, args.base_port + 1]) sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('SharedState', 1), args.manager_address, int(args.manager_port) ) channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler) diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR") eventHandler = diagnostics_dir and createEventHandler( diagnostics_dir, callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1) ) own_address = args.own_address or get_own_ip() print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1 return CumulusService.CumulusService( own_address, channelListener, channelFactory, eventHandler, callbackScheduler, diagnostics_dir, Setup.config(), viewFactory=sharedStateViewFactory )
def createViewFactory(): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() return ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('fora-interpreter', 1), 'localhost', Setup.config().sharedStatePort )
def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1) self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort #create an OutOfProcessDownloader so we can execute commands like 'forever' #from there, instead of forking from the main process (which can run out of memory) self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1) self.desirePublisher = None self._connectionManager = None
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-gateway', 1) channelListener = ChannelListener.SocketListener(args.port) return BackendGatewayService.BackendGatewayService(callbackScheduler, channelListener, args.store_address)
def __init__(self, offlineCacheFunction, newMemLimit, remoteEvaluator=None, newLoadRatio=.5, maxPageSizeInBytes=None, vdmOverride=None): if maxPageSizeInBytes is None: maxPageSizeInBytes = Setup.config().maxPageSizeInBytes if vdmOverride is not None: self.vdm_ = vdmOverride self.offlineCache_ = None else: self.vdm_ = FORANative.VectorDataManager( CallbackSchedulerNative.createSimpleCallbackSchedulerFactory(). createScheduler("LocalEvaluator", 1), maxPageSizeInBytes) self.vdm_.setDropUnreferencedPagesWhenFull(True) self.offlineCache_ = offlineCacheFunction(self.vdm_) if self.offlineCache_ is not None: self.vdm_.setOfflineCache(self.offlineCache_) logging.info("LocalEvaluator Creating VDMC with %s MB", newMemLimit / 1024.0 / 1024.0) self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25)) self.vdm_.setLoadRatio(newLoadRatio) self.remoteEvaluator_ = remoteEvaluator self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) self.callbackScheduler = callbackSchedulerFactory.createScheduler( "Simulator", 1) self.uforaPath = os.path.abspath( os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join( self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config( ).subscribableWebObjectsPort self.desirePublisher = None self._connectionManager = None
def createViewFactory(): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) return ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('fora-interpreter', 1), 'localhost', Setup.config().sharedStatePort)
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) callbackScheduler = callbackSchedulerFactory.createScheduler( 'ufora-gateway', 1) channelListener = ChannelListener.SocketListener(args.port) return BackendGatewayService.BackendGatewayService(callbackScheduler, channelListener, args.store_address)
def handle(self): logging.info("all file descriptors closed") callbackScheduler = CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler( "BackendGatewayService", 1 ) channel = SocketStringChannel.SocketStringChannel(callbackScheduler, self.sock).makeQueuelike(callbackScheduler) logging.info("channel connected") self.serviceRequest(channel) logging.info("Killing self!")
def createService(args): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1) channelListener = MultiChannelListener(callbackScheduler, [args.base_port, args.base_port + 1]) sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( callbackSchedulerFactory.createScheduler('SharedState', 1), args.manager_address, int(args.manager_port) ) channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler) diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR") eventHandler = diagnostics_dir and createEventHandler( diagnostics_dir, callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1) ) own_address = args.own_address or get_own_ip() print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1 config = Setup.config() print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % ( config.cumulusVectorRamCacheMB, config.cumulusMaxRamCacheMB, config.cumulusServiceThreadCount, config.cumulusTrackTcmalloc ) print "Ufora store at %s:%s" % (args.manager_address, args.manager_port) s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory() print "PythonIoTasks threads: %d. Out of process: %s" % ( config.externalDatasetLoaderServiceThreads, s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool ) return CumulusService.CumulusService( own_address, channelListener, channelFactory, eventHandler, callbackScheduler, diagnostics_dir, Setup.config(), viewFactory=sharedStateViewFactory, s3InterfaceFactory=s3InterfaceFactory, objectStore=NullObjectStore.NullObjectStore() )
def __init__(self, socketFd, sharedStateAddress): self.socketFd = socketFd self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.scheduler = self.callbackSchedulerFactory.createScheduler("BackendGatewayRequestHandler", 1) sharedStateHost, sharedStatePort = sharedStateAddress.split(":") sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( self.callbackSchedulerFactory.createScheduler("SharedState", 1), sharedStateHost, int(sharedStatePort) ) self.subscribableHandler = ConnectionHandler.ConnectionHandler( self.scheduler, sharedStateViewFactory, lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler) ) self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def handle(self): logging.info("all file descriptors closed") callbackScheduler = \ CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler( "BackendGatewayService", 1 ) channel = SocketStringChannel.SocketStringChannel( callbackScheduler, self.sock).makeQueuelike(callbackScheduler) logging.info("channel connected") self.serviceRequest(channel) logging.info("Killing self!")
def __init__(self, cumulusVectorRamCacheSizeOverride = DEFAULT_VECTOR_RAM_CACHE_SIZE, cumulusMaxRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE, cumulusThreadCountOverride = DEFAULT_THREAD_COUNT, remoteGatewayRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE, perMachineThroughput = DEFAULT_PER_MACHINE_THROUGHPUT ): self.cumulusMaxRamCacheSizeOverride = cumulusMaxRamCacheSizeOverride self.cumulusVectorRamCacheSizeOverride = cumulusVectorRamCacheSizeOverride self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = self.callbackSchedulerFactory.createScheduler("InMemoryCluster", 1) self.cumulusThreadCountOverride = cumulusThreadCountOverride self.remoteGatewayCacheSize = remoteGatewayRamCacheSizeOverride self.sharedStateManager = SharedStateService.KeyspaceManager( 10001, 1, cachePathOverride="", pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL, maxOpenFiles=100 ) self.sharedStateChannelFactory = ( InMemorySharedStateChannelFactory.InMemoryChannelFactory( self.callbackScheduler, self.sharedStateManager ) ) self.sharedStateViewFactory = ViewFactory.ViewFactory(self.sharedStateChannelFactory) self.client = InMemoryClient(self) self.cumuli = [] self.nextCumulusAddress = 0 self.channelManager = InMemoryChannelManager(self.callbackScheduler, perMachineThroughput) self.inMemoryDemuxingChannel = \ InMemorySharedStateChannelFactory.SerializedToManagerChannelFactory( self.callbackScheduler, self.sharedStateManager, "SharedState" )
def initialize(setupObjectToUse=None, useLocalEvaluator=True, vdmOverride=None): global _evaluator if _evaluator is not None: return import ufora.FORA.python.Evaluator.LocalEvaluator as LocalEvaluator import ufora.FORA.python.Evaluator.CumulusEvaluator as CumulusEvaluator if setupObjectToUse is None: configToUse = Setup.config() else: configToUse = setupObjectToUse.config if useLocalEvaluator: _evaluator = LocalEvaluator.defaultLocalEvaluator(vdmOverride=vdmOverride) else: import ufora.native.CallbackScheduler as CallbackSchedulerNative schedulerFactory = CallbackSchedulerNative.createSimpleCallbackSchedulerFactory() _evaluator = CumulusEvaluator.CumulusEvaluator( schedulerFactory.createScheduler("CumulusEvaluator", 1) )
def __init__(self): callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1) self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState') self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py') self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py') self.webPath = os.path.join(self.uforaPath, 'web/relay') self.relayScript = os.path.join(self.webPath, 'server.coffee') self.relayPort = Setup.config().relayPort self.relayHttpsPort = Setup.config().relayHttpsPort self.sharedStatePort = Setup.config().sharedStatePort self.restApiPort = Setup.config().restApiPort self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort self.desirePublisher = None self._connectionManager = None
def __init__(self, socketFd, sharedStateAddress): self.socketFd = socketFd self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() self.scheduler = self.callbackSchedulerFactory.createScheduler( "BackendGatewayRequestHandler", 1 ) sharedStateHost, sharedStatePort = sharedStateAddress.split(':') sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory( self.callbackSchedulerFactory.createScheduler('SharedState', 1), sharedStateHost, int(sharedStatePort) ) self.subscribableHandler = ConnectionHandler.ConnectionHandler( self.scheduler, sharedStateViewFactory, lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler) ) self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def __init__(self, offlineCacheFunction, newMemLimit, remoteEvaluator=None, newLoadRatio=.5, maxPageSizeInBytes=None, vdmOverride=None): if maxPageSizeInBytes is None: maxPageSizeInBytes = Setup.config().maxPageSizeInBytes if vdmOverride is not None: self.vdm_ = vdmOverride self.offlineCache_ = None else: self.vdm_ = FORANative.VectorDataManager( CallbackSchedulerNative.createSimpleCallbackSchedulerFactory() .createScheduler("LocalEvaluator", 1), maxPageSizeInBytes ) self.vdm_.setDropUnreferencedPagesWhenFull(True) self.offlineCache_ = offlineCacheFunction(self.vdm_) if self.offlineCache_ is not None: self.vdm_.setOfflineCache(self.offlineCache_) logging.info("LocalEvaluator Creating VDMC with %s MB", newMemLimit / 1024.0 / 1024.0) self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25)) self.vdm_.setLoadRatio(newLoadRatio) self.remoteEvaluator_ = remoteEvaluator self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
import sys import argparse import signal import logging import ufora.config.Setup as Setup import ufora.distributed.SharedState.SharedStateService as SharedStateService import ufora.native.CallbackScheduler as CallbackScheduler import ufora.config.Mainline as Mainline if __name__ == "__main__": callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() callbackScheduler = callbackSchedulerFactory.createScheduler("sharedStateScheduler", 1) parser = argparse.ArgumentParser() parser.add_argument('--cacheDir') parsed, remaining = parser.parse_known_args(sys.argv[1:]) def startSharedState(args): if parsed.cacheDir: logging.info("Shared state cache directory: %s", parsed.cacheDir) Setup.config().sharedStateCache = parsed.cacheDir service = SharedStateService.SharedStateService(callbackScheduler)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import argparse import signal import logging import ufora.config.Setup as Setup import ufora.distributed.SharedState.SharedStateService as SharedStateService import ufora.native.CallbackScheduler as CallbackScheduler import ufora.config.Mainline as Mainline if __name__ == "__main__": callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) callbackScheduler = callbackSchedulerFactory.createScheduler( "sharedStateScheduler", 1) parser = argparse.ArgumentParser() parser.add_argument('--cacheDir') parsed, remaining = parser.parse_known_args(sys.argv[1:]) def startSharedState(args): if parsed.cacheDir: logging.info("Shared state cache directory: %s", parsed.cacheDir) Setup.config().sharedStateCache = parsed.cacheDir service = SharedStateService.SharedStateService(callbackScheduler) service.startService() service.blockUntilListening()
def getCallbackSchedulerFactory(): global _callbackSchedulerFactory if _callbackSchedulerFactory is None: _callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory() return _callbackSchedulerFactory
def getCallbackSchedulerFactory(): global _callbackSchedulerFactory if _callbackSchedulerFactory is None: _callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory( ) return _callbackSchedulerFactory