def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
self.callbackScheduler = callbackSchedulerFactory.createScheduler(
"Simulator", 1)
self.uforaPath = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath,
'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath,
'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(
self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config(
).subscribableWebObjectsPort
#create an OutOfProcessDownloader so we can execute commands like 'forever'
#from there, instead of forking from the main process (which can run out of memory)
self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1)
self.desirePublisher = None
self._connectionManager = None
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory
)
def createViewFactory():
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
return ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('fora-interpreter', 1),
'localhost',
Setup.config().sharedStatePort
)
def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1)
self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort
#create an OutOfProcessDownloader so we can execute commands like 'forever'
#from there, instead of forking from the main process (which can run out of memory)
self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1)
self.desirePublisher = None
self._connectionManager = None
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-gateway', 1)
channelListener = ChannelListener.SocketListener(args.port)
return BackendGatewayService.BackendGatewayService(callbackScheduler,
channelListener,
args.store_address)
def __init__(self,
offlineCacheFunction,
newMemLimit,
remoteEvaluator=None,
newLoadRatio=.5,
maxPageSizeInBytes=None,
vdmOverride=None):
if maxPageSizeInBytes is None:
maxPageSizeInBytes = Setup.config().maxPageSizeInBytes
if vdmOverride is not None:
self.vdm_ = vdmOverride
self.offlineCache_ = None
else:
self.vdm_ = FORANative.VectorDataManager(
CallbackSchedulerNative.createSimpleCallbackSchedulerFactory().
createScheduler("LocalEvaluator", 1), maxPageSizeInBytes)
self.vdm_.setDropUnreferencedPagesWhenFull(True)
self.offlineCache_ = offlineCacheFunction(self.vdm_)
if self.offlineCache_ is not None:
self.vdm_.setOfflineCache(self.offlineCache_)
logging.info("LocalEvaluator Creating VDMC with %s MB",
newMemLimit / 1024.0 / 1024.0)
self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25))
self.vdm_.setLoadRatio(newLoadRatio)
self.remoteEvaluator_ = remoteEvaluator
self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
self.callbackScheduler = callbackSchedulerFactory.createScheduler(
"Simulator", 1)
self.uforaPath = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath,
'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath,
'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(
self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config(
).subscribableWebObjectsPort
self.desirePublisher = None
self._connectionManager = None
def createViewFactory():
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
return ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('fora-interpreter', 1),
'localhost',
Setup.config().sharedStatePort)
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
callbackScheduler = callbackSchedulerFactory.createScheduler(
'ufora-gateway', 1)
channelListener = ChannelListener.SocketListener(args.port)
return BackendGatewayService.BackendGatewayService(callbackScheduler,
channelListener,
args.store_address)
def handle(self):
logging.info("all file descriptors closed")
callbackScheduler = CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler(
"BackendGatewayService", 1
)
channel = SocketStringChannel.SocketStringChannel(callbackScheduler, self.sock).makeQueuelike(callbackScheduler)
logging.info("channel connected")
self.serviceRequest(channel)
logging.info("Killing self!")
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
config = Setup.config()
print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % (
config.cumulusVectorRamCacheMB,
config.cumulusMaxRamCacheMB,
config.cumulusServiceThreadCount,
config.cumulusTrackTcmalloc
)
print "Ufora store at %s:%s" % (args.manager_address, args.manager_port)
s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory()
print "PythonIoTasks threads: %d. Out of process: %s" % (
config.externalDatasetLoaderServiceThreads,
s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
)
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory,
s3InterfaceFactory=s3InterfaceFactory,
objectStore=NullObjectStore.NullObjectStore()
)
def __init__(self, socketFd, sharedStateAddress):
self.socketFd = socketFd
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.scheduler = self.callbackSchedulerFactory.createScheduler("BackendGatewayRequestHandler", 1)
sharedStateHost, sharedStatePort = sharedStateAddress.split(":")
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
self.callbackSchedulerFactory.createScheduler("SharedState", 1), sharedStateHost, int(sharedStatePort)
)
self.subscribableHandler = ConnectionHandler.ConnectionHandler(
self.scheduler, sharedStateViewFactory, lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler)
)
self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def handle(self):
logging.info("all file descriptors closed")
callbackScheduler = \
CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler(
"BackendGatewayService",
1
)
channel = SocketStringChannel.SocketStringChannel(
callbackScheduler,
self.sock).makeQueuelike(callbackScheduler)
logging.info("channel connected")
self.serviceRequest(channel)
logging.info("Killing self!")
def __init__(self,
cumulusVectorRamCacheSizeOverride = DEFAULT_VECTOR_RAM_CACHE_SIZE,
cumulusMaxRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
cumulusThreadCountOverride = DEFAULT_THREAD_COUNT,
remoteGatewayRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
perMachineThroughput = DEFAULT_PER_MACHINE_THROUGHPUT
):
self.cumulusMaxRamCacheSizeOverride = cumulusMaxRamCacheSizeOverride
self.cumulusVectorRamCacheSizeOverride = cumulusVectorRamCacheSizeOverride
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = self.callbackSchedulerFactory.createScheduler("InMemoryCluster", 1)
self.cumulusThreadCountOverride = cumulusThreadCountOverride
self.remoteGatewayCacheSize = remoteGatewayRamCacheSizeOverride
self.sharedStateManager = SharedStateService.KeyspaceManager(
10001,
1,
cachePathOverride="",
pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL,
maxOpenFiles=100
)
self.sharedStateChannelFactory = (
InMemorySharedStateChannelFactory.InMemoryChannelFactory(
self.callbackScheduler,
self.sharedStateManager
)
)
self.sharedStateViewFactory = ViewFactory.ViewFactory(self.sharedStateChannelFactory)
self.client = InMemoryClient(self)
self.cumuli = []
self.nextCumulusAddress = 0
self.channelManager = InMemoryChannelManager(self.callbackScheduler, perMachineThroughput)
self.inMemoryDemuxingChannel = \
InMemorySharedStateChannelFactory.SerializedToManagerChannelFactory(
self.callbackScheduler,
self.sharedStateManager,
"SharedState"
)
def __init__(self,
cumulusVectorRamCacheSizeOverride = DEFAULT_VECTOR_RAM_CACHE_SIZE,
cumulusMaxRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
cumulusThreadCountOverride = DEFAULT_THREAD_COUNT,
remoteGatewayRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
perMachineThroughput = DEFAULT_PER_MACHINE_THROUGHPUT
):
self.cumulusMaxRamCacheSizeOverride = cumulusMaxRamCacheSizeOverride
self.cumulusVectorRamCacheSizeOverride = cumulusVectorRamCacheSizeOverride
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = self.callbackSchedulerFactory.createScheduler("InMemoryCluster", 1)
self.cumulusThreadCountOverride = cumulusThreadCountOverride
self.remoteGatewayCacheSize = remoteGatewayRamCacheSizeOverride
self.sharedStateManager = SharedStateService.KeyspaceManager(
10001,
1,
cachePathOverride="",
pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL,
maxOpenFiles=100
)
self.sharedStateChannelFactory = (
InMemorySharedStateChannelFactory.InMemoryChannelFactory(
self.callbackScheduler,
self.sharedStateManager
)
)
self.sharedStateViewFactory = ViewFactory.ViewFactory(self.sharedStateChannelFactory)
self.client = InMemoryClient(self)
self.cumuli = []
self.nextCumulusAddress = 0
self.channelManager = InMemoryChannelManager(self.callbackScheduler, perMachineThroughput)
self.inMemoryDemuxingChannel = \
InMemorySharedStateChannelFactory.SerializedToManagerChannelFactory(
self.callbackScheduler,
self.sharedStateManager,
"SharedState"
)
def initialize(setupObjectToUse=None, useLocalEvaluator=True, vdmOverride=None):
global _evaluator
if _evaluator is not None:
return
import ufora.FORA.python.Evaluator.LocalEvaluator as LocalEvaluator
import ufora.FORA.python.Evaluator.CumulusEvaluator as CumulusEvaluator
if setupObjectToUse is None:
configToUse = Setup.config()
else:
configToUse = setupObjectToUse.config
if useLocalEvaluator:
_evaluator = LocalEvaluator.defaultLocalEvaluator(vdmOverride=vdmOverride)
else:
import ufora.native.CallbackScheduler as CallbackSchedulerNative
schedulerFactory = CallbackSchedulerNative.createSimpleCallbackSchedulerFactory()
_evaluator = CumulusEvaluator.CumulusEvaluator(
schedulerFactory.createScheduler("CumulusEvaluator", 1)
)
def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1)
self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort
self.desirePublisher = None
self._connectionManager = None
def __init__(self,
socketFd,
sharedStateAddress):
self.socketFd = socketFd
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.scheduler = self.callbackSchedulerFactory.createScheduler(
"BackendGatewayRequestHandler",
1
)
sharedStateHost, sharedStatePort = sharedStateAddress.split(':')
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
self.callbackSchedulerFactory.createScheduler('SharedState', 1),
sharedStateHost,
int(sharedStatePort)
)
self.subscribableHandler = ConnectionHandler.ConnectionHandler(
self.scheduler,
sharedStateViewFactory,
lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler)
)
self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def __init__(self,
offlineCacheFunction,
newMemLimit,
remoteEvaluator=None,
newLoadRatio=.5,
maxPageSizeInBytes=None,
vdmOverride=None):
if maxPageSizeInBytes is None:
maxPageSizeInBytes = Setup.config().maxPageSizeInBytes
if vdmOverride is not None:
self.vdm_ = vdmOverride
self.offlineCache_ = None
else:
self.vdm_ = FORANative.VectorDataManager(
CallbackSchedulerNative.createSimpleCallbackSchedulerFactory()
.createScheduler("LocalEvaluator", 1),
maxPageSizeInBytes
)
self.vdm_.setDropUnreferencedPagesWhenFull(True)
self.offlineCache_ = offlineCacheFunction(self.vdm_)
if self.offlineCache_ is not None:
self.vdm_.setOfflineCache(self.offlineCache_)
logging.info("LocalEvaluator Creating VDMC with %s MB", newMemLimit / 1024.0 / 1024.0)
self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25))
self.vdm_.setLoadRatio(newLoadRatio)
self.remoteEvaluator_ = remoteEvaluator
self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
import sys
import argparse
import signal
import logging
import ufora.config.Setup as Setup
import ufora.distributed.SharedState.SharedStateService as SharedStateService
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.config.Mainline as Mainline
if __name__ == "__main__":
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler("sharedStateScheduler", 1)
parser = argparse.ArgumentParser()
parser.add_argument('--cacheDir')
parsed, remaining = parser.parse_known_args(sys.argv[1:])
def startSharedState(args):
if parsed.cacheDir:
logging.info("Shared state cache directory: %s", parsed.cacheDir)
Setup.config().sharedStateCache = parsed.cacheDir
service = SharedStateService.SharedStateService(callbackScheduler)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import argparse
import signal
import logging
import ufora.config.Setup as Setup
import ufora.distributed.SharedState.SharedStateService as SharedStateService
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.config.Mainline as Mainline
if __name__ == "__main__":
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
callbackScheduler = callbackSchedulerFactory.createScheduler(
"sharedStateScheduler", 1)
parser = argparse.ArgumentParser()
parser.add_argument('--cacheDir')
parsed, remaining = parser.parse_known_args(sys.argv[1:])
def startSharedState(args):
if parsed.cacheDir:
logging.info("Shared state cache directory: %s", parsed.cacheDir)
Setup.config().sharedStateCache = parsed.cacheDir
service = SharedStateService.SharedStateService(callbackScheduler)
service.startService()
service.blockUntilListening()
def getCallbackSchedulerFactory():
global _callbackSchedulerFactory
if _callbackSchedulerFactory is None:
_callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
return _callbackSchedulerFactory
def getCallbackSchedulerFactory():
global _callbackSchedulerFactory
if _callbackSchedulerFactory is None:
_callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
return _callbackSchedulerFactory
