def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory
)
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-gateway', 1)
channelListener = ChannelListener.SocketListener(args.port)
return BackendGatewayService.BackendGatewayService(callbackScheduler,
channelListener,
args.store_address)
def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1)
self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort
#create an OutOfProcessDownloader so we can execute commands like 'forever'
#from there, instead of forking from the main process (which can run out of memory)
self.processPool = OutOfProcessDownloader.OutOfProcessDownloaderPool(1)
self.desirePublisher = None
self._connectionManager = None
def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed):
object.__init__(self)
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.callbackSchedulerFactory = self.callbackScheduler.getFactory()
self.numRandVals = numRandVals
self.numRelaxations = numRelaxations
self.maxForRelax = maxForRelax
self.maxForRand = maxForRand
self.seed = seed
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.typed_fora_compiler = self.runtime.getTypedForaCompiler()
if testAxiomsPath is not None:
pathToUse = testAxiomsPath
else:
pathToUse = UNIT_TEST_AXIOMS_PATH
self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse)
self.axiom_groups = []
for i in range(self.axioms.axiomCount):
self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i))
self.symbol_strings = self.loadSymbolStrings()
numpy.random.seed(seed)
def createViewFactory():
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
return ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('fora-interpreter', 1),
'localhost',
Setup.config().sharedStatePort
)
def createWorker_(machineId,
viewFactory,
callbackSchedulerToUse,
threadCount,
memoryLimitMb,
cacheFunction,
pageSizeOverride,
disableEventHandler):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(
callbackSchedulerToUse,
pageSizeOverride if pageSizeOverride is not None else
1 * 1024 * 1024 if memoryLimitMb < 1000 else
5 * 1024 * 1024 if memoryLimitMb < 5000 else
50 * 1024 * 1024
)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
)
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(
viewFactory.createView(),
callbackSchedulerToUse
)
)
cache = cacheFunction()
if disableEventHandler:
eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler()
else:
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (
CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId,
threadCount,
CumulusNative.CumulusCheckpointPolicy.None(),
ExecutionContext.createContextConfiguration(),
""
),
vdm,
cache,
eventHandler
),
vdm,
eventHandler
)
def createClient_(clientId, callbackSchedulerToUse = None):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
vdm.setMemoryLimit(100 * 1024 * 1024, 125 * 1024 * 1024)
return (
CumulusNative.CumulusClient(vdm, clientId, callbackSchedulerToUse),
vdm
)
def handle(self):
logging.info("all file descriptors closed")
callbackScheduler = CallbackScheduler.createSimpleCallbackSchedulerFactory().createScheduler(
"BackendGatewayService", 1
)
channel = SocketStringChannel.SocketStringChannel(callbackScheduler, self.sock).makeQueuelike(callbackScheduler)
logging.info("channel connected")
self.serviceRequest(channel)
logging.info("Killing self!")
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
self.instructionGraph = self.runtime.getInstructionGraph()
self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
config = Setup.config()
print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % (
config.cumulusVectorRamCacheMB,
config.cumulusMaxRamCacheMB,
config.cumulusServiceThreadCount,
config.cumulusTrackTcmalloc
)
print "Ufora store at %s:%s" % (args.manager_address, args.manager_port)
s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory()
print "PythonIoTasks threads: %d. Out of process: %s" % (
config.externalDatasetLoaderServiceThreads,
s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
)
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory,
s3InterfaceFactory=s3InterfaceFactory,
objectStore=NullObjectStore.NullObjectStore()
)
def createWorkersAndClients(
workerCount,
clientCount,
viewFactory = None,
memoryLimitMb = 100,
threadCount = 2,
callbackSchedulerToUse = None
):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
if viewFactory is None:
viewFactory = createInMemorySharedStateViewFactory(callbackSchedulerToUse)
workersVdmsAndEventHandlers = [
createWorker(
machineId(ix),
viewFactory,
memoryLimitMb = memoryLimitMb,
threadCount=threadCount,
callbackSchedulerToUse = callbackSchedulerToUse
) for ix in range(workerCount)
]
clientsAndVdms = [
createClient(
clientId(ix),
callbackSchedulerToUse = callbackSchedulerToUse
)
for ix in range(clientCount)
]
for ix1 in range(len(workersVdmsAndEventHandlers)):
workersVdmsAndEventHandlers[ix1][0].startComputations()
for ix1 in range(len(workersVdmsAndEventHandlers)-1):
for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)):
worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addMachine(machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix2][0].addMachine(machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
for ix1 in range(len(workersVdmsAndEventHandlers)):
for ix2 in range(len(clientsAndVdms)):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse)
return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
def createStorage(vdm):
self.simpleOfflineCache = CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000)
return self.simpleOfflineCache
self.evaluator = LocalEvaluator.LocalEvaluator(
createStorage,
2000000,
maxPageSizeInBytes = 100000
)
self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
def __init__(self, socketFd, sharedStateAddress):
self.socketFd = socketFd
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.scheduler = self.callbackSchedulerFactory.createScheduler("BackendGatewayRequestHandler", 1)
sharedStateHost, sharedStatePort = sharedStateAddress.split(":")
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
self.callbackSchedulerFactory.createScheduler("SharedState", 1), sharedStateHost, int(sharedStatePort)
)
self.subscribableHandler = ConnectionHandler.ConnectionHandler(
self.scheduler, sharedStateViewFactory, lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler)
)
self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)
def __init__(self,
inMemory,
port = None,
cachePathOverride = '',
maxOpenFiles = 256,
inMemChannelFactoryFactory = None,
maxLogFileSizeMb = 10,
pingInterval = None):
self.inMemory = inMemory
self.manager = None
self.callbackScheduler = CallbackScheduler.singletonForTesting()
if self.inMemory:
self.manager = SharedStateService.KeyspaceManager(
10001,
1,
cachePathOverride=cachePathOverride,
pingInterval = IN_MEMORY_HARNESS_PING_INTERVAL if pingInterval is None else pingInterval,
maxOpenFiles=maxOpenFiles,
maxLogFileSizeMb=maxLogFileSizeMb
)
#although named otherwise InMemoryChannelFactory is actually a factory for a channelFactory
# or a channelFactoryFactory
channelFactoryFactory = inMemChannelFactoryFactory if inMemChannelFactoryFactory is not None \
else InMemoryChannelFactory.InMemoryChannelFactory
logging.info(channelFactoryFactory)
self.channelFactory = channelFactoryFactory(self.callbackScheduler, self.manager)
self.viewFactory = ViewFactory.ViewFactory(self.channelFactory)
else:
class Settings(object):
callbackScheduler = self.callbackScheduler
assert port is not None
self.service = SharedStateService.SharedStateService(
self.callbackScheduler,
cachePathOverride=cachePathOverride,
port=port
)
self.service.startService()
self.service.blockUntilListening()
self.viewFactory = ViewFactory.ViewFactory.TcpViewFactory(self.callbackScheduler, "localhost", port)
def __init__(self,
cumulusVectorRamCacheSizeOverride = DEFAULT_VECTOR_RAM_CACHE_SIZE,
cumulusMaxRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
cumulusThreadCountOverride = DEFAULT_THREAD_COUNT,
remoteGatewayRamCacheSizeOverride = DEFAULT_MAX_RAM_CACHE_SIZE,
perMachineThroughput = DEFAULT_PER_MACHINE_THROUGHPUT
):
self.cumulusMaxRamCacheSizeOverride = cumulusMaxRamCacheSizeOverride
self.cumulusVectorRamCacheSizeOverride = cumulusVectorRamCacheSizeOverride
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = self.callbackSchedulerFactory.createScheduler("InMemoryCluster", 1)
self.cumulusThreadCountOverride = cumulusThreadCountOverride
self.remoteGatewayCacheSize = remoteGatewayRamCacheSizeOverride
self.sharedStateManager = SharedStateService.KeyspaceManager(
10001,
1,
cachePathOverride="",
pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL,
maxOpenFiles=100
)
self.sharedStateChannelFactory = (
InMemorySharedStateChannelFactory.InMemoryChannelFactory(
self.callbackScheduler,
self.sharedStateManager
)
)
self.sharedStateViewFactory = ViewFactory.ViewFactory(self.sharedStateChannelFactory)
self.client = InMemoryClient(self)
self.cumuli = []
self.nextCumulusAddress = 0
self.channelManager = InMemoryChannelManager(self.callbackScheduler, perMachineThroughput)
self.inMemoryDemuxingChannel = \
InMemorySharedStateChannelFactory.SerializedToManagerChannelFactory(
self.callbackScheduler,
self.sharedStateManager,
"SharedState"
)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0],
"AxiomJOA_test.txt"))
self.evaluator = LocalEvaluator.LocalEvaluator(
lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000),
10000000,
maxPageSizeInBytes = 100000
)
self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
self.knownModulesAsConstantJOVs = dict()
self.knownModulesAsConstantJOVs["builtin"] = \
FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def createInMemorySharedStateViewFactory(callbackSchedulerToUse = None):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
sharedStateManager = SharedStateService.KeyspaceManager(
10001,
1,
cachePathOverride="",
pingInterval = IN_MEMORY_CLUSTER_SS_PING_INTERVAL,
maxOpenFiles=100
)
sharedStateChannelFactory = (
InMemorySharedStateChannelFactory.InMemoryChannelFactory(
callbackSchedulerToUse.getFactory().createScheduler("SharedState", 1),
sharedStateManager
)
)
return ViewFactory.ViewFactory(sharedStateChannelFactory)
def initialize(setupObjectToUse=None, useLocalEvaluator=True, vdmOverride=None):
global _evaluator
if _evaluator is not None:
return
import ufora.FORA.python.Evaluator.LocalEvaluator as LocalEvaluator
import ufora.FORA.python.Evaluator.CumulusEvaluator as CumulusEvaluator
if setupObjectToUse is None:
configToUse = Setup.config()
else:
configToUse = setupObjectToUse.config
if useLocalEvaluator:
_evaluator = LocalEvaluator.defaultLocalEvaluator(vdmOverride=vdmOverride)
else:
import ufora.native.CallbackScheduler as CallbackSchedulerNative
schedulerFactory = CallbackSchedulerNative.createSimpleCallbackSchedulerFactory()
_evaluator = CumulusEvaluator.CumulusEvaluator(
schedulerFactory.createScheduler("CumulusEvaluator", 1)
)
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
)
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(
viewFactory.createView(),
callbackSchedulerToUse
)
)
cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024)
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (
CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId,
threadCount,
CumulusNative.CumulusCheckpointPolicy.None(),
ExecutionContext.createContextConfiguration(),
""
),
vdm,
cache,
eventHandler
),
vdm,
eventHandler
)
def __init__(self):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.callbackScheduler = callbackSchedulerFactory.createScheduler("Simulator", 1)
self.uforaPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
self.sharedStatePath = os.path.join(self.uforaPath, 'distributed/SharedState')
self.sharedStateMainline = os.path.join(self.sharedStatePath, 'sharedStateMainline.py')
self.gatewayServiceMainline = os.path.join(self.uforaPath, 'scripts/init/ufora-gateway.py')
self.webPath = os.path.join(self.uforaPath, 'web/relay')
self.relayScript = os.path.join(self.webPath, 'server.coffee')
self.relayPort = Setup.config().relayPort
self.relayHttpsPort = Setup.config().relayHttpsPort
self.sharedStatePort = Setup.config().sharedStatePort
self.restApiPort = Setup.config().restApiPort
self.subscribableWebObjectsPort = Setup.config().subscribableWebObjectsPort
self.desirePublisher = None
self._connectionManager = None
def __init__(self,
offlineCacheFunction,
newMemLimit,
remoteEvaluator=None,
newLoadRatio=.5,
maxPageSizeInBytes=None,
vdmOverride=None):
if maxPageSizeInBytes is None:
maxPageSizeInBytes = Setup.config().maxPageSizeInBytes
if vdmOverride is not None:
self.vdm_ = vdmOverride
self.offlineCache_ = None
else:
self.vdm_ = FORANative.VectorDataManager(
CallbackSchedulerNative.createSimpleCallbackSchedulerFactory()
.createScheduler("LocalEvaluator", 1),
maxPageSizeInBytes
)
self.vdm_.setDropUnreferencedPagesWhenFull(True)
self.offlineCache_ = offlineCacheFunction(self.vdm_)
if self.offlineCache_ is not None:
self.vdm_.setOfflineCache(self.offlineCache_)
logging.info("LocalEvaluator Creating VDMC with %s MB", newMemLimit / 1024.0 / 1024.0)
self.vdm_.setMemoryLimit(newMemLimit, int(newMemLimit * 1.25))
self.vdm_.setLoadRatio(newLoadRatio)
self.remoteEvaluator_ = remoteEvaluator
self.cache_ = ComputationCache(self.vdm_, self.offlineCache_)
def __init__(self,
workerCount,
clientCount,
memoryPerWorkerMB,
threadsPerWorker,
s3Service,
objectStore=None,
callbackScheduler=None,
sharedStateViewFactory=None,
ioTaskThreadOverride=None,
useInMemoryCache=True,
channelThroughputMBPerSecond=None,
pageSizeOverride=None,
disableEventHandler=False,
machineIdHashSeed=None
):
self.useInMemoryCache = useInMemoryCache
self.machineIdHashSeed = machineIdHashSeed
if not self.useInMemoryCache:
self.diskCacheCount = 0
if os.getenv("CUMULUS_DATA_DIR") is None:
self.diskCacheStorageDir = tempfile.mkdtemp()
else:
self.diskCacheStorageDir = os.path.join(
os.getenv("CUMULUS_DATA_DIR"),
str(uuid.uuid4())
)
self.ioTaskThreadOverride = ioTaskThreadOverride
self.workerCount = 0
self.disableEventHandler = disableEventHandler
self.clientCount = 0
self.memoryPerWorkerMB = memoryPerWorkerMB
self.threadsPerWorker = threadsPerWorker
self.s3Service = s3Service
self.objectStore = objectStore
if self.objectStore is None:
s3 = s3Service()
if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface):
objectStoreBucket = "object_store_bucket"
s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue')
s3.deleteKey(objectStoreBucket, 'dummyKey')
else:
objectStoreBucket = Setup.config().userDataS3Bucket
self.objectStore = S3ObjectStore.S3ObjectStore(
s3Service,
objectStoreBucket,
prefix="test/")
self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting()
self.sharedStateViewFactory = (
sharedStateViewFactory or createInMemorySharedStateViewFactory(self.callbackScheduler)
)
self.channelThroughputMBPerSecond = channelThroughputMBPerSecond
self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler, 5 * 1024 * 1024)
self.pageSizeOverride = pageSizeOverride
self.rateLimitedChannelGroupsForEachListener = []
self.workersVdmsAndEventHandlers = []
self.machineIds = []
self.machineIdsEverAllocated = 0
self.clientsAndVdms = []
self.loadingServices = []
self.clientTeardownGates = []
self.workerTeardownGates = []
for ix in range(workerCount):
self.addWorker()
for ix in range(clientCount):
self.addClient()
if clientCount:
self.listener = self.getClient(0).createListener()
else:
self.listener = None
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
def __init__(self,
workerCount,
clientCount,
memoryPerWorkerMB,
threadsPerWorker,
s3Service,
objectStore=None,
callbackScheduler=None,
sharedStateViewFactory=None,
ioTaskThreadOverride=None,
useInMemoryCache=True,
channelThroughputMBPerSecond=None,
pageSizeOverride=None,
disableEventHandler=False,
machineIdHashSeed=None):
self.useInMemoryCache = useInMemoryCache
self.machineIdHashSeed = machineIdHashSeed
if not self.useInMemoryCache:
self.diskCacheCount = 0
if os.getenv("CUMULUS_DATA_DIR") is None:
self.diskCacheStorageDir = tempfile.mkdtemp()
else:
self.diskCacheStorageDir = os.path.join(
os.getenv("CUMULUS_DATA_DIR"), str(uuid.uuid4()))
self.ioTaskThreadOverride = ioTaskThreadOverride
self.workerCount = 0
self.disableEventHandler = disableEventHandler
self.clientCount = 0
self.memoryPerWorkerMB = memoryPerWorkerMB
self.threadsPerWorker = threadsPerWorker
self.s3Service = s3Service
self.objectStore = objectStore
if self.objectStore is None:
s3 = s3Service()
if isinstance(s3, InMemoryS3Interface.InMemoryS3Interface):
objectStoreBucket = "object_store_bucket"
s3.setKeyValue(objectStoreBucket, 'dummyKey', 'dummyValue')
s3.deleteKey(objectStoreBucket, 'dummyKey')
else:
objectStoreBucket = Setup.config().userDataS3Bucket
self.objectStore = S3ObjectStore.S3ObjectStore(s3Service,
objectStoreBucket,
prefix="test/")
self.callbackScheduler = callbackScheduler or CallbackScheduler.singletonForTesting(
)
self.sharedStateViewFactory = (sharedStateViewFactory
or createInMemorySharedStateViewFactory(
self.callbackScheduler))
self.channelThroughputMBPerSecond = channelThroughputMBPerSecond
self.resultVDM = ForaNative.VectorDataManager(self.callbackScheduler,
5 * 1024 * 1024)
self.pageSizeOverride = pageSizeOverride
self.rateLimitedChannelGroupsForEachListener = []
self.workersVdmsAndEventHandlers = []
self.machineIds = []
self.machineIdsEverAllocated = 0
self.clientsAndVdms = []
self.loadingServices = []
self.clientTeardownGates = []
self.workerTeardownGates = []
for ix in range(workerCount):
self.addWorker()
for ix in range(clientCount):
self.addClient()
if clientCount:
self.listener = self.getClient(0).createListener()
else:
self.listener = None
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
import sys
import argparse
import signal
import logging
import ufora.config.Setup as Setup
import ufora.distributed.SharedState.SharedStateService as SharedStateService
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.config.Mainline as Mainline
if __name__ == "__main__":
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler("sharedStateScheduler", 1)
parser = argparse.ArgumentParser()
parser.add_argument('--cacheDir')
parsed, remaining = parser.parse_known_args(sys.argv[1:])
def startSharedState(args):
if parsed.cacheDir:
logging.info("Shared state cache directory: %s", parsed.cacheDir)
Setup.config().sharedStateCache = parsed.cacheDir
service = SharedStateService.SharedStateService(callbackScheduler)
def setUpClass(cls):
cls.vdm = ForaNative.VectorDataManager(
CallbackScheduler.singletonForTesting(), 10000000)
def getCallbackSchedulerFactory():
global _callbackSchedulerFactory
if _callbackSchedulerFactory is None:
_callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
return _callbackSchedulerFactory
def computeUsingSeveralWorkers(expressionText,
s3Service,
count,
objectStore=None,
wantsStats=False,
timeout=10,
returnEverything=False,
memoryLimitMb=100,
blockUntilConnected=False,
keepSimulationAlive=False,
sharedStateViewFactory=None,
threadCount=2):
if keepSimulationAlive:
assert returnEverything, \
"can't keep the simulation alive and not return it. how would you shut it down?"
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
if sharedStateViewFactory is None:
sharedStateViewFactory = createInMemorySharedStateViewFactory(
callbackSchedulerToUse = callbackSchedulerToUse
)
workersVdmsAndEventHandlers, clientsAndVdms, viewFactory = (
createWorkersAndClients(
count,
1,
sharedStateViewFactory,
memoryLimitMb = memoryLimitMb,
threadCount = threadCount
)
)
client = clientsAndVdms[0][0]
clientVdm = clientsAndVdms[0][1]
loadingServices = []
for ix in range(len(workersVdmsAndEventHandlers)):
worker = workersVdmsAndEventHandlers[ix][0]
workerVdm = workersVdmsAndEventHandlers[ix][1]
s3InterfaceFactory = s3Service.withMachine(ix)
if objectStore is None:
objectStore = S3ObjectStore.S3ObjectStore(
s3InterfaceFactory,
Setup.config().userDataS3Bucket,
prefix="test/")
loadingService = PythonIoTaskService.PythonIoTaskService(
s3InterfaceFactory,
objectStore,
workerVdm,
worker.getExternalDatasetRequestChannel(callbackSchedulerToUse).makeQueuelike(callbackSchedulerToUse)
)
loadingService.startService()
loadingServices.append(loadingService)
if blockUntilConnected:
for worker,vdm,eventHandler in workersVdmsAndEventHandlers:
blockUntilWorkerIsConnected(worker, 2.0)
if isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
computationDefinition = (
createComputationDefinition(
FORA.extractImplValContainer(
FORA.eval("fun() {" + expressionText + " } ")
),
ForaNative.makeSymbol("Call")
)
)
teardownGates = []
for client, vdm in clientsAndVdms:
teardownGates.append(vdm.getVdmmTeardownGate())
for worker, vdm, eventHandler in workersVdmsAndEventHandlers:
teardownGates.append(vdm.getVdmmTeardownGate())
simulationDict = {
"result": None,
"timedOut": None,
"stats": None,
"clientsAndVdms": clientsAndVdms,
"workersVdmsAndEventHandlers": workersVdmsAndEventHandlers,
"s3Service": s3Service,
"loadingServices": loadingServices,
"sharedStateViewFactory": sharedStateViewFactory,
"client": client,
"teardownGates": teardownGates
}
try:
listener = client.createListener()
computationSubmitTime = time.time()
computationId = client.createComputation(computationDefinition)
client.setComputationPriority(
computationId,
CumulusNative.ComputationPriority(1)
)
if returnEverything:
valAndStatsOrNone = waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=True)
computationReturnTime = time.time()
if valAndStatsOrNone is None:
#we timed out
val = None
stats = None
timedOut = True
else:
val, stats = valAndStatsOrNone
timedOut = False
simulationDict.update({
"result": val,
"stats": stats,
"timedOut": timedOut,
"computationId": computationId,
"listener": listener,
"totalTimeToReturnResult": computationReturnTime - computationSubmitTime
})
return simulationDict
else:
return waitForResult(listener, computationId, clientVdm, timeout=timeout, wantsStats=wantsStats)
finally:
if not keepSimulationAlive:
teardownSimulation(simulationDict)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import time
import ufora.FORA.python.FORA as FORA
import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation
import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.test.PerformanceTestReporter as PerformanceTestReporter
import ufora.FORA.python.Runtime as Runtime
callbackScheduler = CallbackScheduler.singletonForTesting()
class BigboxStringPerformanceTest(unittest.TestCase):
def computeUsingSeveralWorkers(self, *args, **kwds):
return InMemoryCumulusSimulation.computeUsingSeveralWorkers(*args, **kwds)
def stringCreationAndSumTest(self, totalStrings, workers, threadsPerWorker, testName):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
#we wish we could actually test that we achieve saturation here but we can't yet.
text = """Vector.range(%s, String).sum(size)""" % totalStrings
t0 = time.time()
_, simulation = \
self.computeUsingSeveralWorkers(
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import time
import ufora.cumulus.test.InMemoryCumulusSimulation as InMemoryCumulusSimulation
import ufora.distributed.S3.InMemoryS3Interface as InMemoryS3Interface
import ufora.native.CallbackScheduler as CallbackScheduler
import ufora.test.PerformanceTestReporter as PerformanceTestReporter
callbackScheduler = CallbackScheduler.singletonForTesting()
class MultimachineLinearRegressionSimulationTest(unittest.TestCase):
def dataGenerationScript(self, mbOfData, columns):
valueCount = mbOfData * 1024 * 1024 / 8
rowCount = valueCount / columns
return ("""
let data = Vector.range(__columns__) ~~ fun(c) {
Vector.range(__rows__, fun(r) { if (c == 0) 1.0 else Float64(r % (c+2)) })
};
let dfResponse = dataframe.DataFrame(data[-1,]);
let dfPredictors = dataframe.DataFrame(data[,-1]);
