def loadHttpDataset(datasetDescriptor, vdid, vdm): try: data, statusCode = loadHttpRequestDataset( datasetDescriptor.asHttpRequestDataset.dataset ) if statusCode != 200: raise InvalidDatasetException( "Load returned status code %s" % statusCode ) if not vdm.loadByteArrayIntoExternalDatasetPageAsVector(vdid, data): raise DatasetLoadException("Couldn't load dataset into VDM") except InvalidDatasetException as e: if not vdm.loadImplvalIntoUnloadedVectorHandle( vdid, ForaNative.ImplValContainer(e.message)): raise DatasetLoadException("Couldn't load dataset into VDM")
def test_cached_nodes_3(self): text = """fun() { let f = fun() { 0 }; let g = fun() { throw 1 }; cached(f(), g()) }""" cfg = self.parseStringToFunction(text).toCFG(0) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, None, ()) cfgWithFutures.slotCompleted( 0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))) self.assertEqual( cfgWithFutures.getFinalResult().asResult.result.asException. exception, ForaNative.ImplValContainer(1))
def test_return_MakeTuple(self): text = "fun() { let res = 1 + 1; (res, 1,2,3) }" cfg = self.parseStringToFunction(text).toCFG(0) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, None, ()) cfgWithFutures.slotCompleted( 0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)) ) cfgWithFutures.continueSimulation() self.assertEqual( cfgWithFutures.getFinalResult().asResult.result.asResult.result, ForaNative.ImplValContainer((2,1,2,3)) )
def simpleEval(callbackScheduler, *args): """convert 'args' to FORA ImplValContainers, evaluates, and returns a python value. Assumes you don't use cache or vector loads. If you return an exception, this function asserts false. Otherwise, it returns the ImplValContainer result. """ e = ExecutionContext( dataManager=VectorDataManager.constructVDM(callbackScheduler)) e.evaluate(*[FORANative.ImplValContainer(x) for x in args]) tr = e.getFinishedResult() if tr.isFailure(): assert False if tr.isException(): assert False try: return tr.asResult.result.pyval except: return tr.asResult.result
def loadEntireS3Dataset(datasetDescriptor, s3InterfaceFactory, vdid, vdm): try: vectorSlices = produceVDIDSlicesForEntireS3Dataset( s3InterfaceFactory, datasetDescriptor.asEntireS3Dataset.dataset, vdm ) vector = ForaNative.createFORAFreeBinaryVectorFromSlices(vectorSlices, vdm) if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector): raise DatasetLoadException("Couldn't load dataset into VDM") except InvalidDatasetException as e: if not vdm.loadImplvalIntoUnloadedVectorHandle( vdid, ForaNative.ImplValContainer(e.message)): raise DatasetLoadException("Couldn't load dataset into VDM") except: logging.error("WTF: %s", traceback.format_exc())
def disabled_garbage_collection_with_switch(self): cfg = self.parseStringToFunction( "fun(x, f, g) { match (f(x)) with (1) { g(1) } (2) { g(2) } }" ).toCFG(3) nodeValues = (ForaNative.ImplValContainer(1), FORA.extractImplValContainer(FORA.eval("fun(x) { x }")), FORA.extractImplValContainer( FORA.eval("fun(x) { x + 1 }"))) serial = serialSimulation(cfg, nodeValues) self.assertTrue(serial.getSlots()[0].isGarbageCollected()) self.seed(10) for ix in range(10): randomOrder = randomOrderSimulation(cfg, nodeValues) self.assertTrue(randomOrder.getSlots()[0].isGarbageCollected())
def test_resumingAfterCopyDataOutOfPages(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) text = """ fun() { let v = Vector.range(1000).paged; let ix1 = 0 let res = 0 while (true) { res = res + v[ix1] ix1 = (ix1 + 1) % size(v) } res }""" context.placeInEvaluationState( FORANative.ImplValContainer( (FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call))) context.interruptAfterCycleCount(100000) context.compute() paused1 = context.extractPausedComputation() while not context.isVectorLoad(): context.copyValuesOutOfVectorPages() vdm.unloadAllPossible() context.resetInterruptState() context.interruptAfterCycleCount(100000) context.compute() paused2 = context.extractPausedComputation() self.assertTrue( len(paused1.asThread.computation.frames) == len( paused2.asThread.computation.frames))
def test_futures_5(self): funString = "fun(f) { f(0) + f(1) + f(2) }" cfg = self.parseStringToFunction(funString).toCFG(1) funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x }")) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, "block_0Let", (funImplval, )) for ix in range(3): cfgWithFutures.continueSimulation() self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1, 3]) cfgWithFutures.submittableArgs(3) cfgWithFutures.slotCompleted( 3, normalComputationResult(ForaNative.ImplValContainer(2))) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1]) cfgWithFutures.submittableArgs(1) cfgWithFutures.slotCompleted( 1, normalComputationResult(ForaNative.ImplValContainer(1))) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0]) cfgWithFutures.submittableArgs(0) cfgWithFutures.slotCompleted( 0, normalComputationResult(ForaNative.ImplValContainer(0))) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2]) cfgWithFutures.submittableArgs(2) cfgWithFutures.slotCompleted( 2, normalComputationResult(ForaNative.ImplValContainer(2))) cfgWithFutures.continueSimulation() cfgWithFutures.submittableArgs(4) cfgWithFutures.slotCompleted( 4, normalComputationResult(ForaNative.ImplValContainer(4))) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), []) cfgWithFutures.continueSimulation() self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), []) self.assertTrue(cfgWithFutures.hasResolvedToSimpleState()) finalResult = cfgWithFutures.getFinalResult() self.assertEqual(finalResult.asResult.result.asResult.result, ForaNative.ImplValContainer(4))
def test_mutable_values_2(self): text = "fun(f, g) { let x = f(); let y = g(); 1 }" cfg = self.parseStringToFunction(text).toCFG(2) fImplval = FORA.extractImplValContainer( FORA.eval("let f = fun() { Vector.range(10) }; f") ) gImplval = FORA.extractImplValContainer( FORA.eval("let g = fun() { MutableVector(Float64).create(10, 0.0) }; g") ) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, None, (fImplval, gImplval)) cfgWithFutures.continueSimulation() cfgWithFutures.continueSimulation() self.assertFalse(cfgWithFutures.hasResolvedToSimpleState()) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1]) cfgWithFutures.slotCompleted( 1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1)) ) self.assertTrue(cfgWithFutures.mustBailEarly()) self.assertFalse(cfgWithFutures.hasResolvedToSimpleState()) cfgWithFutures.slotCompleted( 0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)) ) self.assertTrue(cfgWithFutures.mustBailEarly()) self.assertTrue(cfgWithFutures.hasResolvedToSimpleState()) finalResult = cfgWithFutures.getFinalResult() self.assertEqual( finalResult.asResult.result.asResult.result, ForaNative.ImplValContainer(1) )
def test_cached_nodes_1(self): text = "fun(f, g) { cached(f(), g()); }" cfg = self.parseStringToFunction(text).toCFG(2) fImplval = FORA.extractImplValContainer(FORA.eval("fun() { 1 }")) gImplVal = FORA.extractImplValContainer(FORA.eval("fun() { 2 }")) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, None, (fImplval, gImplVal)) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0]) res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0)) expectedResult = ForaNative.ImplValContainer((1, 2)) self.assertEqual(res.asResult.result, expectedResult) cfgWithFutures.slotCompleted(0, res) finalResult = cfgWithFutures.getFinalResult() self.assertEqual(finalResult.asResult.result.asResult.result, expectedResult)
def roundtripExecute(self, pyObject, *args): mappings = PureImplementationMappings.PureImplementationMappings() binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry() walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry) ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)] binaryObjectRegistry.defineEndOfStream() data = binaryObjectRegistry.str() streamReader = PythonBinaryStreamToImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) streamReader.read(data) implVals = [streamReader.getObjectById(i) for i in ids] result = InMemoryCumulusSimulation.computeUsingSeveralWorkers( ForaNative.ImplValContainer((implVals[0], ForaNative.makeSymbol("Call")) + tuple(implVals[1:])), InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1) self.assertTrue(result.isResult(), result) result = result.asResult.result converter = PythonBinaryStreamFromImplval.constructConverter( Converter.canonicalPurePythonModule(), self.vdm) root_id, data = converter.write(result) rehydrator = PythonObjectRehydrator( mappings, allowUserCodeModuleLevelLookups=False) return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def convertClassInstanceDescription(self, objectId, classInstanceDescription): classMemberNameToImplVal = { classMemberName: self.convertedValues[memberId] for classMemberName, memberId in classInstanceDescription. classMemberNameToClassMemberId.iteritems() } classImplVal = self.convertedValues[classInstanceDescription.classId] #note that we need to strip off the first character of membernames defined in the #class implval because the object holds 'x' as '@x' so that it doesn't capture #all references to 'x' classMembersInForaDeclarationOrder = \ [str(val)[1:] for val in classImplVal.getDataMembers] assert set(classMembersInForaDeclarationOrder) == \ set(classMemberNameToImplVal.keys()), "%s vs %s" % ( set(classMembersInForaDeclarationOrder), set(classMemberNameToImplVal.keys()) ) classMemberImplVals = [] for classMemberName in classMembersInForaDeclarationOrder: ivc = classMemberNameToImplVal[classMemberName] classMemberImplVals.append(ivc) applyArgs = [classImplVal, Symbol_CreateInstance] + classMemberImplVals convertedValueOrNone = ForaNative.simulateApply( ForaNative.ImplValContainer(tuple(applyArgs))) if convertedValueOrNone is None: raise pyfora.PythonToForaConversionError( ("An internal error occurred: " + "function stage 1 simulation unexpectedly returned None")) self.convertedValues[objectId] = convertedValueOrNone
def createCliqueFinderContext(self, text, *args): maxPageSizeInBytes = 32 * 1024 vdm = FORANative.VectorDataManager(callbackScheduler, maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) argumentImplvals = [] for a in args: context.evaluate( FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")), Symbol_Call) argumentImplvals.append( context.getFinishedResult().asResult.result) actualFunction = FORA.extractImplValContainer(FORA.eval(text)) context.placeInEvaluationState( FORANative.ImplValContainer((actualFunction, Symbol_Call) + tuple(argumentImplvals))) return context, argumentImplvals, vdm
def __init__(self, pyforaBuiltinsModule): self.pyforaBuiltinsModule = pyforaBuiltinsModule self.pythonNameToInstance = {} self.instanceToPythonName = {} self.pyExceptionClass = pyforaBuiltinsModule.getObjectMember( "PyException") self.pyAbortExceptionClass = pyforaBuiltinsModule.getObjectMember( "PyAbortException") self.pyExceptionClassInstanceName = ForaNative.simulateApply( ForaNative.ImplValContainer( (self.pyExceptionClass, ForaNative.makeSymbol("CreateInstance"), ForaNative.ImplValContainer(), ForaNative.ImplValContainer()))).getClassName() self.pyAbortExceptionClassInstanceName = ForaNative.simulateApply( ForaNative.ImplValContainer( (self.pyAbortExceptionClass, ForaNative.makeSymbol("CreateInstance"), ForaNative.ImplValContainer(), ForaNative.ImplValContainer()))).getClassName() self.pythonNameToPyforaName = {} self.pythonNameToPyforaName.update( NamedSingletons.pythonNameToPyforaName) self.pythonNameToPyforaName.update( PyAbortSingletons.pythonNameToPyforaName) for pyName, pyforaName in self.pythonNameToPyforaName.iteritems(): try: instance = FORA.ForaValue.FORAValue( pyforaBuiltinsModule).__getattr__(pyforaName).implVal_ except: logging.error( "Initializing PyforaSingletonAndExceptionConverter failed:" + " couldn't find %s in the pyfora builtins", pyforaName) raise self.pythonNameToInstance[pyName] = instance self.instanceToPythonName[instance] = pyName
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time import ufora.FORA.python.ExecutionContext as ExecutionContext import ufora.FORA.python.FORA as FORA import ufora.native.FORA as FORANative import ufora.native.CallbackScheduler as CallbackScheduler import logging import ufora.test.PerformanceTestReporter as PerformanceTestReporter callbackScheduler = CallbackScheduler.singletonForTesting() Symbol_Call = FORANative.ImplValContainer(FORANative.makeSymbol("Call")) class TestFuturesCliqueFinder(unittest.TestCase): def createCliqueFinderContext(self, text, *args): maxPageSizeInBytes = 32 * 1024 vdm = FORANative.VectorDataManager(callbackScheduler, maxPageSizeInBytes) context = ExecutionContext.ExecutionContext(dataManager=vdm) argumentImplvals = [] for a in args: context.evaluate( FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
def validator(result): return result == ForaNative.ImplValContainer((aVal, aVal))
def normalComputationResult(result): return ForaNative.ComputationResult.Result(result, ForaNative.ImplValContainer())
def createComputationDefinition(*args): return CumulusNative.ComputationDefinition.Root( CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm([ CumulusNative.ComputationDefinitionTerm.Value( ForaNative.ImplValContainer(x), None) for x in args ]))
def sim(memberName): return ForaNative.simulateApply( ForaNative.ImplValContainer((ivc, makeSymbolIvc("Member"), makeSymbolIvc(memberName)))).pyval
def evaluate(self, *args): argIv = FORANative.ImplValContainer( self.expandIfListOrTuple(callAndReturn, *args)) res = self.evaluateDirectly(callAndReturn, *args) return FORANative.updateFreeStoreValues(argIv, res)
def args(self): return (self.baseCV, ForaNative.makeSymbol("RawGetItemByString"), ForaNative.ImplValContainer(self.keyname))
def wireWorkersTogether_(self, ix1, ix2): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) if self.channelThroughputMBPerSecond is not None: worker1Channel1 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel1) worker1Channel2 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel2) worker2Channel1 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel1) worker2Channel2 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel2) self.workersVdmsAndEventHandlers[ix1][0].addMachine(self.machineIds[ix2], [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), self.callbackScheduler) self.workersVdmsAndEventHandlers[ix2][0].addMachine(self.machineIds[ix1], [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def exceptionComputationResult(result): return ForaNative.ComputationResult.Exception( result, ForaNative.ImplValContainer())
def evaluate(context, *args): context.placeInEvaluationState(FORANative.ImplValContainer(args)) context.compute()
def valueIVC(self): if self.valueIVC_ is None: return ForaNative.ImplValContainer() return self.valueIVC_
def args(self): return (self.baseCV, ForaNative.makeSymbol("RawGetItemByInt"), ForaNative.ImplValContainer(self.index))
def evalSubmittableArgs(submittableArgs): def evalApplyTuple(applyTuple, signature = None): varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple))) if signature is None: evalStringTerms = varNames else: evalStringTerms = [] for ix in range(len(signature.terms)): term = signature.terms[ix] if term.isNormal(): name = term.asNormal.name if name is not None: evalStringTerms.append("%s: x_%s" % (name, ix)) else: evalStringTerms.append("x_%s" % ix) elif term.isTupleCall(): evalStringTerms.append("*x_%s" % ix) evalString = evalStringTerms[0] + "`(" + ",".join(evalStringTerms[1:]) + ")" try: return normalComputationResult( FORA.extractImplValContainer( FORA.eval( evalString, locals = { name: val for (name, val) in zip(varNames, applyTuple) }, keepAsForaValue = True ) ) ) except ForaValue.FORAException as e: return exceptionComputationResult( FORA.extractImplValContainer(e.foraVal) ) if submittableArgs.isApply(): return evalApplyTuple( submittableArgs.args.values, submittableArgs.args.signature ) else: # submittableArgs.isCached() tr = () # this corresponds to the logic for Cached nodes in # CFGWithFutures::asSubmittable assert len(submittableArgs.args.values) == 1 for cachedPair in submittableArgs.args.values[0]: applyArgs = [cachedPair[0], ForaNative.makeSymbol("Call")] applyArgs.extend(cachedPair[1]) res = evalApplyTuple(applyArgs) if res.isException(): return res tr = tr + (res.asResult.result,) return normalComputationResult(ForaNative.ImplValContainer(tr))
def wireWorkerToClient_(self, ix1, ix2): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler) self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), self.callbackScheduler) self.clientsAndVdms[ix2][0].addMachine(self.machineIds[ix1], [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def makeSymbolIvc(symbolString): return ForaNative.ImplValContainer(ForaNative.makeSymbol(symbolString))