コード例 #1
0
def loadHttpDataset(datasetDescriptor, vdid, vdm):
    try:
        data, statusCode = loadHttpRequestDataset(
            datasetDescriptor.asHttpRequestDataset.dataset
            )
        if statusCode != 200:
            raise InvalidDatasetException(
                "Load returned status code %s" % statusCode
                )

        if not vdm.loadByteArrayIntoExternalDatasetPageAsVector(vdid, data):
            raise DatasetLoadException("Couldn't load dataset into VDM")
    except InvalidDatasetException as e:
        if not vdm.loadImplvalIntoUnloadedVectorHandle(
                vdid,
                ForaNative.ImplValContainer(e.message)):
            raise DatasetLoadException("Couldn't load dataset into VDM")
コード例 #2
0
    def test_cached_nodes_3(self):
        text = """fun() {
                      let f = fun() { 0 };
                      let g = fun() { throw 1 };
                      cached(f(), g())
                      }"""
        cfg = self.parseStringToFunction(text).toCFG(0)

        cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
            cfg, None, ())

        cfgWithFutures.slotCompleted(
            0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))

        self.assertEqual(
            cfgWithFutures.getFinalResult().asResult.result.asException.
            exception, ForaNative.ImplValContainer(1))
コード例 #3
0
ファイル: CFGWithFutures_test.py プロジェクト: ufora/ufora
    def test_return_MakeTuple(self):
        text = "fun() { let res = 1 + 1; (res, 1,2,3) }"
        cfg = self.parseStringToFunction(text).toCFG(0)

        cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
            cfg, None, ())

        cfgWithFutures.slotCompleted(
            0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
            )

        cfgWithFutures.continueSimulation()

        self.assertEqual(
            cfgWithFutures.getFinalResult().asResult.result.asResult.result,
            ForaNative.ImplValContainer((2,1,2,3))
            )
コード例 #4
0
def simpleEval(callbackScheduler, *args):
    """convert 'args' to FORA ImplValContainers, evaluates, and returns a python value.
    
    Assumes you don't use cache or vector loads.  If you return an exception, this function
    asserts false. Otherwise, it returns the ImplValContainer result.
    """
    e = ExecutionContext(
        dataManager=VectorDataManager.constructVDM(callbackScheduler))
    e.evaluate(*[FORANative.ImplValContainer(x) for x in args])
    tr = e.getFinishedResult()
    if tr.isFailure():
        assert False
    if tr.isException():
        assert False
    try:
        return tr.asResult.result.pyval
    except:
        return tr.asResult.result
コード例 #5
0
def loadEntireS3Dataset(datasetDescriptor, s3InterfaceFactory, vdid, vdm):
    try:
        vectorSlices = produceVDIDSlicesForEntireS3Dataset(
            s3InterfaceFactory,
            datasetDescriptor.asEntireS3Dataset.dataset,
            vdm
            )

        vector = ForaNative.createFORAFreeBinaryVectorFromSlices(vectorSlices, vdm)

        if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector):
            raise DatasetLoadException("Couldn't load dataset into VDM")
    except InvalidDatasetException as e:
        if not vdm.loadImplvalIntoUnloadedVectorHandle(
                vdid,
                ForaNative.ImplValContainer(e.message)):
            raise DatasetLoadException("Couldn't load dataset into VDM")
    except:
        logging.error("WTF: %s", traceback.format_exc())
コード例 #6
0
    def disabled_garbage_collection_with_switch(self):
        cfg = self.parseStringToFunction(
            "fun(x, f, g) { match (f(x)) with (1) { g(1) } (2) { g(2) } }"
        ).toCFG(3)

        nodeValues = (ForaNative.ImplValContainer(1),
                      FORA.extractImplValContainer(FORA.eval("fun(x) { x }")),
                      FORA.extractImplValContainer(
                          FORA.eval("fun(x) { x + 1 }")))

        serial = serialSimulation(cfg, nodeValues)

        self.assertTrue(serial.getSlots()[0].isGarbageCollected())

        self.seed(10)

        for ix in range(10):
            randomOrder = randomOrderSimulation(cfg, nodeValues)
            self.assertTrue(randomOrder.getSlots()[0].isGarbageCollected())
コード例 #7
0
ファイル: ExecutionContext_test.py プロジェクト: ufora/ufora
    def test_resumingAfterCopyDataOutOfPages(self):
        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInternalSplitting=False)

        text = """
        fun() {
            let v = Vector.range(1000).paged;

            let ix1 = 0
            let res = 0
            while (true) {
                res = res + v[ix1]
                ix1 = (ix1 + 1) % size(v)
                }
            res
            }"""

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (FORA.extractImplValContainer(FORA.eval(text)),
                 FORANative.symbol_Call)))
        context.interruptAfterCycleCount(100000)

        context.compute()

        paused1 = context.extractPausedComputation()

        while not context.isVectorLoad():
            context.copyValuesOutOfVectorPages()
            vdm.unloadAllPossible()
            context.resetInterruptState()
            context.interruptAfterCycleCount(100000)
            context.compute()

        paused2 = context.extractPausedComputation()

        self.assertTrue(
            len(paused1.asThread.computation.frames) == len(
                paused2.asThread.computation.frames))
コード例 #8
0
    def test_futures_5(self):
        funString = "fun(f) { f(0) + f(1) + f(2) }"
        cfg = self.parseStringToFunction(funString).toCFG(1)

        funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x }"))
        cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
            cfg, "block_0Let", (funImplval, ))

        for ix in range(3):
            cfgWithFutures.continueSimulation()

        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(),
                         [0, 1, 3])

        cfgWithFutures.submittableArgs(3)
        cfgWithFutures.slotCompleted(
            3, normalComputationResult(ForaNative.ImplValContainer(2)))
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])

        cfgWithFutures.submittableArgs(1)
        cfgWithFutures.slotCompleted(
            1, normalComputationResult(ForaNative.ImplValContainer(1)))
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])

        cfgWithFutures.submittableArgs(0)
        cfgWithFutures.slotCompleted(
            0, normalComputationResult(ForaNative.ImplValContainer(0)))
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])

        cfgWithFutures.submittableArgs(2)
        cfgWithFutures.slotCompleted(
            2, normalComputationResult(ForaNative.ImplValContainer(2)))
        cfgWithFutures.continueSimulation()

        cfgWithFutures.submittableArgs(4)
        cfgWithFutures.slotCompleted(
            4, normalComputationResult(ForaNative.ImplValContainer(4)))
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])

        cfgWithFutures.continueSimulation()
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])

        self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())

        finalResult = cfgWithFutures.getFinalResult()

        self.assertEqual(finalResult.asResult.result.asResult.result,
                         ForaNative.ImplValContainer(4))
コード例 #9
0
ファイル: CFGWithFutures_test.py プロジェクト: ufora/ufora
    def test_mutable_values_2(self):
        text = "fun(f, g) { let x = f(); let y = g(); 1 }"
        cfg = self.parseStringToFunction(text).toCFG(2)
        fImplval = FORA.extractImplValContainer(
            FORA.eval("let f = fun() { Vector.range(10) }; f")
            )
        gImplval = FORA.extractImplValContainer(
            FORA.eval("let g = fun() { MutableVector(Float64).create(10, 0.0) }; g")
            )

        cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
            cfg, None, (fImplval, gImplval))

        cfgWithFutures.continueSimulation()
        cfgWithFutures.continueSimulation()

        self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])

        cfgWithFutures.slotCompleted(
            1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1))
            )

        self.assertTrue(cfgWithFutures.mustBailEarly())
        self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())

        cfgWithFutures.slotCompleted(
            0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
            )

        self.assertTrue(cfgWithFutures.mustBailEarly())
        self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())

        finalResult = cfgWithFutures.getFinalResult()

        self.assertEqual(
            finalResult.asResult.result.asResult.result,
            ForaNative.ImplValContainer(1)
            )
コード例 #10
0
    def test_cached_nodes_1(self):
        text = "fun(f, g) { cached(f(), g()); }"
        cfg = self.parseStringToFunction(text).toCFG(2)
        fImplval = FORA.extractImplValContainer(FORA.eval("fun() { 1 }"))
        gImplVal = FORA.extractImplValContainer(FORA.eval("fun() { 2 }"))

        cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
            cfg, None, (fImplval, gImplVal))

        self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])

        res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))

        expectedResult = ForaNative.ImplValContainer((1, 2))

        self.assertEqual(res.asResult.result, expectedResult)

        cfgWithFutures.slotCompleted(0, res)

        finalResult = cfgWithFutures.getFinalResult()

        self.assertEqual(finalResult.asResult.result.asResult.result,
                         expectedResult)
コード例 #11
0
    def roundtripExecute(self, pyObject, *args):
        mappings = PureImplementationMappings.PureImplementationMappings()
        binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()

        walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)

        ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)]
        binaryObjectRegistry.defineEndOfStream()

        data = binaryObjectRegistry.str()

        streamReader = PythonBinaryStreamToImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        streamReader.read(data)

        implVals = [streamReader.getObjectById(i) for i in ids]

        result = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
            ForaNative.ImplValContainer((implVals[0],
                                         ForaNative.makeSymbol("Call")) +
                                        tuple(implVals[1:])),
            InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1)

        self.assertTrue(result.isResult(), result)

        result = result.asResult.result

        converter = PythonBinaryStreamFromImplval.constructConverter(
            Converter.canonicalPurePythonModule(), self.vdm)

        root_id, data = converter.write(result)

        rehydrator = PythonObjectRehydrator(
            mappings, allowUserCodeModuleLevelLookups=False)

        return rehydrator.convertEncodedStringToPythonObject(data, root_id)
コード例 #12
0
    def convertClassInstanceDescription(self, objectId,
                                        classInstanceDescription):
        classMemberNameToImplVal = {
            classMemberName: self.convertedValues[memberId]
            for classMemberName, memberId in classInstanceDescription.
            classMemberNameToClassMemberId.iteritems()
        }
        classImplVal = self.convertedValues[classInstanceDescription.classId]

        #note that we need to strip off the first character of membernames defined in the
        #class implval because the object holds 'x' as '@x' so that it doesn't capture
        #all references to 'x'
        classMembersInForaDeclarationOrder = \
            [str(val)[1:] for val in classImplVal.getDataMembers]

        assert set(classMembersInForaDeclarationOrder) == \
            set(classMemberNameToImplVal.keys()), "%s vs %s" % (
                set(classMembersInForaDeclarationOrder),
                set(classMemberNameToImplVal.keys())
                )

        classMemberImplVals = []
        for classMemberName in classMembersInForaDeclarationOrder:
            ivc = classMemberNameToImplVal[classMemberName]
            classMemberImplVals.append(ivc)

        applyArgs = [classImplVal, Symbol_CreateInstance] + classMemberImplVals

        convertedValueOrNone = ForaNative.simulateApply(
            ForaNative.ImplValContainer(tuple(applyArgs)))

        if convertedValueOrNone is None:
            raise pyfora.PythonToForaConversionError(
                ("An internal error occurred: " +
                 "function stage 1 simulation unexpectedly returned None"))

        self.convertedValues[objectId] = convertedValueOrNone
コード例 #13
0
    def createCliqueFinderContext(self, text, *args):
        maxPageSizeInBytes = 32 * 1024
        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(dataManager=vdm)

        argumentImplvals = []

        for a in args:
            context.evaluate(
                FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
                Symbol_Call)

            argumentImplvals.append(
                context.getFinishedResult().asResult.result)

        actualFunction = FORA.extractImplValContainer(FORA.eval(text))

        context.placeInEvaluationState(
            FORANative.ImplValContainer((actualFunction, Symbol_Call) +
                                        tuple(argumentImplvals)))

        return context, argumentImplvals, vdm
コード例 #14
0
    def __init__(self, pyforaBuiltinsModule):
        self.pyforaBuiltinsModule = pyforaBuiltinsModule
        self.pythonNameToInstance = {}
        self.instanceToPythonName = {}

        self.pyExceptionClass = pyforaBuiltinsModule.getObjectMember(
            "PyException")
        self.pyAbortExceptionClass = pyforaBuiltinsModule.getObjectMember(
            "PyAbortException")

        self.pyExceptionClassInstanceName = ForaNative.simulateApply(
            ForaNative.ImplValContainer(
                (self.pyExceptionClass,
                 ForaNative.makeSymbol("CreateInstance"),
                 ForaNative.ImplValContainer(),
                 ForaNative.ImplValContainer()))).getClassName()

        self.pyAbortExceptionClassInstanceName = ForaNative.simulateApply(
            ForaNative.ImplValContainer(
                (self.pyAbortExceptionClass,
                 ForaNative.makeSymbol("CreateInstance"),
                 ForaNative.ImplValContainer(),
                 ForaNative.ImplValContainer()))).getClassName()

        self.pythonNameToPyforaName = {}
        self.pythonNameToPyforaName.update(
            NamedSingletons.pythonNameToPyforaName)
        self.pythonNameToPyforaName.update(
            PyAbortSingletons.pythonNameToPyforaName)

        for pyName, pyforaName in self.pythonNameToPyforaName.iteritems():
            try:
                instance = FORA.ForaValue.FORAValue(
                    pyforaBuiltinsModule).__getattr__(pyforaName).implVal_
            except:
                logging.error(
                    "Initializing PyforaSingletonAndExceptionConverter failed:"
                    + " couldn't find %s in the pyfora builtins", pyforaName)
                raise

            self.pythonNameToInstance[pyName] = instance
            self.instanceToPythonName[instance] = pyName
コード例 #15
0
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import unittest
import time
import ufora.FORA.python.ExecutionContext as ExecutionContext
import ufora.FORA.python.FORA as FORA
import ufora.native.FORA as FORANative
import ufora.native.CallbackScheduler as CallbackScheduler
import logging
import ufora.test.PerformanceTestReporter as PerformanceTestReporter

callbackScheduler = CallbackScheduler.singletonForTesting()

Symbol_Call = FORANative.ImplValContainer(FORANative.makeSymbol("Call"))


class TestFuturesCliqueFinder(unittest.TestCase):
    def createCliqueFinderContext(self, text, *args):
        maxPageSizeInBytes = 32 * 1024
        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(dataManager=vdm)

        argumentImplvals = []

        for a in args:
            context.evaluate(
                FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
コード例 #16
0
 def validator(result):
     return result == ForaNative.ImplValContainer((aVal, aVal))
コード例 #17
0
def normalComputationResult(result):
    return ForaNative.ComputationResult.Result(result,
                                               ForaNative.ImplValContainer())
コード例 #18
0
def createComputationDefinition(*args):
    return CumulusNative.ComputationDefinition.Root(
        CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm([
            CumulusNative.ComputationDefinitionTerm.Value(
                ForaNative.ImplValContainer(x), None) for x in args
        ]))
コード例 #19
0
 def sim(memberName):
     return ForaNative.simulateApply(
         ForaNative.ImplValContainer((ivc, makeSymbolIvc("Member"),
                                      makeSymbolIvc(memberName)))).pyval
コード例 #20
0
 def evaluate(self, *args):
     argIv = FORANative.ImplValContainer(
         self.expandIfListOrTuple(callAndReturn, *args))
     res = self.evaluateDirectly(callAndReturn, *args)
     return FORANative.updateFreeStoreValues(argIv, res)
コード例 #21
0
ファイル: PyforaComputedValue.py プロジェクト: ufora/ufora
 def args(self):
     return (self.baseCV,
             ForaNative.makeSymbol("RawGetItemByString"),
             ForaNative.ImplValContainer(self.keyname))
コード例 #22
0
    def wireWorkersTogether_(self, ix1, ix2):
        worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
        worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)

        if self.channelThroughputMBPerSecond is not None:
            worker1Channel1 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel1)
            worker1Channel2 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel2)

            worker2Channel1 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel1)
            worker2Channel2 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel2)

        self.workersVdmsAndEventHandlers[ix1][0].addMachine(self.machineIds[ix2], [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
        self.workersVdmsAndEventHandlers[ix2][0].addMachine(self.machineIds[ix1], [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
コード例 #23
0
def exceptionComputationResult(result):
    return ForaNative.ComputationResult.Exception(
        result, ForaNative.ImplValContainer())
コード例 #24
0
ファイル: ExecutionContext_test.py プロジェクト: ufora/ufora
def evaluate(context, *args):
    context.placeInEvaluationState(FORANative.ImplValContainer(args))
    context.compute()
コード例 #25
0
 def valueIVC(self):
     if self.valueIVC_ is None:
         return ForaNative.ImplValContainer()
     return self.valueIVC_
コード例 #26
0
ファイル: PyforaComputedValue.py プロジェクト: ufora/ufora
 def args(self):
     return (self.baseCV,
             ForaNative.makeSymbol("RawGetItemByInt"),
             ForaNative.ImplValContainer(self.index))
コード例 #27
0
ファイル: CFGWithFutures_test.py プロジェクト: ufora/ufora
def evalSubmittableArgs(submittableArgs):
    def evalApplyTuple(applyTuple, signature = None):
        varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
        if signature is None:
            evalStringTerms = varNames
        else:
            evalStringTerms = []
            for ix in range(len(signature.terms)):
                term = signature.terms[ix]
                if term.isNormal():
                    name = term.asNormal.name
                    if name is not None:
                        evalStringTerms.append("%s: x_%s" % (name, ix))
                    else:
                        evalStringTerms.append("x_%s" % ix)
                elif term.isTupleCall():
                    evalStringTerms.append("*x_%s" % ix)

        evalString = evalStringTerms[0] + "`(" + ",".join(evalStringTerms[1:]) + ")"

        try:
            return normalComputationResult(
                FORA.extractImplValContainer(
                    FORA.eval(
                        evalString,
                        locals = { name: val
                                   for (name, val) in
                                   zip(varNames, applyTuple) },
                        keepAsForaValue = True
                    )
                )
            )
        except ForaValue.FORAException as e:
            return exceptionComputationResult(
                FORA.extractImplValContainer(e.foraVal)
            )

    if submittableArgs.isApply():
        return evalApplyTuple(
            submittableArgs.args.values,
            submittableArgs.args.signature
        )
    else: # submittableArgs.isCached()
        tr = ()

        # this corresponds to the logic for Cached nodes in
        # CFGWithFutures::asSubmittable
        assert len(submittableArgs.args.values) == 1

        for cachedPair in submittableArgs.args.values[0]:
            applyArgs = [cachedPair[0], ForaNative.makeSymbol("Call")]
            applyArgs.extend(cachedPair[1])

            res = evalApplyTuple(applyArgs)

            if res.isException():
                return res

            tr = tr + (res.asResult.result,)

        return normalComputationResult(ForaNative.ImplValContainer(tr))
コード例 #28
0
 def wireWorkerToClient_(self, ix1, ix2):
     workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
     workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
     self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
     self.clientsAndVdms[ix2][0].addMachine(self.machineIds[ix1], [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
コード例 #29
0
def makeSymbolIvc(symbolString):
    return ForaNative.ImplValContainer(ForaNative.makeSymbol(symbolString))