def loadHttpDataset(datasetDescriptor, vdid, vdm):
try:
data, statusCode = loadHttpRequestDataset(
datasetDescriptor.asHttpRequestDataset.dataset
)
if statusCode != 200:
raise InvalidDatasetException(
"Load returned status code %s" % statusCode
)
if not vdm.loadByteArrayIntoExternalDatasetPageAsVector(vdid, data):
raise DatasetLoadException("Couldn't load dataset into VDM")
except InvalidDatasetException as e:
if not vdm.loadImplvalIntoUnloadedVectorHandle(
vdid,
ForaNative.ImplValContainer(e.message)):
raise DatasetLoadException("Couldn't load dataset into VDM")
def test_cached_nodes_3(self):
text = """fun() {
let f = fun() { 0 };
let g = fun() { throw 1 };
cached(f(), g())
}"""
cfg = self.parseStringToFunction(text).toCFG(0)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, ())
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))
self.assertEqual(
cfgWithFutures.getFinalResult().asResult.result.asException.
exception, ForaNative.ImplValContainer(1))
def test_return_MakeTuple(self):
text = "fun() { let res = 1 + 1; (res, 1,2,3) }"
cfg = self.parseStringToFunction(text).toCFG(0)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, ())
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
cfgWithFutures.continueSimulation()
self.assertEqual(
cfgWithFutures.getFinalResult().asResult.result.asResult.result,
ForaNative.ImplValContainer((2,1,2,3))
)
def simpleEval(callbackScheduler, *args):
"""convert 'args' to FORA ImplValContainers, evaluates, and returns a python value.
Assumes you don't use cache or vector loads. If you return an exception, this function
asserts false. Otherwise, it returns the ImplValContainer result.
"""
e = ExecutionContext(
dataManager=VectorDataManager.constructVDM(callbackScheduler))
e.evaluate(*[FORANative.ImplValContainer(x) for x in args])
tr = e.getFinishedResult()
if tr.isFailure():
assert False
if tr.isException():
assert False
try:
return tr.asResult.result.pyval
except:
return tr.asResult.result
def loadEntireS3Dataset(datasetDescriptor, s3InterfaceFactory, vdid, vdm):
try:
vectorSlices = produceVDIDSlicesForEntireS3Dataset(
s3InterfaceFactory,
datasetDescriptor.asEntireS3Dataset.dataset,
vdm
)
vector = ForaNative.createFORAFreeBinaryVectorFromSlices(vectorSlices, vdm)
if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector):
raise DatasetLoadException("Couldn't load dataset into VDM")
except InvalidDatasetException as e:
if not vdm.loadImplvalIntoUnloadedVectorHandle(
vdid,
ForaNative.ImplValContainer(e.message)):
raise DatasetLoadException("Couldn't load dataset into VDM")
except:
logging.error("WTF: %s", traceback.format_exc())
def disabled_garbage_collection_with_switch(self):
cfg = self.parseStringToFunction(
"fun(x, f, g) { match (f(x)) with (1) { g(1) } (2) { g(2) } }"
).toCFG(3)
nodeValues = (ForaNative.ImplValContainer(1),
FORA.extractImplValContainer(FORA.eval("fun(x) { x }")),
FORA.extractImplValContainer(
FORA.eval("fun(x) { x + 1 }")))
serial = serialSimulation(cfg, nodeValues)
self.assertTrue(serial.getSlots()[0].isGarbageCollected())
self.seed(10)
for ix in range(10):
randomOrder = randomOrderSimulation(cfg, nodeValues)
self.assertTrue(randomOrder.getSlots()[0].isGarbageCollected())
def test_resumingAfterCopyDataOutOfPages(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
text = """
fun() {
let v = Vector.range(1000).paged;
let ix1 = 0
let res = 0
while (true) {
res = res + v[ix1]
ix1 = (ix1 + 1) % size(v)
}
res
}"""
context.placeInEvaluationState(
FORANative.ImplValContainer(
(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)))
context.interruptAfterCycleCount(100000)
context.compute()
paused1 = context.extractPausedComputation()
while not context.isVectorLoad():
context.copyValuesOutOfVectorPages()
vdm.unloadAllPossible()
context.resetInterruptState()
context.interruptAfterCycleCount(100000)
context.compute()
paused2 = context.extractPausedComputation()
self.assertTrue(
len(paused1.asThread.computation.frames) == len(
paused2.asThread.computation.frames))
def test_futures_5(self):
funString = "fun(f) { f(0) + f(1) + f(2) }"
cfg = self.parseStringToFunction(funString).toCFG(1)
funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval, ))
for ix in range(3):
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(),
[0, 1, 3])
cfgWithFutures.submittableArgs(3)
cfgWithFutures.slotCompleted(
3, normalComputationResult(ForaNative.ImplValContainer(2)))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(1)))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(0)))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
cfgWithFutures.submittableArgs(2)
cfgWithFutures.slotCompleted(
2, normalComputationResult(ForaNative.ImplValContainer(2)))
cfgWithFutures.continueSimulation()
cfgWithFutures.submittableArgs(4)
cfgWithFutures.slotCompleted(
4, normalComputationResult(ForaNative.ImplValContainer(4)))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(4))
def test_mutable_values_2(self):
text = "fun(f, g) { let x = f(); let y = g(); 1 }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(
FORA.eval("let f = fun() { Vector.range(10) }; f")
)
gImplval = FORA.extractImplValContainer(
FORA.eval("let g = fun() { MutableVector(Float64).create(10, 0.0) }; g")
)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, gImplval))
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1))
)
self.assertTrue(cfgWithFutures.mustBailEarly())
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
self.assertTrue(cfgWithFutures.mustBailEarly())
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(1)
)
def test_cached_nodes_1(self):
text = "fun(f, g) { cached(f(), g()); }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun() { 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun() { 2 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, gImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
expectedResult = ForaNative.ImplValContainer((1, 2))
self.assertEqual(res.asResult.result, expectedResult)
cfgWithFutures.slotCompleted(0, res)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
expectedResult)
def roundtripExecute(self, pyObject, *args):
mappings = PureImplementationMappings.PureImplementationMappings()
binaryObjectRegistry = BinaryObjectRegistry.BinaryObjectRegistry()
walker = PyObjectWalker.PyObjectWalker(mappings, binaryObjectRegistry)
ids = [walker.walkPyObject(o) for o in [pyObject] + list(args)]
binaryObjectRegistry.defineEndOfStream()
data = binaryObjectRegistry.str()
streamReader = PythonBinaryStreamToImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
streamReader.read(data)
implVals = [streamReader.getObjectById(i) for i in ids]
result = InMemoryCumulusSimulation.computeUsingSeveralWorkers(
ForaNative.ImplValContainer((implVals[0],
ForaNative.makeSymbol("Call")) +
tuple(implVals[1:])),
InMemoryS3Interface.InMemoryS3InterfaceFactory(), 1)
self.assertTrue(result.isResult(), result)
result = result.asResult.result
converter = PythonBinaryStreamFromImplval.constructConverter(
Converter.canonicalPurePythonModule(), self.vdm)
root_id, data = converter.write(result)
rehydrator = PythonObjectRehydrator(
mappings, allowUserCodeModuleLevelLookups=False)
return rehydrator.convertEncodedStringToPythonObject(data, root_id)
def convertClassInstanceDescription(self, objectId,
classInstanceDescription):
classMemberNameToImplVal = {
classMemberName: self.convertedValues[memberId]
for classMemberName, memberId in classInstanceDescription.
classMemberNameToClassMemberId.iteritems()
}
classImplVal = self.convertedValues[classInstanceDescription.classId]
#note that we need to strip off the first character of membernames defined in the
#class implval because the object holds 'x' as '@x' so that it doesn't capture
#all references to 'x'
classMembersInForaDeclarationOrder = \
[str(val)[1:] for val in classImplVal.getDataMembers]
assert set(classMembersInForaDeclarationOrder) == \
set(classMemberNameToImplVal.keys()), "%s vs %s" % (
set(classMembersInForaDeclarationOrder),
set(classMemberNameToImplVal.keys())
)
classMemberImplVals = []
for classMemberName in classMembersInForaDeclarationOrder:
ivc = classMemberNameToImplVal[classMemberName]
classMemberImplVals.append(ivc)
applyArgs = [classImplVal, Symbol_CreateInstance] + classMemberImplVals
convertedValueOrNone = ForaNative.simulateApply(
ForaNative.ImplValContainer(tuple(applyArgs)))
if convertedValueOrNone is None:
raise pyfora.PythonToForaConversionError(
("An internal error occurred: " +
"function stage 1 simulation unexpectedly returned None"))
self.convertedValues[objectId] = convertedValueOrNone
def createCliqueFinderContext(self, text, *args):
maxPageSizeInBytes = 32 * 1024
vdm = FORANative.VectorDataManager(callbackScheduler,
maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
argumentImplvals = []
for a in args:
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
Symbol_Call)
argumentImplvals.append(
context.getFinishedResult().asResult.result)
actualFunction = FORA.extractImplValContainer(FORA.eval(text))
context.placeInEvaluationState(
FORANative.ImplValContainer((actualFunction, Symbol_Call) +
tuple(argumentImplvals)))
return context, argumentImplvals, vdm
def __init__(self, pyforaBuiltinsModule):
self.pyforaBuiltinsModule = pyforaBuiltinsModule
self.pythonNameToInstance = {}
self.instanceToPythonName = {}
self.pyExceptionClass = pyforaBuiltinsModule.getObjectMember(
"PyException")
self.pyAbortExceptionClass = pyforaBuiltinsModule.getObjectMember(
"PyAbortException")
self.pyExceptionClassInstanceName = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(self.pyExceptionClass,
ForaNative.makeSymbol("CreateInstance"),
ForaNative.ImplValContainer(),
ForaNative.ImplValContainer()))).getClassName()
self.pyAbortExceptionClassInstanceName = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(self.pyAbortExceptionClass,
ForaNative.makeSymbol("CreateInstance"),
ForaNative.ImplValContainer(),
ForaNative.ImplValContainer()))).getClassName()
self.pythonNameToPyforaName = {}
self.pythonNameToPyforaName.update(
NamedSingletons.pythonNameToPyforaName)
self.pythonNameToPyforaName.update(
PyAbortSingletons.pythonNameToPyforaName)
for pyName, pyforaName in self.pythonNameToPyforaName.iteritems():
try:
instance = FORA.ForaValue.FORAValue(
pyforaBuiltinsModule).__getattr__(pyforaName).implVal_
except:
logging.error(
"Initializing PyforaSingletonAndExceptionConverter failed:"
+ " couldn't find %s in the pyfora builtins", pyforaName)
raise
self.pythonNameToInstance[pyName] = instance
self.instanceToPythonName[instance] = pyName
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import time
import ufora.FORA.python.ExecutionContext as ExecutionContext
import ufora.FORA.python.FORA as FORA
import ufora.native.FORA as FORANative
import ufora.native.CallbackScheduler as CallbackScheduler
import logging
import ufora.test.PerformanceTestReporter as PerformanceTestReporter
callbackScheduler = CallbackScheduler.singletonForTesting()
Symbol_Call = FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
class TestFuturesCliqueFinder(unittest.TestCase):
def createCliqueFinderContext(self, text, *args):
maxPageSizeInBytes = 32 * 1024
vdm = FORANative.VectorDataManager(callbackScheduler,
maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
argumentImplvals = []
for a in args:
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
def validator(result):
return result == ForaNative.ImplValContainer((aVal, aVal))
def normalComputationResult(result):
return ForaNative.ComputationResult.Result(result,
ForaNative.ImplValContainer())
def createComputationDefinition(*args):
return CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm([
CumulusNative.ComputationDefinitionTerm.Value(
ForaNative.ImplValContainer(x), None) for x in args
]))
def sim(memberName):
return ForaNative.simulateApply(
ForaNative.ImplValContainer((ivc, makeSymbolIvc("Member"),
makeSymbolIvc(memberName)))).pyval
def evaluate(self, *args):
argIv = FORANative.ImplValContainer(
self.expandIfListOrTuple(callAndReturn, *args))
res = self.evaluateDirectly(callAndReturn, *args)
return FORANative.updateFreeStoreValues(argIv, res)
def args(self):
return (self.baseCV,
ForaNative.makeSymbol("RawGetItemByString"),
ForaNative.ImplValContainer(self.keyname))
def wireWorkersTogether_(self, ix1, ix2):
worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
if self.channelThroughputMBPerSecond is not None:
worker1Channel1 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel1)
worker1Channel2 = self.rateLimitedChannelGroupsForEachListener[ix1].wrap(worker1Channel2)
worker2Channel1 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel1)
worker2Channel2 = self.rateLimitedChannelGroupsForEachListener[ix2].wrap(worker2Channel2)
self.workersVdmsAndEventHandlers[ix1][0].addMachine(self.machineIds[ix2], [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
self.workersVdmsAndEventHandlers[ix2][0].addMachine(self.machineIds[ix1], [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def exceptionComputationResult(result):
return ForaNative.ComputationResult.Exception(
result, ForaNative.ImplValContainer())
def evaluate(context, *args):
context.placeInEvaluationState(FORANative.ImplValContainer(args))
context.compute()
def valueIVC(self):
if self.valueIVC_ is None:
return ForaNative.ImplValContainer()
return self.valueIVC_
def args(self):
return (self.baseCV,
ForaNative.makeSymbol("RawGetItemByInt"),
ForaNative.ImplValContainer(self.index))
def evalSubmittableArgs(submittableArgs):
def evalApplyTuple(applyTuple, signature = None):
varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
if signature is None:
evalStringTerms = varNames
else:
evalStringTerms = []
for ix in range(len(signature.terms)):
term = signature.terms[ix]
if term.isNormal():
name = term.asNormal.name
if name is not None:
evalStringTerms.append("%s: x_%s" % (name, ix))
else:
evalStringTerms.append("x_%s" % ix)
elif term.isTupleCall():
evalStringTerms.append("*x_%s" % ix)
evalString = evalStringTerms[0] + "`(" + ",".join(evalStringTerms[1:]) + ")"
try:
return normalComputationResult(
FORA.extractImplValContainer(
FORA.eval(
evalString,
locals = { name: val
for (name, val) in
zip(varNames, applyTuple) },
keepAsForaValue = True
)
)
)
except ForaValue.FORAException as e:
return exceptionComputationResult(
FORA.extractImplValContainer(e.foraVal)
)
if submittableArgs.isApply():
return evalApplyTuple(
submittableArgs.args.values,
submittableArgs.args.signature
)
else: # submittableArgs.isCached()
tr = ()
# this corresponds to the logic for Cached nodes in
# CFGWithFutures::asSubmittable
assert len(submittableArgs.args.values) == 1
for cachedPair in submittableArgs.args.values[0]:
applyArgs = [cachedPair[0], ForaNative.makeSymbol("Call")]
applyArgs.extend(cachedPair[1])
res = evalApplyTuple(applyArgs)
if res.isException():
return res
tr = tr + (res.asResult.result,)
return normalComputationResult(ForaNative.ImplValContainer(tr))
def wireWorkerToClient_(self, ix1, ix2):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient(clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
self.clientsAndVdms[ix2][0].addMachine(self.machineIds[ix1], [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def makeSymbolIvc(symbolString):
return ForaNative.ImplValContainer(ForaNative.makeSymbol(symbolString))
