def eval(toEvaluate,
locals=None,
parsePath=None,
keepAsForaValue=False,
nameScope="<eval>"):
"""evaluate some FORA code and return the result.
toEvaluate - a string containing FORA code to evaluate
locals - a dictionary of variables that may be free in the expression. If
they are assigned to, 'locals' is updated.
parsePath - context information about the location from which the evaluation
is requested
nameScope - the name scope to use for functions and objects. if the expression is a simple
'fun' or 'object' expression, this is the name it will have
returns a ForaValue.FORAValue or raises a ForaValue.FORAException
"""
binding = {}
if locals:
for key in locals:
value = locals[key]
if isinstance(value, ForaNative.ImplValContainer):
binding[key] = value
if isinstance(value, ForaValue.FORAValue):
binding[key] = value.implVal_
else:
binding[key] = ForaValue.FORAValue(value).implVal_
try:
changes = LOCAL_EVAL.evaluateExpression(toEvaluate, binding, parsePath,
nameScope)
mergedChanges = {}
result = None
for c in changes:
for name, value in c.iteritems():
mergedChanges[name] = value
result = mergedChanges['result']
if isinstance(result, ParseException.ParseException):
raise result
if isinstance(result, ForaValue.FORAException):
result.foraVal = ForaValue.FORAValue(result.foraVal)
if not keepAsForaValue:
result.foraVal = result.foraVal.toPythonObject()
raise result
result = ForaValue.FORAValue(result)
if not keepAsForaValue:
result = result.toPythonObject()
return result
finally:
if locals is not None:
for key in binding:
locals[key] = ForaValue.FORAValue(binding[key])
if not keepAsForaValue:
locals[key] = locals[key].toPythonObject()
def evaluate(self, expr, args, assignments, lets, binding, statementTerm):
exprAsFunction = expr.toFunction()
args = [
exprAsFunction.implVal_, ForaValue.FORAValue.symbol_Call.implVal_
] + args
res = Evaluator.evaluator().evaluate(*args)
# res is a ComputationResult instance, defined in ufora/FORA/Core/ComputationResult.hppml
#@type ComputationResult =
# Exception of ImplValContainer exception
# -| Result of ImplValContainer result
# -| Failure of ErrorState error
resVal = None
if res.isResult():
resVal = res.asResult.result
elif res.isException():
resVal = res.asException.exception
elif res.isFailure():
raise ForaValue.FORAFailure(res.asFailure.error)
# At this point, resVal is an ImplValContainer
resVal = ForaValue.FORAValue(resVal).implVal_
boundValues = {}
if res.isException():
#the exception it a tuple ((exception, (a1, a2, ...)), stacktrace)
#we want to propagate (exception, stacktrace)
exceptionAndVariables, stacktrace = resVal
exception, variableAssignments = exceptionAndVariables
#iterate through the binding and update the original
for ix, a in enumerate(assignments):
boundValues[a] = binding[a] = variableAssignments[ix]
boundValues['result'] = ForaValue.FORAException(
(exception, ForaValue.FORAValue(stacktrace)))
else:
#packAssignedVarsIntoTuple puts the result in the first tuple element
#and the assigned variables in the second
actualResult, boundSymbolValues, assignedOutputs = resVal
#iterate through the binding and update the original
for ix, a in enumerate(lets):
boundValues[a] = binding[a] = boundSymbolValues[ix]
for ix, a in enumerate(assignments):
boundValues[a] = binding[a] = assignedOutputs[ix]
boundValues['result'] = actualResult
return boundValues
def test_module_names(self):
m1 = self.parseAndBind(("builtin", "", [('a', 'f: 10')]))
moduleValue = m1.asModule.result
innerModuleValue = m1['a'].asModule.result
self.assertTrue(moduleValue is not None)
self.assertTrue(innerModuleValue is not None)
self.assertEqual(str(ForaValue.FORAValue(innerModuleValue)),
"builtin.a")
self.assertEqual(str(ForaValue.FORAValue(moduleValue)), "builtin")
def unwrapExceptionIVC(self, exceptionIVC):
try:
return str(ForaValue.FORAValue(exceptionIVC))
except:
logging.error("calling 'str' on %s failed: %s", exceptionIVC,
traceback.format_exc())
return "<unknown exception>"
def importModuleByName(modulename, moduleImportParentList=None):
result = importModuleAndMemberByName(modulename, moduleImportParentList)
if result is not None:
if result[1] is not None:
import ufora.FORA.python.FORA as FORA
return getattr(ForaValue.FORAValue(result[0]), result[1]).implVal_
else:
return result[0]
else:
return None
def importModule(modulePath):
#TODO BUG anybody: why is this here? It was getting passed as the
#searchForFreeVariables argument to importModuleFromPath for some reason
ModuleImporter.builtinModuleImplVal()
return ForaValue.FORAValue(
ModuleImporter.importModuleFromPath(
modulePath,
True
)
)
def initialize(setupObjectToUse=None, useLocalEvaluator=True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def pythonToFORA(x):
"""take a python object 'x' and create the biggest FORA value possible.
copies as much data into FORA as it can.
"""
return ForaValue.FORAValue(ForaNative.pythonToFORA(x))
def makeTag(tagname):
return ForaValue.FORAValue(ForaNative.makeTag(tagname))
def makeSymbol(symbolname):
return ForaValue.FORAValue(ForaNative.makeSymbol(symbolname))
def reloadBuiltin():
global _builtin
ModuleImporter.initialize(reimport=True)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def getResultAsJson(self, *args):
"""If we are over the complexity limit, None, else the result encoded as json"""
if self.computedValue.isFailure:
return None
if self.computedValue.valueIVC is None:
return None
value = self.computedValue.valueIVC
if self.computedValue.isException:
if value.isTuple():
#the first element is the exception. The second element is the stacktrace and variables.
value = value[0]
c = PyforaObjectConverter.PyforaObjectConverter()
#ask the objectConverter to convert this python object to something
#we can send back to the server as json
transformer = PyforaToJsonTransformer.PyforaToJsonTransformer(
self.maxBytecount)
try:
def extractVectorContents(vectorIVC):
if len(vectorIVC) == 0:
return {'listContents': []}
vec = ComputedValue.ComputedValueVector(
vectorImplVal=vectorIVC)
vecSlice = vec.entireSlice
res = None
preventPythonArrayExtraction = False
#see if it's a string. This is the only way to be holding a Vector of char
if vectorIVC.isVectorOfChar():
res = vecSlice.extractVectorDataAsNumpyArray()
if res is not None:
res = {'string': res.tostring()}
#see if it's simple enough to transmit as numpy data
if res is None and len(vectorIVC.getVectorElementsJOR()
) == 1 and len(vectorIVC) > 1:
res = vecSlice.extractVectorDataAsNumpyArrayInChunks()
if res is not None:
firstElement = vecSlice.extractVectorItemAsIVC(0)
if firstElement is None:
#note we can't import this at the top of the file because this file gets imported
#during the build process, which doesn't have pyfora installed.
import pyfora.Exceptions as Exceptions
raise Exceptions.ForaToPythonConversionError(
"Shouldn't be possible to download data as numpy, and then not get the first value"
)
res = {
'firstElement': firstElement,
'contentsAsNumpyArrays': res
}
else:
if not vecSlice.vdmThinksIsLoaded():
#there's a race condition where the data could be loaded between now and
#the call to 'extractVectorDataAsPythonArray'. This prevents it.
preventPythonArrayExtraction = True
#see if we can extract the data as a regular pythonlist
if not preventPythonArrayExtraction and res is None:
res = vecSlice.extractVectorDataAsPythonArray()
if res is not None:
res = {'listContents': res}
if res is None:
vecSlice.increaseRequestCount()
return None
return res
try:
res = c.transformPyforaImplval(value, transformer,
extractVectorContents)
except Exception as e:
import pyfora
if self.computedValue.isException and isinstance(
e, pyfora.ForaToPythonConversionError):
return {
'result': {
"untranslatableException":
str(
ForaValue.FORAValue(
self.computedValue.valueIVC))
},
'isException': True,
'trace':
self.computedValue.exceptionCodeLocationsAsJson
}
elif isinstance(e, pyfora.ForaToPythonConversionError):
return {'foraToPythonConversionError': e.message}
else:
raise
if transformer.anyListsThatNeedLoading:
return None
else:
if self.computedValue.isException:
return {
'result': res,
'isException': True,
'trace':
self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {'result': res, 'isException': False}
except PyforaToJsonTransformer.HaltTransformationException:
if self.computedValue.isException:
return {
'maxBytesExceeded': True,
'isException': True,
'trace': self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {'maxBytesExceeded': True, 'isException': False}
def result(self):
"""If we are over the complexity limit, None, else the result encoded as json"""
if self.computedValue.isFailure:
return None
if self.computedValue.valueIVC is None:
return None
value = self.computedValue.valueIVC
if self.computedValue.isException:
if value.isTuple():
#the first element is the exception. The second element is the stacktrace and variables.
value = value[0]
c = PyforaObjectConverter.PyforaObjectConverter()
#ask the objectConverter to convert this python object to something
#we can send back to the server as json
transformer = PyforaToJsonTransformer.PyforaToJsonTransformer(
self.maxBytecount)
try:
vectorSlicesNeedingLoad = []
def extractVectorContents(vectorIVC):
if len(vectorIVC) == 0:
return []
vec = ComputedValue.ComputedValueVector(
vectorImplVal=vectorIVC)
vecSlice = vec.entireSlice
if vectorIVC.isVectorOfChar():
res = vecSlice.extractVectorDataAsNumpyArray()
if res is not None:
res = res.tostring()
else:
res = vecSlice.extractVectorDataAsPythonArray()
if res is None:
vectorSlicesNeedingLoad.append(vecSlice)
vecSlice.increaseRequestCount()
return None
return res
try:
res = c.transformPyforaImplval(value, transformer,
extractVectorContents)
except Exception as e:
import pyfora
if self.computedValue.isException and isinstance(
e, pyfora.ForaToPythonConversionError):
return {
'result': {
"untranslatableException":
str(
ForaValue.FORAValue(
self.computedValue.valueIVC))
},
'isException': True,
'trace':
self.computedValue.exceptionCodeLocationsAsJson
}
elif isinstance(e, pyfora.ForaToPythonConversionError):
return {'foraToPythonConversionError': e.message}
else:
raise
if transformer.anyListsThatNeedLoading:
return None
else:
if self.computedValue.isException:
return {
'result': res,
'isException': True,
'trace':
self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {'result': res, 'isException': False}
except PyforaToJsonTransformer.HaltTransformationException:
if self.computedValue.isException:
return {
'maxBytesExceeded': True,
'isException': True,
'trace': self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {'maxBytesExceeded': True, 'isException': False}
import ufora.FORA.python.ParseException as ParseException import ufora.FORA.python.StatementTerm as StatementTerm import ufora.FORA.python.Runtime as Runtime import ufora.FORA.python.ExecutionContext as ExecutionContext import ufora.FORA.python.Evaluator.Evaluator as Evaluator import ufora.FORA.python.ForaValue as ForaValue import ufora.FORA.python.ModuleImporter as ModuleImporter import ufora.FORA.VectorDataManager.VectorDataManager as VectorDataManager import logging import traceback Function = ForaNative.Function Nothing = ForaValue.FORAValue(ForaNative.Nothing) nothing = ForaValue.FORAValue(ForaNative.nothing) true = ForaValue.FORAValue(ForaNative.true) false = ForaValue.FORAValue(ForaNative.false) Int64 = ForaValue.FORAValue(ForaNative.Int64) UInt64 = ForaValue.FORAValue(ForaNative.UInt64) Int32 = ForaValue.FORAValue(ForaNative.Int32) UInt32 = ForaValue.FORAValue(ForaNative.UInt32) Int16 = ForaValue.FORAValue(ForaNative.Int16) UInt16 = ForaValue.FORAValue(ForaNative.UInt16) Int8 = ForaValue.FORAValue(ForaNative.Int8) UInt8 = ForaValue.FORAValue(ForaNative.UInt8) UInt1 = ForaValue.FORAValue(ForaNative.UInt1) Bool = UInt1 Float32 = ForaValue.FORAValue(ForaNative.Float32) Float64 = ForaValue.FORAValue(ForaNative.Float64)
def createComputedValue(*strings):
return ComputedValue.ComputedValue(
args=tuple([(FORA.extractImplValContainer(
ForaValue.FORAValue(FORA.eval(x))) if isinstance(x, str) else x)
for x in strings]))
def toFunction(self):
"""convert to a function with one argument per free variable"""
return ForaValue.FORAValue(
self.nativeExpression_.toFunctionImplval(False))
def registerModuleByName(name, moduleValue):
"""registers a value as a globally reachable object in FORA
moduleValue can be anything - if convertable to a python object, it will be
"""
modulesByName_[name] = ForaValue.FORAValue(moduleValue).implVal_
def evaluate(self, expr, args, assignments, lets, binding, statementTerm):
exprAsFunction = expr.toFunction()
args = [exprAsFunction.implVal_, ForaValue.FORAValue.symbol_Call.implVal_] + args
res = Evaluator.evaluator().evaluate(*args)
# res is a ComputationResult instance, defined in ufora/FORA/Core/ComputationResult.hppml
#@type ComputationResult =
# Exception of ImplValContainer exception, ImplValContainer computationLog
# -| Result of ImplValContainer result, ImplValContainer computationLog
# -| Failure of ErrorState error
resVal = None
logs = None
if res.isResult():
resVal = res.asResult.result
logs = res.asResult.computationLog
elif res.isException():
resVal = res.asException.exception
logs = res.asException.computationLog
elif res.isFailure():
raise ForaValue.FORAFailure(res.asFailure.error)
# At this point, resVal and logs are both ImplValContainers.
resVal = ForaValue.FORAValue(resVal).implVal_
if logs is not None and logs.isVector() and logs.getVectorSize() > 0:
if logs.getVectorSize() > 50:
for ix in range(50):
print "log> " + ForaValue.FORAValue(logs)[ix]
print " and", logs.getVectorSize() - 50, "additional log messages..."
else:
print "log> " + ForaValue.FORAValue("\nlog> ").join(ForaValue.FORAValue(logs))
boundValues = {}
if res.isException():
#the exception it a tuple ((exception, (a1, a2, ...)), stacktrace)
#we want to propagate (exception, stacktrace)
exceptionAndVariables, stacktrace = resVal
exception, variableAssignments = exceptionAndVariables
#iterate through the binding and update the original
for ix, a in enumerate(assignments):
boundValues[a] = binding[a] = variableAssignments[ix]
boundValues['result'] = ForaValue.FORAException((exception, ForaValue.FORAValue(stacktrace)))
else:
#packAssignedVarsIntoTuple puts the result in the first tuple element
#and the assigned variables in the second
actualResult, boundSymbolValues, assignedOutputs = resVal
#iterate through the binding and update the original
for ix, a in enumerate(lets):
boundValues[a] = binding[a] = boundSymbolValues[ix]
for ix, a in enumerate(assignments):
boundValues[a] = binding[a] = assignedOutputs[ix]
boundValues['result'] = actualResult
return boundValues
def __init__(self,
callbackScheduler,
sharedStateViewFactory,
computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(
sharedStateViewFactory.createView()
)
logging.info("attached shared state view.")
simultaneously(
initSynchronizer,
initValueGateway
)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(
self.graph,
self.computedValueGateway,
self.synchronizer
)
def test_large_string_alloc_fails_and_raises_foravalue_error(self):
for ix in range(10):
val = ForaValue.FORAValue(self.stringAllocShouldFailFun(ix))
self.assertRaises(ForaValue.FORAFailure, val)
def getResultAsJson(self, *args):
"""If we are over the complexity limit, None, else the result encoded as json"""
if self.computedValue.isFailure:
return None
if self.computedValue.valueIVC is None:
return None
value = self.computedValue.valueIVC
if self.computedValue.isException:
if value.isTuple():
#the first element is the exception. The second element is the stacktrace and variables.
value = value[0]
c = PyforaObjectConverter.PyforaObjectConverter()
try:
def extractVectorContents(vectorIVC):
if len(vectorIVC) == 0:
return {'listContents': []}
#if this is an unpaged vector we can handle it without callback
vdm = ComputedValueGateway.getGateway().vdm
if vdm.vectorDataIsLoaded(vectorIVC, 0, len(vectorIVC)) and vectorIVC.isVectorEntirelyUnpaged():
#see if it's a string. This is the only way to be holding a Vector of char
if vectorIVC.isVectorOfChar():
res = vdm.extractVectorContentsAsNumpyArray(vectorIVC, 0, len(vectorIVC))
assert res is not None
return {'string': res.tostring()}
#see if it's simple enough to transmit as numpy data
if len(vectorIVC.getVectorElementsJOR()) == 1 and len(vectorIVC) > 1:
firstElement = vdm.extractVectorItem(vectorIVC, 0)
if isOfSimpleType(firstElement):
res = vdm.extractVectorContentsAsNumpyArray(vectorIVC, 0, len(vectorIVC))
if res is not None:
assert len(res) == len(vectorIVC)
return {'contentsAsNumpyArray': res}
#see if we can extract the data as a regular pythonlist
res = vdm.extractVectorContentsAsPythonArray(vectorIVC, 0, len(vectorIVC))
assert res is not None
return {'listContents': res}
vec = ComputedValue.ComputedValueVector(vectorImplVal=vectorIVC)
vecSlice = vec.entireSlice
res = None
preventPythonArrayExtraction = False
#see if it's a string. This is the only way to be holding a Vector of char
if vectorIVC.isVectorOfChar():
res = vecSlice.extractVectorDataAsNumpyArray()
if res is not None:
res = {'string': res.tostring()}
#see if it's simple enough to transmit as numpy data
if res is None and len(vectorIVC.getVectorElementsJOR()) == 1 and len(vectorIVC) > 1:
res = vecSlice.extractVectorDataAsNumpyArray()
if res is not None:
firstElement = vecSlice.extractVectorItemAsIVC(0)
if firstElement is None:
#note we can't import this at the top of the file because this file gets imported
#during the build process, which doesn't have pyfora installed.
import pyfora.Exceptions as Exceptions
raise Exceptions.ForaToPythonConversionError(
"Shouldn't be possible to download data as numpy, and then not get the first value"
)
if isOfSimpleType(firstElement):
res = {'contentsAsNumpyArray': res}
else:
res = None
else:
if not vecSlice.vdmThinksIsLoaded():
#there's a race condition where the data could be loaded between now and
#the call to 'extractVectorDataAsPythonArray'. This prevents it.
preventPythonArrayExtraction = True
#see if we can extract the data as a regular pythonlist
if not preventPythonArrayExtraction and res is None:
res = vecSlice.extractVectorDataAsPythonArray()
if res is not None:
res = {'listContents': res}
if res is None:
vecSlice.increaseRequestCount()
return None
return res
try:
import pyfora.BinaryObjectRegistry as BinaryObjectRegistry
stream = BinaryObjectRegistry.BinaryObjectRegistry()
root_id, needsLoading = c.transformPyforaImplval(
value,
stream,
extractVectorContents,
self.maxBytecount
)
if needsLoading:
return None
result_to_send = {'data': base64.b64encode(stream.str()), 'root_id': root_id}
except Exception as e:
import pyfora
if self.computedValue.isException and isinstance(e, pyfora.ForaToPythonConversionError):
return {
'result': {
"untranslatableException": str(ForaValue.FORAValue(self.computedValue.valueIVC))
},
'isException': True,
'trace': self.computedValue.exceptionCodeLocationsAsJson
}
elif isinstance(e, pyfora.ForaToPythonConversionError):
return {
'foraToPythonConversionError': e.message
}
else:
raise
if self.computedValue.isException:
return {
'result': result_to_send,
'isException': True,
'trace': self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {
'result': result_to_send,
'isException': False
}
except PyforaToJsonTransformer.HaltTransformationException:
if self.computedValue.isException:
return {
'maxBytesExceeded': True,
'isException': True,
'trace': self.computedValue.exceptionCodeLocationsAsJson
}
else:
return {'maxBytesExceeded': True, 'isException': False}