def onFailure(result):
if isinstance(result, Exception):
onExpanded(result)
else:
onExpanded(
Exceptions.PyforaError(
"Unknown error translating to dictionary of proxies: %s"
+ str(result)))
def getUnresolvedFreeVariableExceptionWithTrace(e, sourceFileName):
chainWithPos = e.freeVarChainWithPos
varLine = chainWithPos.pos.lineno
varName = chainWithPos.var[0]
return UnresolvedFreeVariableExceptionWithTrace(
'''unable to resolve free variable '%s' for pyfora conversion''' %
varName, [Exceptions.makeTraceElement(sourceFileName, varLine)])
def statusChanged(jsonStatus):
if not self.closed:
if jsonStatus is not None:
if jsonStatus['status'] == 'failure':
onFailedCallback(
Exceptions.PyforaError(jsonStatus['message']))
else:
onCompletedCallback(None)
def _translate_download_result(self, jsonResult):
if 'foraToPythonConversionError' in jsonResult:
return Exceptions.ForaToPythonConversionError(
str(jsonResult['foraToPythonConversionError']))
if not jsonResult['isException']:
if 'maxBytesExceeded' in jsonResult:
return Exceptions.ResultExceededBytecountThreshold()
else:
return self.objectRehydrator.convertEncodedStringToPythonObject(
base64.b64decode(jsonResult['result']['data']),
jsonResult['result']['root_id'])
result = self.objectRehydrator.convertEncodedStringToPythonObject(
base64.b64decode(jsonResult['result']['data']),
jsonResult['result']['root_id'])
return Exceptions.ComputationError(result, jsonResult['trace'])
def mapPythonInstanceToPyforaInstance(self, instance):
assert instance is self.instance
if self.pureClass is None:
raise Exceptions.PyforaNotImplementedError(
"conversion of '%s' not yet implemented" %
(instance.__name__
if hasattr(instance, "__name__") else instance))
return self.pureClass()
def convertJsonResultToPythonObject(self, jsonResult):
if 'primitive' in jsonResult:
res = jsonResult['primitive']
if isinstance(res, unicode):
return intern(str(res))
else:
return res
if 'tuple' in jsonResult:
return tuple([self.convertJsonResultToPythonObject(x) for x in jsonResult['tuple']])
if 'list' in jsonResult:
return [self.convertJsonResultToPythonObject(x) for x in jsonResult['list']]
if 'dict' in jsonResult:
return {
self.convertJsonResultToPythonObject(key): self.convertJsonResultToPythonObject(val) \
for key, val in zip(jsonResult['dict']['keys'], jsonResult['dict']['values'])
}
if 'untranslatableException' in jsonResult:
return Exceptions.ForaToPythonConversionError(
"untranslatable FORA exception: %s" % jsonResult['untranslatableException']
)
if 'singleton' in jsonResult:
singletonName = jsonResult['singleton']
return NamedSingletons.singletonNameToObject[singletonName]
if 'InvalidPyforaOperation' in jsonResult:
return Exceptions.InvalidPyforaOperation(jsonResult['InvalidPyforaOperation'])
if 'builtinException' in jsonResult:
builtinExceptionTypeName = jsonResult['builtinException']
builtinExceptionType = NamedSingletons.singletonNameToObject[builtinExceptionTypeName]
args = self.convertJsonResultToPythonObject(jsonResult['args'])
return builtinExceptionType(*args)
if 'classInstance' in jsonResult:
members = {k:self.convertJsonResultToPythonObject(v) for k,v in jsonResult['members'].iteritems()}
classObject = self.convertJsonResultToPythonObject(jsonResult['classInstance'])
return self._invertPureClassInstanceIfNecessary(self._instantiateClass(classObject, members))
if 'functionInstance' in jsonResult:
members = {k:self.convertJsonResultToPythonObject(v) for k,v in jsonResult['members'].iteritems()}
return self._instantiateFunction(jsonResult['functionInstance'][0], jsonResult['functionInstance'][1], members)
if 'classObject' in jsonResult:
members = {k:self.convertJsonResultToPythonObject(v) for k,v in jsonResult['members'].iteritems()}
return self._classObjectFromFilenameAndLine(jsonResult['classObject'][0], jsonResult['classObject'][1], members)
if 'stacktrace' in jsonResult:
return jsonResult['stacktrace']
raise Exceptions.ForaToPythonConversionError("not implemented: cant convert %s" % jsonResult)
def _cachedCompute(self):
if not self._isComputed:
if PyAstUtil.isScopeNode(self._root):
self.generic_visit(self._root)
self._isComputed = True
else:
raise Exceptions.InternalError(
"'%s' called on unsupported node-type (%s)" %
(self.__class__.__name__, type(self._root)))
def getPurePythonTypes(self):
"""Return the pure-python type that this mapping converts to.
This should return a list of python classes that this mapper knows how to invert.
"""
#subclasses should implement
raise Exceptions.PyforaNotImplementedError(
"'%s' not implemented yet for type '%s'" %
(self.getPurePythonTypes.__name__, type(self).__name__))
def toLocal(self):
"""Downloads the remote object.
Returns:
A :class:`~pyfora.Future.Future` that resolves to the python
object that this :class:`RemotePythonObject` represents.
"""
raise Exceptions.PyforaNotImplementedError(
"'%s' not implemented yet for type '%s'" %
(self.toLocal.__name__, type(self).__name__))
def _subchainAndResolutionOrNone(self, pyObject, chainWithPosition):
if PyforaInspect.isfunction(pyObject):
return self._lookupChainInFunction(pyObject, chainWithPosition)
if PyforaInspect.isclass(pyObject):
return self._lookupChainInClass(pyObject, chainWithPosition)
raise Exceptions.PythonToForaConversionError(
"don't know how to resolve %s in %s (line:%s)" %
(chainWithPosition.var, pyObject, chainWithPosition.pos.lineno))
def collectPossiblyUninitializedLocalVariables(pyAstNode):
"""Returns the possibly free local variables of the code rooted at `pyAstNode."""
# Context doesn't play a role below, but we reuse the code for checking `pyAstNode
if not PyAstUtil.isScopeNode(pyAstNode):
raise Exceptions.InternalError(
"Unsupported type of root node in Analysis (%s)" % type(pyAstNode))
possiblyUninitVisitor = _PossiblyUninitializedScopedVisitor()
possiblyUninitVisitor.visit(pyAstNode)
return possiblyUninitVisitor.getPossiblyUninitializedLocalVariables()
def _convertUnresolvedFreeVariableExceptionAndRaise(e, sourceFileName):
logging.error(
"Converter raised an UnresolvedFreeVariableException exception: %s",
traceback.format_exc())
chainWithPos = e.freeVarChainWithPos
varLine = chainWithPos.pos.lineno
varName = chainWithPos.var[0]
raise UnresolvedFreeVariableExceptionWithTrace(
'''unable to resolve free variable '%s' for pyfora conversion''' %
varName, [Exceptions.makeTraceElement(sourceFileName, varLine)])
def getMappablePythonTypes(self):
"""Return the python type that this mapping knows how to convert.
This should return a list of python types and classes. Any time the
converter sees an instance of one of these types, Pyfora will invoke
this class to provide an alternate, translatable form of the instance.
"""
#subclasses should implement
raise Exceptions.PyforaNotImplementedError(
"'%s' not implemented yet for type '%s'" %
(self.getMappablePythonTypes.__name__, type(self).__name__))
def getSourceFilenameAndText(pyObject):
try:
sourceFile = PyforaInspect.getsourcefile(pyObject)
except TypeError as e:
raise Exceptions.CantGetSourceTextError(e.message)
if sourceFile not in sourceFileCache_:
sourceFileCache_[sourceFile] = "".join(
PyforaInspect.getlines(sourceFile))
return sourceFileCache_[sourceFile], sourceFile
def _convertUnresolvedFreeVariableExceptionAndRaise(e, sourceFileName):
logging.error(
"Converter raised an UnresolvedFreeVariableException exception: %s",
traceback.format_exc())
chainWithPos = e.freeVarChainWithPos
varLine = chainWithPos.pos.lineno
varName = chainWithPos.var[0]
raise UnresolvedFreeVariableExceptionWithTrace(
'''unable to resolve free variable '%s' for pyfora conversion''' % varName,
[Exceptions.makeTraceElement(sourceFileName, varLine)]
)
def getSourceText(pyObject):
try:
source, lineno = PyforaInspect.getsourcelines(pyObject)
except TypeError as e:
raise Exceptions.CantGetSourceTextError(e.message)
# Create a prefix of (lineno-1) blank lines to keep track of line numbers for error reporting
blankLines = os.linesep * (lineno - 1)
# We don't know how to avoid the use of `textwrap.dedent to get the code
# though `ast.parse, which means that the computed column_numbers may be
# off and we shouldn't report them.
return textwrap.dedent(blankLines + "".join(source))
def _computeSubchainAndTerminalValueAlongModules(self, rootValue, chain):
ix = 1
subchain, terminalValue = chain[:ix], rootValue
while PyforaInspect.ismodule(terminalValue):
if ix >= len(chain):
#we're terminating at a module
raise Exceptions.PythonToForaConversionError(
"Can't convert the module %s" % str(terminalValue))
if not hasattr(terminalValue, chain[ix]):
raise Exceptions.PythonToForaConversionError(
"Module %s has no member %s" %
(str(terminalValue), chain[ix]))
terminalValue = getattr(terminalValue, chain[ix])
ix += 1
subchain = chain[:ix]
return subchain, terminalValue
def _collectDataMembersSetInInitAst(initAst):
_assertOnlySimpleStatements(initAst)
if len(initAst.args.args) == 0:
raise Exceptions.PythonToForaConversionError(
"the `__init__ method is missing a first, positional, " \
"`self argument (line %s)."
% (initAst.lineno)
)
selfArg = initAst.args.args[0]
if not isinstance(selfArg, ast.Name):
raise Exceptions.InternalError(
"the `self argument to the `__init__ method" \
" is not of type `ast.Name (line %s)."
% (initAst.lineno)
)
return _extractSimpleSelfMemberAssignments(initFunctionDef=initAst,
selfName=selfArg.id)
def extractVectorContents(vectorIVC):
if len(vectorIVC) == 0:
return {'listContents': []}
vec = ComputedValue.ComputedValueVector(
vectorImplVal=vectorIVC)
vecSlice = vec.entireSlice
res = None
preventPythonArrayExtraction = False
#see if it's a string. This is the only way to be holding a Vector of char
if vectorIVC.isVectorOfChar():
res = vecSlice.extractVectorDataAsNumpyArray()
if res is not None:
res = {'string': res.tostring()}
#see if it's simple enough to transmit as numpy data
if res is None and len(vectorIVC.getVectorElementsJOR()
) == 1 and len(vectorIVC) > 1:
res = vecSlice.extractVectorDataAsNumpyArrayInChunks()
if res is not None:
firstElement = vecSlice.extractVectorItemAsIVC(0)
if firstElement is None:
#note we can't import this at the top of the file because this file gets imported
#during the build process, which doesn't have pyfora installed.
import pyfora.Exceptions as Exceptions
raise Exceptions.ForaToPythonConversionError(
"Shouldn't be possible to download data as numpy, and then not get the first value"
)
res = {
'firstElement': firstElement,
'contentsAsNumpyArrays': res
}
else:
if not vecSlice.vdmThinksIsLoaded():
#there's a race condition where the data could be loaded between now and
#the call to 'extractVectorDataAsPythonArray'. This prevents it.
preventPythonArrayExtraction = True
#see if we can extract the data as a regular pythonlist
if not preventPythonArrayExtraction and res is None:
res = vecSlice.extractVectorDataAsPythonArray()
if res is not None:
res = {'listContents': res}
if res is None:
vecSlice.increaseRequestCount()
return None
return res
def getSourceFilenameAndText(pyObject):
try:
sourceFile = PyforaInspect.getsourcefile(pyObject)
except TypeError as e:
raise Exceptions.CantGetSourceTextError(e.message)
if sourceFile in sourceFileCache_:
return sourceFileCache_[sourceFile], sourceFile
with open(sourceFile, "r") as f:
tr = f.read()
sourceFileCache_[sourceFile] = tr
return tr, sourceFile
def _cachedCompute(self):
if not self._isComputed:
if isScopeNode(self._root):
if isinstance(self._root, ast.FunctionDef) and hasattr(
self._root, 'arguments'):
self._root.arguments.lineno = self._root.lineno
self._root.arguments.col_offset = self._root.col_offset
self.generic_visit(self._root)
self._isComputed = True
else:
raise Exceptions.InternalError(
"'%s' called on unsupported node-type (%s)" %
(self.__class__.__name__, type(self._root)))
def getSourceFilenameAndText(pyObject):
try:
sourceFile = PyforaInspect.getsourcefile(pyObject)
except TypeError as e:
raise Exceptions.CantGetSourceTextError(e.message)
if sourceFile is None:
raise Exceptions.CantGetSourceTextError(
"can't get source lines for file %s" % sourceFile
)
linesOrNone = PyforaInspect.getlines(sourceFile)
if linesOrNone is None:
raise Exceptions.CantGetSourceTextError(
"can't get source lines for file %s" % sourceFile
)
if sourceFile not in sourceFileCache_:
sourceFileCache_[sourceFile] = "".join(linesOrNone)
return sourceFileCache_[sourceFile], sourceFile
def onExpanded(jsonResult):
result = jsonResult
if not isinstance(jsonResult, Exception):
if jsonResult['isException']:
result = Exceptions.ComputationError(
self.objectRehydrator.convertJsonResultToPythonObject(
jsonResult['result']), jsonResult['trace'])
else:
assert isinstance(jsonResult['dictOfProxies'], dict)
result = {
k: RemotePythonObject.ComputedRemotePythonObject(
v, self, False)
for k, v in jsonResult['dictOfProxies'].iteritems()
}
self._resolve_future(future, result)
def onExpanded(jsonResult):
result = jsonResult
if not isinstance(jsonResult, Exception):
if jsonResult['isException']:
result = Exceptions.ComputationError(
self.objectRehydrator.convertJsonResultToPythonObject(
jsonResult['result']), jsonResult['trace'])
else:
assert isinstance(jsonResult['tupleOfComputedValues'],
tuple)
result = tuple(
RemotePythonObject.ComputedRemotePythonObject(
val, self, False)
for val in jsonResult['tupleOfComputedValues'])
self._resolve_future(future, result)
def populateModuleMembers(self, path):
if path in self.moduleClassesAndFunctionsByPath:
return
res = self.moduleClassesAndFunctionsByPath[path] = {}
module = self.moduleForFile(path)
if module is not None and self.canPopulateForPath(path):
for leafItemName in module.__dict__:
leafItemValue = module.__dict__[leafItemName]
if PyforaInspect.isclass(
leafItemValue) or PyforaInspect.isfunction(
leafItemValue):
try:
sourcePath = PyforaInspect.getsourcefile(leafItemValue)
if sourcePath is not None:
if os.path.samefile(path, sourcePath):
_, lineNumber = PyforaInspect.findsource(
leafItemValue)
lineNumberToUse = lineNumber + 1
if lineNumberToUse in res and res[
lineNumberToUse] is not leafItemValue:
raise Exceptions.ForaToPythonConversionError(
("PythonObjectRehydratorHelpers got a line number collision at lineNumber %s"
", between %s and %s"),
lineNumberToUse, leafItemValue,
res[lineNumber + 1])
res[lineNumberToUse] = leafItemValue
else:
self.populateModuleMembers(sourcePath)
except Exceptions.ForaToPythonConversionError:
raise
except PyforaInspect.PyforaInspectError:
pass
except IOError:
#this gets raised when PyforaInspect can't find a file it needs
pass
except Exception as e:
logging.critical(
"PyforaInspect threw an exception: %s. tb = %s", e,
traceback.format_exc())
def _resolveChainByDict(self, chain, boundVariables):
freeVariable = chain[0]
if freeVariable in boundVariables:
rootValue = boundVariables[freeVariable]
subchain, terminalValue = self._computeSubchainAndTerminalValueAlongModules(
rootValue, chain)
return subchain, terminalValue
if hasattr(__builtin__, freeVariable):
rootValue = getattr(__builtin__, freeVariable)
return self._computeSubchainAndTerminalValueAlongModules(
rootValue, chain)
raise Exceptions.PythonToForaConversionError(
"don't know how to resolve free variable `%s`" % freeVariable)
def submit(self, fn, *args, **kwargs):
"""Submits a callable to be executed on the server with the provided arguments 'args'.
kwargs are not currently supported.
Returns:
A Future representing the given call. The future will eventually resolve to a
RemotePythonObject instance.
"""
self._raiseIfClosed()
if len(kwargs) > 0:
raise Exceptions.PyforaNotImplementedError(
"Keyword arguments not supported yet")
# TODO: make this truly async
# don't block on the 'define' calls
futures = [self.define(fn)] + [self.define(arg) for arg in args]
results = [f.result() for f in futures]
return results[0](*results[1:])
def onExpanded(jsonResult):
if isinstance(jsonResult, Exception):
future.set_exception(jsonResult)
return
if jsonResult['isException']:
result = self.objectRehydrator.convertJsonResultToPythonObject(
jsonResult['result'])
future.set_exception(
Exceptions.ComputationError(result, jsonResult['trace']))
return
assert isinstance(jsonResult['tupleOfComputedValues'], tuple)
tupleOfProxies = \
tuple([
RemotePythonObject.ComputedRemotePythonObject(val, self) \
for val in jsonResult['tupleOfComputedValues']
])
future.set_result(tupleOfProxies)
def _resolveChainInPyObject(self, chain, pyObject):
"""
This name could be improved.
Returns a `subchain, terminalPyValue` pair: this represents the deepest value
we can get to in the member chain `chain` on `pyObject` taking members only
along modules (or "empty" modules)
"""
subchainAndResolutionOrNone = self._subchainAndResolutionOrNone(
pyObject, chain)
if subchainAndResolutionOrNone is None:
raise Exceptions.PythonToForaConversionError(
"don't know how to resolve %s in %s" % (chain, pyObject))
subchain, terminalValue = subchainAndResolutionOrNone
if id(terminalValue) in self._convertedObjectCache:
terminalValue = self._convertedObjectCache[id(terminalValue)][1]
return subchain, terminalValue
def onExpanded(jsonResult):
if isinstance(jsonResult, Exception):
future.set_exception(jsonResult)
return
if jsonResult['isException']:
result = self.objectRehydrator.convertJsonResultToPythonObject(
jsonResult['result'])
future.set_exception(
Exceptions.ComputationError(result, jsonResult['trace']))
return
assert isinstance(jsonResult['dictOfProxies'], dict)
dictOfProxies = {}
for k, v in jsonResult['dictOfProxies'].iteritems():
dictOfProxies[
k] = RemotePythonObject.ComputedRemotePythonObject(
v, self)
future.set_result(dictOfProxies)
def onResultCallback(jsonResult):
result = jsonResult
try:
if not isinstance(jsonResult, Exception):
result = self._translate_download_result(jsonResult)
except Exception as e:
# TODO need a better way of wrapping exceptions.
# Alexandros has some ideas here, but this is
# better than the experience without the wrapping
# (which is hanging)
def clip(s):
if len(s) > 250:
return s[:250] + "... (" + str(
len(s) - 250) + " characters remaining)"
return s
logging.error(
"Rehydration failed: %s\nResult was %s of type %s",
traceback.format_exc(), clip(repr(jsonResult)),
type(jsonResult))
result = Exceptions.ForaToPythonConversionError(e)
self._resolve_future(future, result)
def _classOrFunctionDefinition(self, pyObject, classOrFunction):
"""
`_classOrFunctionDefinition: create a `_FunctionDefinition` or
`_ClassDefinition` out of a python class or function, recursively visiting
the resolvable free variable member access chains in `pyObject` as well
as the source file object.
Args:
`pyObject`: a python class or function.
`classOrFunction`: should either be `_FunctionDefinition` or
`_ClassDefinition`.
Returns:
a `_FunctionDefinition` or `_ClassDefinition`.
"""
if pyObject.__name__ == '__inline_fora':
raise Exceptions.PythonToForaConversionError(
"in pyfora, '__inline_fora' is a reserved word"
)
sourceFileText, sourceFileName = PyAstUtil.getSourceFilenameAndText(pyObject)
_, sourceLine = PyAstUtil.getSourceLines(pyObject)
sourceAst = PyAstUtil.pyAstFromText(sourceFileText)
if classOrFunction is _FunctionDefinition:
pyAst = PyAstUtil.functionDefOrLambdaAtLineNumber(sourceAst, sourceLine)
else:
assert classOrFunction is _ClassDefinition
pyAst = PyAstUtil.classDefAtLineNumber(sourceAst, sourceLine)
assert sourceLine == pyAst.lineno
try:
freeVariableMemberAccessChainResolutions = \
self._computeAndResolveFreeVariableMemberAccessChainsInAst(
pyObject, pyAst
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(e, sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, location) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(
path=sourceFileName,
lineNumber=location[0]
)
)
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(
fileName=sourceFileName,
fileText=sourceFileText
)
)
return classOrFunction(
sourceFileId=sourceFileId,
lineNumber=sourceLine,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions
)
def _registerWithBlock(self, objectId, pyObject):
"""
`_registerWithBlock`: register a `PyforaWithBlock.PyforaWithBlock`
with `self.objectRegistry`.
Recursively call `walkPyObject` on the resolvable free variable
member access chains in the block and on the file object.
"""
lineNumber = pyObject.lineNumber
sourceTree = PyAstUtil.pyAstFromText(pyObject.sourceText)
withBlockAst = PyAstUtil.withBlockAtLineNumber(sourceTree, lineNumber)
withBlockFun = ast.FunctionDef(
name="",
args=ast.arguments(args=[], defaults=[], kwarg=None, vararg=None),
body=withBlockAst.body,
decorator_list=[],
lineno=lineNumber,
col_offset=0
)
if PyAstUtil.hasReturnInOuterScope(withBlockFun):
raise Exceptions.BadWithBlockError(
"return statement not supported in pyfora with-block (line %s)" %
PyAstUtil.getReturnLocationsInOuterScope(withBlockFun)[0])
if PyAstUtil.hasYieldInOuterScope(withBlockFun):
raise Exceptions.BadWithBlockError(
"yield expression not supported in pyfora with-block (line %s)" %
PyAstUtil.getYieldLocationsInOuterScope(withBlockFun)[0])
freeVariableMemberAccessChainsWithPositions = \
self._freeMemberAccessChainsWithPositions(withBlockFun)
boundValuesInScopeWithPositions = \
PyAstFreeVariableAnalyses.collectBoundValuesInScope(
withBlockFun, getPositions=True)
for boundValueWithPosition in boundValuesInScopeWithPositions:
val, pos = boundValueWithPosition
if val not in pyObject.unboundLocals and val in pyObject.boundVariables:
freeVariableMemberAccessChainsWithPositions.add(
PyAstFreeVariableAnalyses.VarWithPosition(var=(val,), pos=pos)
)
try:
freeVariableMemberAccessChainResolutions = \
self._resolveFreeVariableMemberAccessChains(
freeVariableMemberAccessChainsWithPositions, pyObject.boundVariables
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(e, pyObject.sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, position) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(
path=pyObject.sourceFileName,
lineNumber=position.lineno
)
)
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(
fileName=pyObject.sourceFileName
)
)
self._objectRegistry.defineWithBlock(
objectId=objectId,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions,
sourceFileId=sourceFileId,
lineNumber=lineNumber
)
def addToTrace(self, elmt):
Exceptions.checkTraceElement(elmt)
self.trace.insert(0, elmt)
