def test_TransvisitorsDontModifyTree(self):
tree = ast.parse(self.some_python_code)
# deep-copy tree because transformers modify the AST in place
# making the test of areAstsIdentical(tree, tree') meaningless
tree1 = copy.deepcopy(tree)
noopTransformer = NodeVisitorBases.SemanticOrderNodeTransvisitor()
tree2 = noopTransformer.visit(tree1)
self.assertIsNotNone(tree2)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree2))
scopeMgr = NodeVisitorBases.InScopeSaveRestoreValue(
lambda: True, lambda x: None)
noopTransformer = NodeVisitorBases.GenericScopedTransvisitor(scopeMgr)
tree3 = noopTransformer.visit(tree1)
self.assertIsNotNone(tree3)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree3))
freeVarsVisitor = PyAstFreeVariableAnalyses._FreeVariableMemberAccessChainsTransvisitor(
)
tree4 = freeVarsVisitor.visit(tree1)
self.assertIsNotNone(tree4)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree4))
def test_collectDataMembersSetInInit_error_2(self):
class E2:
def __init__(*args):
self.x = 0
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstUtil.collectDataMembersSetInInit(E2)
def test_collectDataMembersSetInInit_error_2(self):
class E2:
def __init__(*args):
self.x = 0
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstUtil.collectDataMembersSetInInit(E2)
def _classObjectFromFilenameAndLine(self, filename, lineNumber, members):
"""Construct a class object given its textual definition."""
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.getAstFromFilePath(filename)
classAst = PyAstUtil.classDefAtLineNumber(sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(members)
self.importModuleMagicVariables(globalScope, filename)
try:
code = compile(ast.Module([classAst]), filename, 'exec')
exec code in globalScope, outputLocals
except:
logging.error("Failed to instantiate class at %s:%s\n%s", filename, lineNumber, traceback.format_exc())
raise Exceptions.PyforaError("Failed to instantiate class at %s:%s" % (filename, lineNumber))
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def _instantiateFunction(self, filename, lineNumber, memberDictionary):
"""Instantiate a function instance."""
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.getAstFromFilePath(filename)
functionAst = PyAstUtil.functionDefOrLambdaAtLineNumber(
sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(memberDictionary)
self.importModuleMagicVariables(globalScope, filename)
if isinstance(functionAst, ast.Lambda):
expr = ast.Expression()
expr.body = functionAst
expr.lineno = functionAst.lineno
expr.col_offset = functionAst.col_offset
code = compile(expr, filename, 'eval')
return eval(code, globalScope, outputLocals)
else:
code = compile(ast.Module([functionAst]), filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def _instantiateFunction(self, filename, lineNumber, memberDictionary):
"""Instantiate a function instance."""
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.getAstFromFilePath(filename)
functionAst = PyAstUtil.functionDefOrLambdaAtLineNumber(sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(memberDictionary)
self.importModuleMagicVariables(globalScope, filename)
if isinstance(functionAst, ast.Lambda):
expr = ast.Expression()
expr.body = functionAst
expr.lineno = functionAst.lineno
expr.col_offset = functionAst.col_offset
code = compile(expr, filename, 'eval')
return eval(code, globalScope, outputLocals)
else:
code = compile(ast.Module([functionAst]), filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def _classObjectFromFilenameAndLine(self, filename, lineNumber, members):
"""Construct a class object given its textual definition."""
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.getAstFromFilePath(filename)
classAst = PyAstUtil.classDefAtLineNumber(sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(members)
self.importModuleMagicVariables(globalScope, filename)
try:
code = compile(ast.Module([classAst]), filename, 'exec')
exec code in globalScope, outputLocals
except:
logging.error("Failed to instantiate class at %s:%s\n%s", filename,
lineNumber, traceback.format_exc())
raise Exceptions.PyforaError(
"Failed to instantiate class at %s:%s" %
(filename, lineNumber))
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def test_collectDataMembersSetInInit_error_3(self):
class E3:
def __init__(self):
self.x = 0
def f(x):
return x
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstUtil.collectDataMembersSetInInit(E3)
def test_collectDataMembersSetInInit_error_4(self):
class E4:
def __init__(self):
self.x = 0
class c(object):
def __init__(self):
self.z = 0
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstUtil.collectDataMembersSetInInit(E4)
def augmentRaiseFunction(raiseFunction, path, line, col):
codeAst = PyAstUtil.getAstFromFilePath(path)
enclosingFunctionName = PyAstUtil.findEnclosingFunctionName(codeAst, line)
vis = AugmentRaiseFunctionModificationVisitor(line, col, enclosingFunctionName)
module = ast.parse(augmentRaiseFunctionTemplate)
vis.visit(module)
code = compile(module, path, 'exec')
exec code in globals(), locals()
return _augmentRaiseFunctionTempl(raiseFunction)
def augmentRaiseFunction(raiseFunction, path, line, col):
codeAst = PyAstUtil.getAstFromFilePath(path)
enclosingFunctionName = PyAstUtil.findEnclosingFunctionName(codeAst, line)
vis = AugmentRaiseFunctionModificationVisitor(line, col,
enclosingFunctionName)
module = ast.parse(augmentRaiseFunctionTemplate)
vis.visit(module)
code = compile(module, path, 'exec')
exec code in globals(), locals()
return _augmentRaiseFunctionTempl(raiseFunction)
def test_hasReturnInOuterScope(self):
def f():
x = 0
return x
if x:
return x
else:
return
def f():
yield 4
class D1:
def f(self):
return 0
for x in xrange(3):
while False:
return x
else:
return x
else:
return x
x = [f() for _ in xrange(1000) if f() > 0]
return None
ast = PyAstUtil.pyAstFor(f)
self.assertEqual(PyAstUtil.countReturnsInOuterScope(ast.body[0]), 7)
self.assertEqual(PyAstUtil.countYieldsInOuterScope(ast.body[0]), 0)
self.assertTrue(PyAstUtil.hasReturnInOuterScope(ast.body[0]))
self.assertFalse(PyAstUtil.hasYieldInOuterScope(ast.body[0]))
self.assertTrue(PyAstUtil.hasReturnOrYieldInOuterScope(ast.body[0]))
returnLocs = PyAstUtil.getReturnLocationsInOuterScope(ast.body[0])
yieldLocs = PyAstUtil.getYieldLocationsInOuterScope(ast.body[0])
returnLocs = [x - returnLocs[0] for x in returnLocs]
yieldLocs = [x - yieldLocs[0] for x in yieldLocs]
self.assertEqual(returnLocs, [0, 2, 4, 12, 14, 16, 18])
self.assertEqual(yieldLocs, [])
def test_returnFunctions(self):
y = 2
def toReturn(x):
return x * y
def f():
return toReturn
shouldBeToReturn = self.evaluateWithExecutor(f)
self.assertEqual(shouldBeToReturn(10), toReturn(10))
self.assertEqual(str(shouldBeToReturn.__name__), str(toReturn.__name__))
self.assertEqual(PyAstUtil.getSourceText(shouldBeToReturn), PyAstUtil.getSourceText(toReturn))
def test_countYieldsInOuterScope(self):
def f():
x = 0
yield x
if x:
yield x
else:
yield
def f():
return 4
class D1:
def f(self):
yield 0
for x in xrange(3):
while False:
yield x
else:
yield x
else:
yield x
x = [f() for _ in xrange(1000) if f() > 0]
yield None
ast = PyAstUtil.pyAstFor(f)
self.assertEqual(PyAstUtil.countReturnsInOuterScope(ast.body[0]), 0)
self.assertEqual(PyAstUtil.countYieldsInOuterScope(ast.body[0]), 7)
self.assertFalse(PyAstUtil.hasReturnInOuterScope(ast.body[0]))
self.assertTrue(PyAstUtil.hasYieldInOuterScope(ast.body[0]))
self.assertTrue(PyAstUtil.hasReturnOrYieldInOuterScope(ast.body[0]))
returnLocs = PyAstUtil.getReturnLocationsInOuterScope(ast.body[0])
yieldLocs = PyAstUtil.getYieldLocationsInOuterScope(ast.body[0])
returnLocs = map(lambda x: x - returnLocs[0] if len(returnLocs) > 0 else 0, returnLocs)
yieldLocs = map(lambda x: x - yieldLocs[0] if len(yieldLocs) > 0 else 0, yieldLocs)
self.assertEqual(returnLocs, [])
self.assertEqual(yieldLocs, [0, 2, 4, 12, 14, 16, 18])
def test_returnFunctions(self):
y = 2
def toReturn(x):
return x * y
def f():
return toReturn
shouldBeToReturn = self.evaluateWithExecutor(f)
self.assertEqual(shouldBeToReturn(10), toReturn(10))
self.assertEqual(str(shouldBeToReturn.__name__),
str(toReturn.__name__))
self.assertEqual(PyAstUtil.getSourceText(shouldBeToReturn),
PyAstUtil.getSourceText(toReturn))
def _registerClassInstance(self, objectId, classInstance):
"""
`_registerClassInstance`: register a `class` instance
with `self.objectRegistry`.
Recursively call `walkPyObject` on the class of the `classInstance`
and on the data members of the instance.
"""
classObject = classInstance.__class__
classId = self.walkPyObject(classObject)
try:
dataMemberNames = PyAstUtil.collectDataMembersSetInInit(
classObject)
except Exceptions.CantGetSourceTextError:
self._raiseConversionErrorForSourceTextError(classInstance)
except:
logging.error('Failed on %s (of type %s)', classInstance,
type(classInstance))
raise
classMemberNameToClassMemberId = {}
for dataMemberName in dataMemberNames:
memberId = self.walkPyObject(getattr(classInstance,
dataMemberName))
classMemberNameToClassMemberId[dataMemberName] = memberId
self._objectRegistry.defineClassInstance(
objectId=objectId,
classId=classId,
classMemberNameToClassMemberId=classMemberNameToClassMemberId)
def _registerClassInstance(self, objectId, classInstance):
"""
`_registerClassInstance`: register a `class` instance
with `self.objectRegistry`.
Recursively call `walkPyObject` on the class of the `classInstance`
and on the data members of the instance.
"""
classObject = classInstance.__class__
classId = self.walkPyObject(classObject)
try:
dataMemberNames = PyAstUtil.collectDataMembersSetInInit(classObject)
except Exceptions.CantGetSourceTextError:
self._raiseConversionErrorForSourceTextError(classInstance)
except:
logging.error('Failed on %s (of type %s)',
classInstance, type(classInstance))
raise
classMemberNameToClassMemberId = {}
for dataMemberName in dataMemberNames:
memberId = self.walkPyObject(getattr(classInstance, dataMemberName))
classMemberNameToClassMemberId[dataMemberName] = memberId
self._objectRegistry.defineClassInstance(
objectId=objectId,
classId=classId,
classMemberNameToClassMemberId=classMemberNameToClassMemberId
)
def _getNativePythonFunctionDefFromWithBlockDescription(self,
withBlockDescription,
objectIdToObjectDefinition):
sourceText = objectIdToObjectDefinition[withBlockDescription.sourceFileId].text
sourceLineOffsets = PythonAstConverter.computeLineOffsets(sourceText)
sourceTree = ast.parse(sourceText)
withTree = PyAstUtil.withBlockAtLineNumber(
sourceTree,
withBlockDescription.lineNumber
)
withBodyAsFunctionAst = ast.FunctionDef(
name="__withBodyFunction",
lineno=withTree.lineno,
col_offset=withTree.col_offset,
args=ast.arguments(
args=[],
vararg=None,
kwarg=None,
defaults=[]),
body=withTree.body,
decorator_list=[]
)
nativeWithBodyAst = PythonAstConverter.convertPythonAstToForaPythonAst(
withBodyAsFunctionAst,
sourceLineOffsets
)
return nativeWithBodyAst
def getFreeVariableMemberAccessChains(pyAstNode,
isClassContext=None,
getPositions=False):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
vis = _FreeVariableMemberAccessChainsVisitor()
vis.visit(pyAstNode)
return vis.getFreeVariablesMemeberAccessChains(getPositions)
def getFreeVariableMemberAccessChains(pyAstNode,
isClassContext=None,
getPositions=False,
exclude_predicate=None):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
vis = _FreeVariableMemberAccessChainsTransvisitor(exclude_predicate)
vis.visit(pyAstNode)
return vis.getFreeVariablesMemberAccessChains(getPositions)
def getFreeVariableMemberAccessChains(pyAstNode,
isClassContext=None,
getPositions=False,
exclude_predicate=None):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
vis = _FreeVariableMemberAccessChainsVisitor(exclude_predicate)
vis.visit(pyAstNode)
return vis.getFreeVariablesMemeberAccessChains(getPositions)
def test_collectDataMembersSetInInit_3(self):
class C3:
def __init__(self):
self.x = self.y = 0
dataMembers = PyAstUtil.collectDataMembersSetInInit(C3)
self.assertEqual(set(dataMembers), set(['x', 'y']))
def test_collectDataMembersSetInInit_4(self):
class C4:
def __init__(self, arg):
(self.x, self.y), self.z = arg
dataMembers = PyAstUtil.collectDataMembersSetInInit(C4)
self.assertEqual(set(dataMembers), set(['x', 'y', 'z']))
def test_collectDataMembersSetInInit_1(self):
class C:
def __init__(self, x):
self.x = x
self.y = x**2.0
dataMembers = PyAstUtil.collectDataMembersSetInInit(C)
self.assertEqual(set(dataMembers), set(['x', 'y']))
def test_collectDataMembersSetInInit_2(self):
class C2:
def __init__(self, x):
if x > 0:
self.x = x
dataMembers = PyAstUtil.collectDataMembersSetInInit(C2)
# in our translation, we're _always_ producing an x member
self.assertEqual(set(dataMembers), set(['x']))
def test_collectDataMembersSetInInit_4(self):
class C4:
def __init__(self, arg):
(self.x, self.y), self.z = arg
dataMembers = PyAstUtil.collectDataMembersSetInInit(C4)
self.assertEqual(
dataMembers,
set(['x', 'y', 'z'])
)
def test_collectDataMembersSetInInit_3(self):
class C3:
def __init__(self):
self.x = self.y = 0
dataMembers = PyAstUtil.collectDataMembersSetInInit(C3)
self.assertEqual(
dataMembers,
set(['x', 'y'])
)
def test_collectDataMembersSetInInit_1(self):
class C:
def __init__(self, x):
self.x = x
self.y = x ** 2.0
dataMembers = PyAstUtil.collectDataMembersSetInInit(C)
self.assertEqual(
dataMembers,
set(['x', 'y'])
)
def test_FreeVariableTransformer_3(self):
tree = ast.parse(
textwrap.dedent("""
def g(y):
return x.y.z.__str__()
"""))
# deep-copy tree because transformers modify the AST in place
# making the test of areAstsIdentical(tree, tree') meaningless
tree1 = copy.deepcopy(tree)
tree2 = PyAstFreeVariableAnalyses.collapseFreeVariableMemberAccessChains(
tree1, {('x', 'y', 'z'): 'x_y_z'}, isClassContext=False)
self.assertIsNotNone(tree2)
self.assertFalse(PyAstUtil.areAstsIdentical(tree, tree2))
def test_collectDataMembersSetInInit_2(self):
class C2:
def __init__(self, x):
if x > 0:
self.x = x
dataMembers = PyAstUtil.collectDataMembersSetInInit(C2)
# in our translation, we're _always_ producing an x member
self.assertEqual(
dataMembers,
set(['x'])
)
def test_freeVariablesMemberAccessChain_onFunctionDefNode(self):
tree1 = ast.parse(
textwrap.dedent("""
def g(arg):
if arg < 0:
return x + arg
return x * h(arg - 1, g)
"""))
res = PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(
tree1)
self.assertEqual(set([('h', ), ('x', )]), res)
tree2 = PyAstUtil.functionDefOrLambdaAtLineNumber(tree1, 2)
self.assertEqual(
set([('h', ), ('x', )]),
PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(
tree2, False))
def test_freeVariablesMemberAccessChain_onFunctionDefNode(self):
tree1 = ast.parse(
textwrap.dedent(
"""
def g(arg):
if arg < 0:
return x + arg
return x * h(arg - 1, g)
"""
)
)
res = PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree1)
self.assertEqual(set([("h",), ("x",)]), res)
tree2 = PyAstUtil.functionDefOrLambdaAtLineNumber(tree1, 2)
self.assertEqual(
set([("h",), ("x",)]), PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree2, False)
)
def test_hasReturnInOuterScope(self):
def f():
x = 0
return x
if x:
return x
else:
return
def f():
yield 4
class D1:
def f(self):
return 0
for x in xrange(3):
while False:
return x
else:
return x
else:
return x
x = [f() for _ in xrange(1000) if f() > 0]
return None
ast = PyAstUtil.pyAstFor(f)
self.assertEqual(PyAstUtil.countReturnsInOuterScope(ast.body[0]), 7)
self.assertEqual(PyAstUtil.countYieldsInOuterScope(ast.body[0]), 0)
self.assertTrue(PyAstUtil.hasReturnInOuterScope(ast.body[0]))
self.assertFalse(PyAstUtil.hasYieldInOuterScope(ast.body[0]))
self.assertTrue(PyAstUtil.hasReturnOrYieldInOuterScope(ast.body[0]))
returnLocs = PyAstUtil.getReturnLocationsInOuterScope(ast.body[0])
yieldLocs = PyAstUtil.getYieldLocationsInOuterScope(ast.body[0])
returnLocs = [x - returnLocs[0] for x in returnLocs]
yieldLocs = [x - yieldLocs[0] for x in yieldLocs]
self.assertEqual(returnLocs, [0, 2, 4, 12, 14, 16, 18])
self.assertEqual(yieldLocs, [])
def _registerClassInstance(self, objectId, classInstance):
"""
`_registerClassInstance`: register a `class` instance
with `self.objectRegistry`.
Recursively call `walkPyObject` on the class of the `classInstance`
and on the data members of the instance.
"""
classObject = classInstance.__class__
classId = self.walkPyObject(classObject)
dataMemberNames = classInstance.__dict__.keys() if hasattr(classInstance, '__dict__') \
else PyAstUtil.collectDataMembersSetInInit(classObject)
classMemberNameToClassMemberId = {}
for dataMemberName in dataMemberNames:
memberId = self.walkPyObject(getattr(classInstance, dataMemberName))
classMemberNameToClassMemberId[dataMemberName] = memberId
self._objectRegistry.defineClassInstance(
objectId=objectId,
classId=classId,
classMemberNameToClassMemberId=classMemberNameToClassMemberId
)
def _registerClassInstance(self, objectId, classInstance):
"""
`_registerClassInstance`: register a `class` instance
with `self.objectRegistry`.
Recursively call `walkPyObject` on the class of the `classInstance`
and on the data members of the instance.
"""
classObject = classInstance.__class__
classId = self.walkPyObject(classObject)
dataMemberNames = classInstance.__dict__.keys() if hasattr(classInstance, '__dict__') \
else PyAstUtil.collectDataMembersSetInInit(classObject)
classMemberNameToClassMemberId = {}
for dataMemberName in dataMemberNames:
memberId = self.walkPyObject(getattr(classInstance,
dataMemberName))
classMemberNameToClassMemberId[dataMemberName] = memberId
self._objectRegistry.defineClassInstance(
objectId=objectId,
classId=classId,
classMemberNameToClassMemberId=classMemberNameToClassMemberId)
def getFreeVariables(pyAstNode, isClassContext=None, getPositions=False):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
freeVarsVisitor = _FreeVarsVisitor()
freeVarsVisitor.visit(pyAstNode)
return freeVarsVisitor.getFreeVars(getPositions=getPositions)
def _registerWithBlock(self, objectId, pyObject):
"""
`_registerWithBlock`: register a `PyforaWithBlock.PyforaWithBlock`
with `self.objectRegistry`.
Recursively call `walkPyObject` on the resolvable free variable
member access chains in the block and on the file object.
"""
lineNumber = pyObject.lineNumber
sourceTree = PyAstUtil.pyAstFromText(pyObject.sourceText)
withBlockAst = PyAstUtil.withBlockAtLineNumber(sourceTree, lineNumber)
withBlockFun = ast.FunctionDef(name="",
args=ast.arguments(args=[],
defaults=[],
kwarg=None,
vararg=None),
body=withBlockAst.body,
decorator_list=[],
lineno=lineNumber,
col_offset=0)
if PyAstUtil.hasReturnInOuterScope(withBlockFun):
raise Exceptions.InvalidPyforaOperation(
"return statement not supported in pyfora with-block (line %s)"
% PyAstUtil.getReturnLocationsInOuterScope(withBlockFun)[0])
if PyAstUtil.hasYieldInOuterScope(withBlockFun):
raise Exceptions.InvalidPyforaOperation(
"yield expression not supported in pyfora with-block (line %s)"
% PyAstUtil.getYieldLocationsInOuterScope(withBlockFun)[0])
freeVariableMemberAccessChainsWithPositions = \
self._freeMemberAccessChainsWithPositions(withBlockFun)
boundValuesInScopeWithPositions = \
PyAstFreeVariableAnalyses.collectBoundValuesInScope(
withBlockFun, getPositions=True)
for boundValueWithPosition in boundValuesInScopeWithPositions:
val, pos = boundValueWithPosition
if val not in pyObject.unboundLocals and val in pyObject.boundVariables:
freeVariableMemberAccessChainsWithPositions.add(
PyAstFreeVariableAnalyses.VarWithPosition(var=(val, ),
pos=pos))
try:
freeVariableMemberAccessChainResolutions = \
self._resolveFreeVariableMemberAccessChains(
freeVariableMemberAccessChainsWithPositions, pyObject.boundVariables
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(
e, pyObject.sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, position) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(path=pyObject.sourceFileName,
lineNumber=position.lineno))
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(fileName=pyObject.sourceFileName))
self._objectRegistry.defineWithBlock(
objectId=objectId,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions,
sourceFileId=sourceFileId,
lineNumber=lineNumber
)
def instantiateFunction(self, filename, lineNumber, memberDictionary,
file_text):
"""Instantiate a function instance."""
memberDictionary = {
k: v
for k, v in memberDictionary.iteritems()
if not isinstance(v, Exceptions.PyforaNameError)
}
objectOrNone = self.moduleLevelObject(filename, lineNumber)
if objectOrNone is not None:
return objectOrNone
sourceAst = PyAstUtil.pyAstFromText(file_text)
functionAst = PyAstUtil.functionDefOrLambdaOrWithBlockAtLineNumber(
sourceAst, lineNumber)
outputLocals = {}
globalScope = {}
globalScope.update(memberDictionary)
self.importModuleMagicVariables(globalScope, filename)
if isinstance(functionAst, ast.Lambda):
expr = ast.FunctionDef()
expr.name = '__pyfora_lambda_builder__'
expr.args = ast.arguments()
expr.args.args = []
expr.args.defaults = []
expr.args.vararg = None
expr.args.kwarg = None
expr.decorator_list = []
expr.lineno = functionAst.lineno - 1
expr.col_offset = functionAst.col_offset
return_statement = ast.Return(functionAst)
expr.body = [return_statement]
expr = updatePyAstMemberChains(ast.Module([expr],
lineno=1,
col_offset=0),
globalScope,
isClassContext=True)
code = compile(expr, filename, 'exec')
exec code in globalScope, outputLocals
return list(outputLocals.values())[0]()
elif isinstance(functionAst, ast.With):
expr = ast.FunctionDef()
expr.name = '__pyfora_with_block_as_function__'
expr.args = ast.arguments()
expr.args.args = []
expr.args.defaults = []
expr.args.vararg = None
expr.args.kwarg = None
expr.decorator_list = []
#make sure we copy the list - if we use the existing one, we will mess up the
#cached copy!
expr.body = list(functionAst.body)
expr.lineno = functionAst.lineno - 1
expr.col_offset = functionAst.col_offset
bound_variables = PyAstFreeVariableAnalyses.collectBoundValuesInScope(
expr)
return_dict_list_src = "[" + ",".join(
"'%s'" % k for k in bound_variables) + "]"
return_dict_expr_src = "{{k: __locals[k] for k in {return_dict_list_src} if k in __locals}}".format(
return_dict_list_src=return_dict_list_src)
return_dict_expr_src = "(lambda __locals: {return_dict_expr_src})(dict(locals()))".format(
return_dict_expr_src=return_dict_expr_src)
return_dict_expr = ast.parse(return_dict_expr_src).body[0].value
return_statement = ast.Return(
ast.Tuple(
[return_dict_expr,
ast.Num(0), ast.Num(0)], ast.Load()))
return_statement_exception = ast.Return(
ast.Tuple([
return_dict_expr,
ast.Call(
ast.Name("__pyfora_get_exception_traceback__",
ast.Load()), [], [], None, None),
ast.Name("__pyfora_exception_var__", ast.Load())
], ast.Load()))
expr.body.append(return_statement)
handler = ast.ExceptHandler(
None, ast.Name("__pyfora_exception_var__", ast.Store()),
[return_statement_exception])
#now wrap in a try-catch block
curBody = list(expr.body)
expr.body = [ast.TryExcept(curBody, [handler], [])]
#for every incoming variable 'x' that's also assigned to, create a dummy '__pyfora_var_guard_x' that actually
#takes the value in from the surrounding scope, and immediately assign it
for var in memberDictionary:
if var in bound_variables:
newVar = "__pyfora_var_guard_" + var
var_copy_expr = ast.Assign(
targets=[ast.Name(var, ast.Store())],
value=ast.Name(newVar, ast.Load()))
globalScope[newVar] = globalScope[var]
del globalScope[var]
expr.body = [var_copy_expr] + expr.body
expr = updatePyAstMemberChains(expr,
globalScope,
isClassContext=True)
ast.fix_missing_locations(expr)
def extractTrace():
return sys.exc_info()[2]
globalScope['__pyfora_get_exception_traceback__'] = extractTrace
code = compile(ast.Module([expr]), filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
else:
functionAst = updatePyAstMemberChains(ast.Module([functionAst],
lineno=1,
col_offset=0),
globalScope,
isClassContext=False)
code = compile(functionAst, filename, 'exec')
exec code in globalScope, outputLocals
assert len(outputLocals) == 1
return list(outputLocals.values())[0]
def _classOrFunctionDefinition(self, pyObject, classOrFunction):
"""
`_classOrFunctionDefinition: create a `_FunctionDefinition` or
`_ClassDefinition` out of a python class or function, recursively visiting
the resolvable free variable member access chains in `pyObject` as well
as the source file object.
Args:
`pyObject`: a python class or function.
`classOrFunction`: should either be `_FunctionDefinition` or
`_ClassDefinition`.
Returns:
a `_FunctionDefinition` or `_ClassDefinition`.
"""
if pyObject.__name__ == '__inline_fora':
raise Exceptions.PythonToForaConversionError(
"in pyfora, '__inline_fora' is a reserved word")
try:
sourceFileText, sourceFileName = PyAstUtil.getSourceFilenameAndText(
pyObject)
except Exceptions.CantGetSourceTextError:
self._raiseConversionErrorForSourceTextError(pyObject)
except:
logging.error('Failed on %s (of type %s)', pyObject,
type(pyObject))
raise
_, sourceLine = PyforaInspect.getsourcelines(pyObject)
sourceAst = PyAstUtil.pyAstFromText(sourceFileText)
if classOrFunction is _FunctionDefinition:
pyAst = PyAstUtil.functionDefOrLambdaAtLineNumber(
sourceAst, sourceLine)
else:
assert classOrFunction is _ClassDefinition
pyAst = PyAstUtil.classDefAtLineNumber(sourceAst, sourceLine)
try:
freeVariableMemberAccessChainResolutions = \
self._computeAndResolveFreeVariableMemberAccessChainsInAst(
pyObject, pyAst
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(e, sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, location) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(path=sourceFileName,
lineNumber=location[0]))
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(fileName=sourceFileName,
fileText=sourceFileText))
return classOrFunction(
sourceFileId=sourceFileId,
lineNumber=sourceLine,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions
)
def collapseFreeVariableMemberAccessChains(pyAstNode,
chain_to_name,
isClassContext=None):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
vis = _FreeVariableMemberAccessChainsCollapsingTransformer(chain_to_name)
return ast.fix_missing_locations(vis.visit(pyAstNode))
def _classOrFunctionDefinition(self, pyObject, classOrFunction):
"""
`_classOrFunctionDefinition: create a `_FunctionDefinition` or
`_ClassDefinition` out of a python class or function, recursively visiting
the resolvable free variable member access chains in `pyObject` as well
as the source file object.
Args:
`pyObject`: a python class or function.
`classOrFunction`: should either be `_FunctionDefinition` or
`_ClassDefinition`.
Returns:
a `_FunctionDefinition` or `_ClassDefinition`.
"""
if pyObject.__name__ == '__inline_fora':
raise Exceptions.PythonToForaConversionError(
"in pyfora, '__inline_fora' is a reserved word"
)
sourceFileText, sourceFileName = PyAstUtil.getSourceFilenameAndText(pyObject)
_, sourceLine = PyAstUtil.getSourceLines(pyObject)
sourceAst = PyAstUtil.pyAstFromText(sourceFileText)
if classOrFunction is _FunctionDefinition:
pyAst = PyAstUtil.functionDefOrLambdaAtLineNumber(sourceAst, sourceLine)
else:
assert classOrFunction is _ClassDefinition
pyAst = PyAstUtil.classDefAtLineNumber(sourceAst, sourceLine)
assert sourceLine == pyAst.lineno
try:
freeVariableMemberAccessChainResolutions = \
self._computeAndResolveFreeVariableMemberAccessChainsInAst(
pyObject, pyAst
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(e, sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, location) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(
path=sourceFileName,
lineNumber=location[0]
)
)
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(
fileName=sourceFileName,
fileText=sourceFileText
)
)
return classOrFunction(
sourceFileId=sourceFileId,
lineNumber=sourceLine,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions
)
def _registerWithBlock(self, objectId, pyObject):
"""
`_registerWithBlock`: register a `PyforaWithBlock.PyforaWithBlock`
with `self.objectRegistry`.
Recursively call `walkPyObject` on the resolvable free variable
member access chains in the block and on the file object.
"""
lineNumber = pyObject.lineNumber
sourceTree = PyAstUtil.pyAstFromText(pyObject.sourceText)
withBlockAst = PyAstUtil.withBlockAtLineNumber(sourceTree, lineNumber)
withBlockFun = ast.FunctionDef(
name="",
args=ast.arguments(args=[], defaults=[], kwarg=None, vararg=None),
body=withBlockAst.body,
decorator_list=[],
lineno=lineNumber,
col_offset=0
)
if PyAstUtil.hasReturnInOuterScope(withBlockFun):
raise Exceptions.BadWithBlockError(
"return statement not supported in pyfora with-block (line %s)" %
PyAstUtil.getReturnLocationsInOuterScope(withBlockFun)[0])
if PyAstUtil.hasYieldInOuterScope(withBlockFun):
raise Exceptions.BadWithBlockError(
"yield expression not supported in pyfora with-block (line %s)" %
PyAstUtil.getYieldLocationsInOuterScope(withBlockFun)[0])
freeVariableMemberAccessChainsWithPositions = \
self._freeMemberAccessChainsWithPositions(withBlockFun)
boundValuesInScopeWithPositions = \
PyAstFreeVariableAnalyses.collectBoundValuesInScope(
withBlockFun, getPositions=True)
for boundValueWithPosition in boundValuesInScopeWithPositions:
val, pos = boundValueWithPosition
if val not in pyObject.unboundLocals and val in pyObject.boundVariables:
freeVariableMemberAccessChainsWithPositions.add(
PyAstFreeVariableAnalyses.VarWithPosition(var=(val,), pos=pos)
)
try:
freeVariableMemberAccessChainResolutions = \
self._resolveFreeVariableMemberAccessChains(
freeVariableMemberAccessChainsWithPositions, pyObject.boundVariables
)
except UnresolvedFreeVariableException as e:
_convertUnresolvedFreeVariableExceptionAndRaise(e, pyObject.sourceFileName)
try:
processedFreeVariableMemberAccessChainResolutions = {}
for chain, (resolution, position) in \
freeVariableMemberAccessChainResolutions.iteritems():
processedFreeVariableMemberAccessChainResolutions['.'.join(chain)] = \
self.walkPyObject(resolution)
except UnresolvedFreeVariableExceptionWithTrace as e:
e.addToTrace(
Exceptions.makeTraceElement(
path=pyObject.sourceFileName,
lineNumber=position.lineno
)
)
raise
sourceFileId = self.walkPyObject(
_FileDescription.cachedFromArgs(
fileName=pyObject.sourceFileName
)
)
self._objectRegistry.defineWithBlock(
objectId=objectId,
freeVariableMemberAccessChainsToId=\
processedFreeVariableMemberAccessChainResolutions,
sourceFileId=sourceFileId,
lineNumber=lineNumber
)
def replaceUsesWithCalls(pyAstNode, names, isClassContext):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
vis = _NameToNameCallTransformer(names)
return ast.fix_missing_locations(vis.visit(pyAstNode))
def getFreeVariables(pyAstNode, isClassContext=None, getPositions=False):
pyAstNode = PyAstUtil.getRootInContext(pyAstNode, isClassContext)
freeVarsVisitor = _FreeVarsVisitor()
freeVarsVisitor.visit(pyAstNode)
return freeVarsVisitor.getFreeVars(getPositions=getPositions)