def updatePyAstVariableUsages(pyAst, variablesInScope, varsToTurnIntoCalls,
isClassContext):
#in the variables we've been handed, every member access chain 'x.y.z' has been
#replaced with an actual variable lookup of the form 'x.y.z'. We need to perform
#this same replacement in the actual python source code we're running, since
#we may not actually have enough information to recover 'x'
replacements = {}
for possible_replacement in variablesInScope:
if '.' in possible_replacement:
replacements[tuple(
possible_replacement.split('.'))] = possible_replacement
#we need to deepcopy the AST since the collapser modifies the AST, and this is just a
#slice of a cached tree.
pyAst = ast.fix_missing_locations(copy.deepcopy(pyAst))
if varsToTurnIntoCalls:
pyAst = PyAstFreeVariableAnalyses.replaceUsesWithCalls(
pyAst, set(varsToTurnIntoCalls), isClassContext)
pyAst = PyAstFreeVariableAnalyses.collapseFreeVariableMemberAccessChains(
pyAst, replacements, isClassContext=isClassContext)
return ast.fix_missing_locations(pyAst)
def test_freeVariables_globalStmt_2(self):
tree = ast.parse(
textwrap.dedent("""
def f():
global x
return x
"""))
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstFreeVariableAnalyses.getFreeVariables(tree)
def test_freeVariables_globalStmt_2(self):
tree = ast.parse(
textwrap.dedent(
"""
def f():
global x
return x
"""
)
)
with self.assertRaises(Exceptions.PythonToForaConversionError):
PyAstFreeVariableAnalyses.getFreeVariables(tree)
def test_freeVariables_function_context(self):
tree = ast.parse(
textwrap.dedent(
"""
def fib(x):
if x < 2:
return 1
else:
return fib(x-1) + fib(x-2)
"""
)
)
self.assertEqual(set([]), PyAstFreeVariableAnalyses.getFreeVariables(tree.body[0], isClassContext=False))
self.assertEqual(set(["fib"]), PyAstFreeVariableAnalyses.getFreeVariables(tree.body[0], isClassContext=True))
def test_TransvisitorsDontModifyTree(self):
tree = ast.parse(self.some_python_code)
# deep-copy tree because transformers modify the AST in place
# making the test of areAstsIdentical(tree, tree') meaningless
tree1 = copy.deepcopy(tree)
noopTransformer = NodeVisitorBases.SemanticOrderNodeTransvisitor()
tree2 = noopTransformer.visit(tree1)
self.assertIsNotNone(tree2)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree2))
scopeMgr = NodeVisitorBases.InScopeSaveRestoreValue(
lambda: True, lambda x: None)
noopTransformer = NodeVisitorBases.GenericScopedTransvisitor(scopeMgr)
tree3 = noopTransformer.visit(tree1)
self.assertIsNotNone(tree3)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree3))
freeVarsVisitor = PyAstFreeVariableAnalyses._FreeVariableMemberAccessChainsTransvisitor(
)
tree4 = freeVarsVisitor.visit(tree1)
self.assertIsNotNone(tree4)
self.assertTrue(PyAstUtil.areAstsIdentical(tree, tree4))
def test_freeVariables_CompsAndGenExp2(self):
tree = ast.parse(
textwrap.dedent(
"""
[elt01 for elt1 in container1]
{elt02 for elt2 in container2}
{elt03: elt04 for elt4 in container4 for elt3 in container3}
(x0*y0 for x in range(10) for y in bar(x))
"""
)
)
expectedResult = set(
[
"container1",
"container2",
"container3",
"container4",
"bar",
"range",
"elt01",
"elt02",
"elt03",
"elt04",
"x0",
"y0",
]
)
self.assertEqual(expectedResult, PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_multiple_assignment(self):
tree = ast.parse(
textwrap.dedent("""
x = y = z = w
"""))
self.assertEqual(set(['w']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Assign(self):
tree = ast.parse(
textwrap.dedent("""
(x, y), z = w
"""))
self.assertEqual(set(['w']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_classDef_1(self):
tree = ast.parse(
textwrap.dedent("""class C(object):
def f(x):
return len(f)"""))
self.assertEqual(set(['object', 'len', 'f']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Assign_2(self):
tree = ast.parse(
textwrap.dedent("""
w = ((x, y), z)
"""))
self.assertEqual(set(['x', 'y', 'z']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_SetComp_1(self):
tree = ast.parse(
textwrap.dedent("""
{v for x in q}
"""))
self.assertEqual(set(['v', 'q']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_DictComp_1(self):
tree = ast.parse(
textwrap.dedent("""
d = { x: y for x, y in o.f() }
"""))
self.assertEqual(set(['o']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_DictComp_2(self):
tree = ast.parse(
textwrap.dedent("""
d = { x1: y1 for x2, y2 in o.f() }
"""))
self.assertEqual(set(['o', 'x1', 'y1']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_2(self):
tree = ast.parse(
textwrap.dedent("""
[val for val in x if val == free]
"""))
self.assertEqual(set(['free', 'x']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_4(self):
tree = ast.parse(
textwrap.dedent("""
[(x, y) for x in [1,2,3] for y in [3,1,4] if x != y]
"""))
self.assertEqual(set(),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_functionDef_2(self):
tree = ast.parse(
textwrap.dedent("""def outer():
def f(x):
return len(f)"""))
self.assertEqual(set(['len']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_memberAccessChain_2(self):
tree = ast.parse("(1).y.z.w")
self.assertIsNone(
PyAstFreeVariableAnalyses._memberAccessChainOrNone(
tree.body[0].value
)
)
def test_freeVariables_Lambda_1(self):
tree = ast.parse(
textwrap.dedent("""
lambda x, y = z, *args, **kwargs: (x, y, args, kwargs, free)
"""))
self.assertEqual(set(['z', 'free']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_inAssignment(self):
tree = ast.parse(
textwrap.dedent("""
def f(arg):
y = x
return y
"""))
self.assertEqual(set(['x']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_name_substring_bug(self):
tree = ast.parse(
textwrap.dedent("""
al = 10
l
"""))
expectedResult = set(['l'])
self.assertEqual(expectedResult,
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_functionCalls(self):
tree = ast.parse(
textwrap.dedent("""
x = 2
f(x, y, z = 2, w = x, q = free, *args, **kwargs)
"""))
self.assertEqual(set(['f', 'y', 'args', 'kwargs', 'free']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariableMemberAccessChains_6(self):
tree = ast.parse(
textwrap.dedent("""
q = x.y.z = 2
"""))
res = PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree)
self.assertEqual(set([('x', 'y', 'z')]), res)
def test_freeVariables_classDef_4(self):
tree = ast.parse(
textwrap.dedent("""
class A: pass
class B(A, C, D): pass
"""))
self.assertEqual(set(('C', 'D')),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_withStatement(self):
tree = ast.parse(
textwrap.dedent("""
def f(arg):
with x as e:
return e
"""))
self.assertEqual(set(['x']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_classLocalVar_1(self):
tree = ast.parse(
textwrap.dedent("""
class C(object):
x = 0
"""))
self.assertEqual(set(['object']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_memberAccessChain_1(self):
tree = ast.parse("x.y.z.w")
self.assertEqual(
('x', 'y', 'z', 'w'),
PyAstFreeVariableAnalyses._memberAccessChainOrNone(
tree.body[0].value
)
)
def test_freeVariables_For(self):
tree = ast.parse(
textwrap.dedent("""
for x in elt:
x + 2
"""))
self.assertEqual(set(['elt']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_3(self):
tree = ast.parse(
textwrap.dedent("""
x = [1,2,3]
[y for val in x]
"""))
self.assertEqual(set(['y']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_functionDef_2(self):
tree = ast.parse(
textwrap.dedent(
"""def outer():
def f(x):
return len(f)"""
)
)
self.assertEqual(set(["len"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_classDef_1(self):
tree = ast.parse(
textwrap.dedent(
"""class C(object):
def f(x):
return len(f)"""
)
)
self.assertEqual(set(["object", "len", "f"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Assign(self):
tree = ast.parse(
textwrap.dedent(
"""
(x, y), z = w
"""
)
)
self.assertEqual(set(["w"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Lambda_1(self):
tree = ast.parse(
textwrap.dedent(
"""
lambda x, y = z, *args, **kwargs: (x, y, args, kwargs, free)
"""
)
)
self.assertEqual(set(["z", "free"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_2(self):
tree = ast.parse(
textwrap.dedent(
"""
[val for val in x if val == free]
"""
)
)
self.assertEqual(set(["free", "x"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_4(self):
tree = ast.parse(
textwrap.dedent(
"""
[(x, y) for x in [1,2,3] for y in [3,1,4] if x != y]
"""
)
)
self.assertEqual(set(), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_SetComp_1(self):
tree = ast.parse(
textwrap.dedent(
"""
{v for x in q}
"""
)
)
self.assertEqual(set(["v", "q"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_DictComp_1(self):
tree = ast.parse(
textwrap.dedent(
"""
d = { x: y for x, y in o.f() }
"""
)
)
self.assertEqual(set(["o"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_DictComp_2(self):
tree = ast.parse(
textwrap.dedent(
"""
d = { x1: y1 for x2, y2 in o.f() }
"""
)
)
self.assertEqual(set(["o", "x1", "y1"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariablesMemberAccessChain_onFunctionDefNode(self):
tree1 = ast.parse(
textwrap.dedent("""
def g(arg):
if arg < 0:
return x + arg
return x * h(arg - 1, g)
"""))
res = PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(
tree1)
self.assertEqual(set([('h', ), ('x', )]), res)
tree2 = PyAstUtil.functionDefOrLambdaAtLineNumber(tree1, 2)
self.assertEqual(
set([('h', ), ('x', )]),
PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(
tree2, False))
def test_freeVariables_function_context(self):
tree = ast.parse(
textwrap.dedent("""
def fib(x):
if x < 2:
return 1
else:
return fib(x-1) + fib(x-2)
"""))
self.assertEqual(
set([]),
PyAstFreeVariableAnalyses.getFreeVariables(tree.body[0],
isClassContext=False))
self.assertEqual(
set(['fib']),
PyAstFreeVariableAnalyses.getFreeVariables(tree.body[0],
isClassContext=True))
def test_variables_assigned_by_generator_comp_are_free(self):
tree = ast.parse(
textwrap.dedent("""
x = range(10)
if isinstance(x, slice):
return list(x[slice] for slice in x)
"""))
expectedResult = set(['range', 'isinstance', 'slice', 'list'])
self.assertEqual(expectedResult,
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Sequence_4(self):
tree = ast.parse(
textwrap.dedent("""
def f():
x = 3
x + 2
"""))
self.assertEqual(set(),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_notFreeNotDefinedOnAllPaths_2(self):
tree = ast.parse(
textwrap.dedent("""
def f(arg):
if arg:
x = 3
return x
"""))
self.assertEqual(set([]),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_inAssignment(self):
tree = ast.parse(
textwrap.dedent(
"""
def f(arg):
y = x
return y
"""
)
)
self.assertEqual(set(["x"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_name_substring_bug(self):
tree = ast.parse(
textwrap.dedent(
"""
al = 10
l
"""
)
)
expectedResult = set(["l"])
self.assertEqual(expectedResult, PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_ListComp_3(self):
tree = ast.parse(
textwrap.dedent(
"""
x = [1,2,3]
[y for val in x]
"""
)
)
self.assertEqual(set(["y"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_withStatement(self):
tree = ast.parse(
textwrap.dedent(
"""
def f(arg):
with x as e:
return e
"""
)
)
self.assertEqual(set(["x"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def _freeMemberAccessChainsWithPositions(self, pyAst):
# ATz: just added 'False' as a 2nd argument, but we may need to check
# that whenever pyAst is a FunctionDef node, its context is not a class
# (i.e., it is not an instance method). In that case, we need to pass
# 'True' as the 2nd argument.
freeVariableMemberAccessChains = \
PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(
pyAst, isClassContext=False, getPositions=True
)
return freeVariableMemberAccessChains
def test_call_and_then_member_chain(self):
tree = ast.parse(
textwrap.dedent(
"""
def f():
return g(1).__str__()
"""
)
)
self.assertEqual(set([("g",)]), PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree))
def test_freeVariables_For(self):
tree = ast.parse(
textwrap.dedent(
"""
for x in elt:
x + 2
"""
)
)
self.assertEqual(set(["elt"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_classLocalVar_1(self):
tree = ast.parse(
textwrap.dedent(
"""
class C(object):
x = 0
"""
)
)
self.assertEqual(set(["object"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariablesMemberAccessChain_onFunctionDefNode(self):
tree1 = ast.parse(
textwrap.dedent(
"""
def g(arg):
if arg < 0:
return x + arg
return x * h(arg - 1, g)
"""
)
)
res = PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree1)
self.assertEqual(set([("h",), ("x",)]), res)
tree2 = PyAstUtil.functionDefOrLambdaAtLineNumber(tree1, 2)
self.assertEqual(
set([("h",), ("x",)]), PyAstFreeVariableAnalyses.getFreeVariableMemberAccessChains(tree2, False)
)
def test_freeVariables_whileLoop(self):
tree = ast.parse(
textwrap.dedent("""
def f(arg):
tr = 0
while x in range(0, arg):
tr += 1
return tr
"""))
self.assertEqual(set(['x', 'range']),
PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_functionCalls(self):
tree = ast.parse(
textwrap.dedent(
"""
x = 2
f(x, y, z = 2, w = x, q = free, *args, **kwargs)
"""
)
)
self.assertEqual(set(["f", "y", "args", "kwargs", "free"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_variables_assigned_by_generator_comp_are_free(self):
tree = ast.parse(
textwrap.dedent(
"""
x = range(10)
if isinstance(x, slice):
return list(x[slice] for slice in x)
"""
)
)
expectedResult = set(["range", "isinstance", "slice", "list"])
self.assertEqual(expectedResult, PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_Sequence_4(self):
tree = ast.parse(
textwrap.dedent(
"""
def f():
x = 3
x + 2
"""
)
)
self.assertEqual(set(), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_notFreeNotDefinedOnAllPaths_2(self):
tree = ast.parse(
textwrap.dedent(
"""
def f(arg):
if arg:
x = 3
return x
"""
)
)
self.assertEqual(set([]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_whileLoop(self):
tree = ast.parse(
textwrap.dedent(
"""
def f(arg):
tr = 0
while x in range(0, arg):
tr += 1
return tr
"""
)
)
self.assertEqual(set(["x", "range"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_freeVariables_forLoop(self):
tree = ast.parse(
textwrap.dedent(
"""
def f(arg):
tr = x
for x in xrange(0, arg):
tr += x
return tr
"""
)
)
self.assertEqual(set(["xrange"]), PyAstFreeVariableAnalyses.getFreeVariables(tree))
def test_variables_assigned_in_interior_scope(self):
tree = ast.parse(
textwrap.dedent(
"""
def f():
x = 10
return y
return x
"""
)
)
expectedResult = set(["x", "y"])
self.assertEqual(expectedResult, PyAstFreeVariableAnalyses.getFreeVariables(tree))
