def test_simple_control_dependency_2(self):
# If condition evaluated to false, with two relevant variables (but no influence on result)
def func() -> int:
foo = 1
bar = 2
result = 3
if foo == bar:
result = 1
return result
init_basic_block = BasicBlock([
# result = 3
Instr("LOAD_CONST", arg=3),
Instr("STORE_FAST", arg="result"),
])
return_basic_block = BasicBlock([
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(init_basic_block)
expected_instructions.extend(return_basic_block)
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_call_without_arguments(self):
module_block = BasicBlock([
# def callee():
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="callee"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="callee"),
# result = callee()
Instr("LOAD_NAME", arg="callee"),
Instr("CALL_FUNCTION", arg=0),
Instr("STORE_NAME", arg="result"),
# return result
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
callee_block = BasicBlock([
Instr("LOAD_CONST", arg=0),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(callee_block)
module_file = "simple_call.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_simple_control_dependency_4(self):
# If-elif-else with else branch true
def func() -> int:
foo = 1
bar = 2
if foo == bar:
result = 1
elif foo > bar:
result = 2
else:
result = 3
return result
return_block = BasicBlock([
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE")
])
else_block = BasicBlock([
# result = 3
Instr("LOAD_CONST", arg=3),
Instr("STORE_FAST", arg="result")
])
elif_cond = BasicBlock([
# elif foo == 1:
Instr("LOAD_FAST", arg="foo"),
Instr("LOAD_FAST", arg="bar"),
Instr("COMPARE_OP", arg=Compare.GT),
Instr("POP_JUMP_IF_FALSE", arg=else_block),
])
if_cond = BasicBlock([
# if foo == bar
Instr("LOAD_FAST", arg="foo"),
Instr("LOAD_FAST", arg="bar"),
Instr("COMPARE_OP", arg=Compare.EQ),
Instr("POP_JUMP_IF_FALSE", arg=elif_cond),
])
init_block = BasicBlock([
# foo = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_FAST", arg="foo"),
# bar = 2
Instr("LOAD_CONST", arg=2),
Instr("STORE_FAST", arg="bar"),
])
expected_instructions = []
expected_instructions.extend(init_block)
expected_instructions.extend(if_cond)
expected_instructions.extend(elif_cond)
expected_instructions.extend(else_block)
expected_instructions.extend(return_block)
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_data_dependency_3(self):
# Transitive explicit (full cover) dependencies
def func() -> int:
foo = 1
result = 1 + foo
return result
expected_instructions = [
# foo = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_FAST", arg="foo"),
# result = 1 + foo
Instr("LOAD_CONST", arg=1),
Instr("LOAD_FAST", arg="foo"),
Instr("BINARY_ADD"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
]
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_builtin_addresses(self):
def func():
test_dict = {1: "one", 2: "two"}
# noinspection PyListCreation
test_list = [1, 2]
test_list.append(3)
result = test_dict.get(1)
return result
function_block = BasicBlock([
# test_dict = {1: "one", 2: "two"}
Instr("LOAD_CONST", arg="one"),
Instr("LOAD_CONST", arg="two"),
Instr("LOAD_CONST", arg=(1, 2)),
Instr("BUILD_CONST_KEY_MAP", arg=2),
Instr("STORE_FAST", arg="test_dict"),
# result = test_dict.get(1)
Instr("LOAD_FAST", arg="test_dict"),
Instr("LOAD_METHOD", arg="get"),
Instr("LOAD_CONST", arg=1),
Instr("CALL_METHOD", arg=1),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_builtin_addresses")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_nested_class(self):
def func():
# STORE_DEREF, LOAD_CLOSURE, LOAD_CLASSDEREF
x = []
class NestedClass:
y = x
class_attr = NestedClass.y
result = class_attr
return result
freevar_x = FreeVar("x")
cellvar_x = CellVar("x")
function_block = BasicBlock([
# x = []
Instr("BUILD_LIST", arg=0),
Instr("STORE_DEREF", arg=cellvar_x),
# class NestedClass:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CLOSURE", arg=cellvar_x),
Instr("BUILD_TUPLE", arg=1),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("MAKE_FUNCTION", arg=8),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_FAST", arg="NestedClass"),
# class_attr = NestedClass.y
Instr("LOAD_FAST", arg="NestedClass"),
Instr("LOAD_ATTR", arg="y"),
Instr("STORE_FAST", arg="class_attr"),
# result = class_attr
Instr("LOAD_FAST", arg="class_attr"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
nested_class_block = BasicBlock([
# y = x
Instr("LOAD_CLASSDEREF", arg=freevar_x),
Instr("STORE_NAME", arg="y"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
expected_instructions.extend(nested_class_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_nested_class")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_closures(self):
# Closure function
freevar_foo = FreeVar("foo")
cellvar_foo = CellVar("foo")
module_block = BasicBlock([
# def outer_function(foo):
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="outer_function"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="outer_function"),
# inner = outer_function('a')
Instr("LOAD_NAME", arg="outer_function"),
Instr("LOAD_CONST", arg="a"),
Instr("CALL_FUNCTION", arg=1),
Instr("STORE_NAME", arg="inner"),
# result = inner("abc")
Instr("LOAD_NAME", arg="inner"),
Instr("LOAD_CONST", arg="abc"),
Instr("CALL_FUNCTION", arg=1),
Instr("STORE_NAME", arg="result"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
outer_function_block = BasicBlock([
# def inner_function(bar):
Instr("LOAD_CLOSURE", arg=cellvar_foo),
Instr("BUILD_TUPLE", arg=1),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="outer_function.<locals>.inner_function"),
Instr("MAKE_FUNCTION", arg=8),
Instr("STORE_FAST", arg="inner_function"),
# return inner
Instr("LOAD_FAST", arg="inner_function"),
Instr("RETURN_VALUE"),
])
inner_function_block = BasicBlock([
# return foo in bar
Instr("LOAD_DEREF", arg=freevar_foo),
Instr("LOAD_FAST", arg="bar"),
Instr("COMPARE_OP", arg=Compare.IN),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(outer_function_block)
expected_instructions.extend(inner_function_block)
module_file = "closure.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(expected_instructions), len(dynamic_slice.sliced_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def __evaltest(agent,spec):
rawaddr = spec['query']['rawaddr']
expected = spec['result']
print("rawaddr = '%s'" % rawaddr)
print("expected = %s" % expected)
response = agent.get_lookup(rawaddr)
print("response = %s" % response)
status = compare(response,expected)
print("status = %s" % status)
def test_data_dependency_4(self):
# Explicit attribute dependencies (full cover)
module_block = BasicBlock([
# class Foo:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Foo"),
Instr("MAKE_FUNCTION", arg=0),
Instr("LOAD_CONST", arg="Foo"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="Foo"),
# ob.attr1 = 1
Instr("LOAD_CONST", arg=1),
Instr("LOAD_NAME", arg="ob"),
Instr("STORE_ATTR", arg="attr1"),
# ob.attr2 = ob.attr2.append(ob.attr1)
Instr("LOAD_NAME", arg="ob"),
Instr("LOAD_ATTR", arg="attr2"),
Instr("LOAD_METHOD", arg="append"),
Instr("LOAD_NAME", arg="ob"),
Instr("LOAD_ATTR", arg="attr1"),
Instr("CALL_METHOD", arg=1),
Instr("LOAD_NAME", arg="ob"),
Instr("STORE_ATTR", arg="attr2"),
# result = ob.attr2
Instr("LOAD_NAME", arg="ob"),
Instr("LOAD_ATTR", arg="attr2"),
Instr("STORE_NAME", arg="result"),
# return
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
class_attr_block = BasicBlock([
# attr2 = [1, 2, 3]
Instr("LOAD_CONST", arg=1),
Instr("LOAD_CONST", arg=2),
Instr("LOAD_CONST", arg=3),
Instr("BUILD_LIST", arg=3),
Instr("STORE_NAME", arg="attr2"),
# return
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(class_attr_block)
module_file = "attribute_dependencies.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def evaltest(agent,r):
print("::: eval r = %s" % r)
print("::: eval query = %s" % r['query'])
print("::: eval expected = %s" % r['result'])
_bbl = r['query'].get('bbl')
_bin = r['query'].get('bin')
expected = r['result']
bldg_list = agent.get_buildings(_bbl,_bin)
print("building.type = %s" % type(bldg_list))
# print("building.len = %d" % len(bldg_list))
print("building = %s" % bldg_list)
status = compare(bldg_list,expected)
print("status = %s" % status)
def test_call_unused_argument(self):
# Call with two arguments, one of which is used in the callee
module_block = BasicBlock([
# def callee():
Instr("LOAD_NAME", arg="int"),
Instr("LOAD_NAME", arg="int"),
Instr("LOAD_CONST", arg=('a', 'b')),
Instr("BUILD_CONST_KEY_MAP", arg=2),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="callee"),
Instr("MAKE_FUNCTION", arg=4),
Instr("STORE_NAME", arg="callee"),
# foo = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_NAME", arg="foo"),
# bar = 2
# This argument is not used by the callee and should therefore be excluded.
# But it is an implicit data dependency of the call and is incorrectly and imprecisely included.
# Instr("LOAD_CONST", arg=2),
# Instr("STORE_NAME", arg="bar"),
# result = callee()
Instr("LOAD_NAME", arg="callee"),
Instr("LOAD_NAME", arg="foo"),
# Instr("LOAD_NAME", arg="bar"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="result"),
# return result
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
callee_block = BasicBlock([
# return a
Instr("LOAD_FAST", arg="a"),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(callee_block)
module_file = "simple_call_arg.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def evaltest(agent,r):
query = r['query']
result = r['result']
if ARGS.loud:
print("query = %s" % query)
try:
taxlot = perform(agent,query)
except Exception as e:
print("FAILED: %s" % e)
if ARGS.fail:
raise e
return False
if ARGS.loud:
print("result = %s" % result)
print("taxlot = %s" % taxlot)
return compare(taxlot,result)
def evaltest(endpoint,agent,spec):
query = spec['query']
expected = spec['result']
if LOUD:
print("query = '%s'" % query)
print("expected = %s" % expected)
try:
response = agent.dispatch(endpoint,query)
if LOUD:
print("response = %s" % response)
return compare(response,expected)
except Exception as e:
print("EXCEPT %s" % str(e))
if THROW:
raise e
return False
def test_simple_loop(self):
def func():
result = 0
for i in range(0, 3):
result += i
return result
return_block = BasicBlock([
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE")
])
loop_header = BasicBlock([
Instr("FOR_ITER", arg=return_block),
])
loop_block = BasicBlock([
# result += i
Instr("STORE_FAST", arg="i"),
Instr("LOAD_FAST", arg="result"),
Instr("LOAD_FAST", arg="i"),
Instr("INPLACE_ADD"),
Instr("STORE_FAST", arg="result"),
Instr("JUMP_ABSOLUTE", arg=loop_header),
])
loop_setup = BasicBlock([
# for i in range(0, 3):
Instr("LOAD_GLOBAL", arg="range"),
Instr("LOAD_CONST", arg=0),
Instr("LOAD_CONST", arg=3),
Instr("CALL_FUNCTION", arg=2),
Instr("GET_ITER"),
])
init_block = BasicBlock([
Instr("LOAD_CONST", arg=0),
Instr("STORE_FAST", arg="result"),
])
expected_instructions = []
expected_instructions.extend(init_block)
expected_instructions.extend(loop_setup)
expected_instructions.extend(loop_header)
expected_instructions.extend(loop_block)
expected_instructions.extend(return_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_simple_loop")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_call_with_arguments(self):
# Call with two arguments, one of which is used in the callee
module_block = BasicBlock([
# def callee():
Instr("LOAD_NAME", arg="int"),
Instr("LOAD_NAME", arg="int"),
Instr("LOAD_CONST", arg=('a', 'b')),
Instr("BUILD_CONST_KEY_MAP", arg=2),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="callee"),
Instr("MAKE_FUNCTION", arg=4),
Instr("STORE_NAME", arg="callee"),
# foo = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_NAME", arg="foo"),
# bar = 2
Instr("LOAD_CONST", arg=2),
Instr("STORE_NAME", arg="bar"),
# result = callee()
Instr("LOAD_NAME", arg="callee"),
Instr("LOAD_NAME", arg="foo"),
Instr("LOAD_NAME", arg="bar"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="result"),
# return result
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
callee_block = BasicBlock([
# return a
Instr("LOAD_FAST", arg="a"),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(callee_block)
module_file = "simple_call_arg.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_simple_control_dependency_1(self):
# If condition evaluated to true, with relevant variable foo
def func() -> int:
foo = 1
result = 3
if foo == 1:
result = 1
return result
return_basic_block = BasicBlock([
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE")
])
if_basic_block = BasicBlock([
# result = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_FAST", arg="result"),
])
init_basic_block = BasicBlock([
# foo = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_FAST", arg="foo"),
# if foo == 1
Instr("LOAD_FAST", arg="foo"),
Instr("LOAD_CONST", arg=1),
Instr("COMPARE_OP", arg=Compare.EQ),
Instr("POP_JUMP_IF_FALSE", arg=return_basic_block),
])
expected_instructions = []
expected_instructions.extend(init_basic_block)
expected_instructions.extend(if_basic_block)
expected_instructions.extend(return_basic_block)
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_with_extended_arg(self):
def func():
p = [1, 2, 3, 4, 5, 6]
# noinspection PyUnusedLocal
unused = p
q, r, *s, t = p # With extended argument
result = q, r
return result
module_block = BasicBlock([
# p = [1, 2, 3, 4, 5, 6]
Instr("LOAD_CONST", arg=1),
Instr("LOAD_CONST", arg=2),
Instr("LOAD_CONST", arg=3),
Instr("LOAD_CONST", arg=4),
Instr("LOAD_CONST", arg=5),
Instr("LOAD_CONST", arg=6),
Instr("BUILD_LIST", arg=6),
Instr("STORE_FAST", arg="p"),
# q, r, *s, t = p
Instr("LOAD_FAST", arg="p"),
# Instr("EXTENDED_ARG", arg=1), # EXTENDED_ARG can not be in a slice
Instr("UNPACK_EX", arg=258),
Instr("STORE_FAST", arg="q"),
Instr("STORE_FAST", arg="r"),
# result = q
Instr("LOAD_FAST", arg="q"),
Instr("LOAD_FAST", arg="r"),
Instr("BUILD_TUPLE", arg=2),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(module_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_with_extended_arg")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_data_dependency_1(self):
# Implicit data dependency at return, explicit (full cover) for result
def func() -> int:
result = 1
return result
expected_instructions = [
# result = 1
Instr("LOAD_CONST", arg=1),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
]
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_mod_untraced_object(self):
def func():
lst = [('foo', '3'), ('bar', '1'), ('foobar', '2')]
lst.sort(
) # This is incorrectly excluded, since it is not known that the method modifies the list
result = lst
return result
function_block = BasicBlock([
# lst = [('foo', '3'), ('bar', '1'), ('foobar', '2')]
Instr("LOAD_CONST", arg=('foo', '3')),
Instr("LOAD_CONST", arg=('bar', '1')),
Instr("LOAD_CONST", arg=('foobar', '2')),
Instr("BUILD_LIST", arg=3),
Instr("STORE_FAST", arg="lst"),
# lst.sort()
# This is incorrectly excluded, since it is not known that the method modifies the list
Instr("LOAD_FAST", arg="lst"),
Instr("LOAD_METHOD", arg="sort"),
Instr("CALL_METHOD", arg=0),
Instr("POP_TOP"),
# result = lst
Instr("LOAD_FAST", arg="lst"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
dynamic_slice = slice_function_at_return(
func.__code__, test_name="test_mod_untraced_object")
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_import_star(self):
# IMPORT_STAR with access to immutable variable
main_module_block = BasicBlock([
# from tests.slicer.example_modules.import_star_def import *
Instr("IMPORT_NAME",
"tests.slicer.example_modules.import_star_def"),
Instr("IMPORT_STAR"),
# result = Foo.test
Instr("LOAD_NAME", arg="star_imported"),
Instr("STORE_NAME", arg="result"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
dependency_module_block = BasicBlock([
# star_imported = "test"
Instr("LOAD_CONST", arg="test"),
Instr("STORE_NAME", arg="star_imported"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(main_module_block)
expected_instructions.extend(dependency_module_block)
module_dependency_file = "import_star_def.py"
module_dependency_path = example_modules_path + module_dependency_file
instrument_module(module_dependency_path)
module_file = "import_star_main.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
compile_module(module_dependency_path)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_data_dependency_composite(self):
# Composite type dependencies, which are way to broad
def func():
# noinspection PyListCreation
foo_list = [1, 2,
3] # the only list operation which should be included
foo_list.append(
4
) # should no be included, and is not included (good and limitation at the same time)
foo_list += [5] # should no be included, but is
foo_list[2:3] = [0, 0] # should no be included, but is
result = foo_list[0] # correctly included
return result # correctly included
expected_instructions = [
# foo_list = [1, 2, 3]
Instr("LOAD_CONST", arg=1),
Instr("LOAD_CONST", arg=2),
Instr("LOAD_CONST", arg=3),
Instr("BUILD_LIST", arg=3),
Instr("STORE_FAST", arg="foo_list"),
# result = foo_list[0]
Instr("LOAD_FAST", arg="foo_list"),
Instr("LOAD_CONST", arg=0),
Instr("BINARY_SUBSCR"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
]
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_lambda(self):
def func():
x = lambda a: a + 10
result = x(1)
return result
function_block = BasicBlock([
# x = lambda a: a + 10
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="IntegrationTestLanguageFeatures.test_lambda.<locals>.func.<locals>.<lambda>"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_FAST", arg="x"),
# result = x(1)
Instr("LOAD_FAST", arg="x"),
Instr("LOAD_CONST", arg=1),
Instr("CALL_FUNCTION", arg=1),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
lambda_block = BasicBlock([
# lambda a: a + 10
Instr("LOAD_FAST", arg="a"),
Instr("LOAD_CONST", arg=10),
Instr("BINARY_ADD"),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
expected_instructions.extend(lambda_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_lambda")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_object_modification_call(self):
def func():
class NestedClass:
def __init__(self):
self.x = 1
def inc_x(self):
self.x = self.x + 1
ob = NestedClass()
ob.inc_x()
result = ob.x
return result
function_block = BasicBlock([
# class NestedClass:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("MAKE_FUNCTION", arg=0),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_FAST", arg="NestedClass"),
# ob = NestedClass()
Instr("LOAD_FAST", arg="NestedClass"),
Instr("CALL_FUNCTION", arg=0),
Instr("STORE_FAST", arg="ob"),
# ob.inc_x()
Instr("LOAD_FAST", arg="ob"),
Instr("LOAD_METHOD", arg="inc_x"),
Instr("CALL_METHOD", arg=0),
# result = ob.x
Instr("LOAD_FAST", arg="ob"),
Instr("LOAD_ATTR", arg="x"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
nested_class_block = BasicBlock([
# Definition of dunder methods are wrongly excluded, since these are not explicitly loaded
# def __init__(self):
# Instr("LOAD_CONST", arg=dummy_code_object),
# Instr("LOAD_CONST", arg="IntegrationTestLanguageFeatures.test_object_modification_call.<locals>."
# "func.<locals>.NestedClass.__init__"),
# Instr("MAKE_FUNCTION", arg=0),
# Instr("STORE_NAME", arg="__init__"),
# def inc_x(self):
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="IntegrationTestLanguageFeatures.test_object_modification_call.<locals>."
"func.<locals>.NestedClass.inc_x"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="inc_x"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
init_block = BasicBlock([
# self.x = 1
Instr("LOAD_CONST", arg=1),
Instr("LOAD_FAST", arg="self"),
Instr("STORE_ATTR", arg="x"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
inc_x_block = BasicBlock([
# self.x = self.x + 1
Instr("LOAD_FAST", arg="self"),
Instr("LOAD_ATTR", arg="x"),
Instr("LOAD_CONST", arg=1),
Instr("BINARY_ADD"),
Instr("LOAD_FAST", arg="self"),
Instr("STORE_ATTR", arg="x"),
# This "None return" is not included, because the return value is not used
# Instr("LOAD_CONST", arg=None),
# Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
expected_instructions.extend(nested_class_block)
expected_instructions.extend(init_block)
expected_instructions.extend(inc_x_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_object_modification_call")
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_dunder_definition(self):
def func():
class NestedClass:
def __init__(
self
): # Definition of dunder methods wrongly excluded, these are not explicitly loaded
self.x = 1
result = NestedClass()
return result
function_block = BasicBlock([
# class NestedClass:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("MAKE_FUNCTION", arg=0),
Instr("LOAD_CONST", arg="NestedClass"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_FAST", arg="NestedClass"),
# result = NestedClass()
Instr("LOAD_FAST", arg="NestedClass"),
Instr("CALL_FUNCTION", arg=0),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
nested_class_block = BasicBlock([
# Definition of dunder methods are wrongly excluded, since these are not explicitly loaded
# def __init__(self):
Instr("LOAD_CONST", arg=dummy_code_object),
Instr(
"LOAD_CONST",
arg=
"IntegrationTestLanguageFeatures.test_object_modification_call.<locals>."
"func.<locals>.NestedClass.__init__"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="__init__"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
init_block = BasicBlock([
# self.x = 1
Instr("LOAD_CONST", arg=1),
Instr("LOAD_FAST", arg="self"),
Instr("STORE_ATTR", arg="x"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
expected_instructions.extend(nested_class_block)
expected_instructions.extend(init_block)
dynamic_slice = slice_function_at_return(
func.__code__, test_name="test_dunder_definition")
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_exception(self):
# Exception
def func():
foo = 1
bar = 0
try:
result = 0 / 0
except ZeroDivisionError:
result = foo + bar
return result
self.assertEqual(func(), 1)
dummy_block = BasicBlock([])
return_block = BasicBlock([
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE")
])
try_block = BasicBlock([
# result = foo / bar <- did somehow effect slicing criterion...
Instr("LOAD_FAST", arg="foo"),
Instr("LOAD_FAST", arg="bar"),
Instr("BINARY_TRUE_DIVIDE"),
# except ZeroDivisionError:
Instr("DUP_TOP"),
Instr("LOAD_GLOBAL", arg="ZeroDivisionError"),
Instr("COMPARE_OP", arg=Compare.EXC_MATCH),
Instr("POP_JUMP_IF_FALSE", arg=dummy_block),
])
except_block = BasicBlock([
# result = foo + bar
Instr("LOAD_FAST", arg="foo"),
Instr("LOAD_FAST", arg="bar"),
Instr("BINARY_ADD"),
Instr("STORE_FAST", arg="result"),
Instr("JUMP_FORWARD", arg=dummy_block),
])
function_block = BasicBlock([
# foo = 1
Instr("LOAD_CONST",
arg=1), # <- excluded because no stack simulation
Instr("STORE_FAST", arg="foo"),
# bar = 0
Instr("LOAD_CONST",
arg=0), # <- excluded because no stack simulation
Instr("STORE_FAST", arg="bar"),
# try:
# Instr("SETUP_FINALLY", arg=try_block),
])
expected_instructions = []
expected_instructions.extend(return_block)
expected_instructions.extend(except_block)
expected_instructions.extend(function_block)
expected_instructions.extend(try_block)
dynamic_slice = slice_function_at_return(func.__code__)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_nested_class_2(self):
# Critical test to ensure that the attributes converted to variables
# are taken from the correct scope.
def func():
# STORE_DEREF, LOAD_CLOSURE, LOAD_CLASSDEREF
x1 = [1]
x2 = [2]
class Bar:
foo = x1 # included!
class Foo:
foo = x2 # NOT included
y = x2 # included
y = Foo.y # NOT included
class_attr = Bar.foo
class_attr2 = Bar.Foo.y
result = class_attr + class_attr2
return result
freevar_x1 = FreeVar("x1")
cellvar_x1 = CellVar("x1")
freevar_x2 = FreeVar("x2")
cellvar_x2 = CellVar("x2")
function_block = BasicBlock([
# x1 = [1]
Instr("LOAD_CONST", arg=1),
Instr("BUILD_LIST", arg=1),
Instr("STORE_DEREF", arg=cellvar_x1),
# x2 = [2]
Instr("LOAD_CONST", arg=2),
Instr("BUILD_LIST", arg=1),
Instr("STORE_DEREF", arg=cellvar_x2),
# class Bar:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CLOSURE", arg=cellvar_x1),
Instr("LOAD_CLOSURE", arg=freevar_x2),
Instr("BUILD_TUPLE", arg=2),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Bar"),
Instr("MAKE_FUNCTION", arg=8),
Instr("LOAD_CONST", arg="Bar"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_FAST", arg="Bar"),
# class_attr = Bar.y
Instr("LOAD_FAST", arg="Bar"),
Instr("LOAD_ATTR", arg="foo"),
Instr("STORE_FAST", arg="class_attr"),
# class_attr2 = Bar.Foo.y
Instr("LOAD_FAST", arg="Bar"),
Instr("LOAD_ATTR", arg="Foo"),
Instr("LOAD_ATTR", arg="y"),
Instr("STORE_FAST", arg="class_attr2"),
# result = class_attr + class_attr2
Instr("LOAD_FAST", arg="class_attr"),
Instr("LOAD_FAST", arg="class_attr2"),
Instr("BINARY_ADD"),
Instr("STORE_FAST", arg="result"),
# return result
Instr("LOAD_FAST", arg="result"),
Instr("RETURN_VALUE"),
])
bar_block = BasicBlock([
# class Foo:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CLOSURE", arg=cellvar_x2),
Instr("BUILD_TUPLE", arg=1),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Foo"),
Instr("MAKE_FUNCTION", arg=8),
Instr("LOAD_CONST", arg="Foo"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="Foo"),
# foo = x1
Instr("LOAD_CLASSDEREF", arg=freevar_x1),
Instr("STORE_NAME", arg="foo"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
foo_block = BasicBlock([
# y = x2
Instr("LOAD_CLASSDEREF", arg=freevar_x2),
Instr("STORE_NAME", arg="y"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(function_block)
expected_instructions.extend(foo_block)
expected_instructions.extend(bar_block)
dynamic_slice = slice_function_at_return(func.__code__, test_name="test_nested_class_2")
self.assertEqual(func(), [1, 2])
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_generators(self):
# YIELD_VALUE and YIELD_FROM
abc_generator = BasicBlock([
# a = "a"
Instr("LOAD_CONST", arg="a"),
Instr("STORE_FAST", arg="a"),
# yield a
Instr("LOAD_FAST", arg="a"),
Instr("YIELD_VALUE"),
])
abc_xyz_generator = BasicBlock([
# x = "x"
Instr("LOAD_CONST", arg="x"),
Instr("STORE_FAST", arg="x"),
# yield from abc_generator()
Instr("LOAD_GLOBAL", arg="abc_generator"),
Instr("CALL_FUNCTION", arg=0),
Instr("GET_YIELD_FROM_ITER"),
Instr("LOAD_CONST", arg=None),
Instr("YIELD_FROM"),
# yield x
Instr("LOAD_FAST", arg="x"),
Instr("YIELD_VALUE"),
])
end_block = BasicBlock([
# return result
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
loop_block = BasicBlock([
Instr("STORE_NAME", arg="letter"),
])
loop_header = BasicBlock([
Instr("FOR_ITER", arg=end_block),
])
loop_if_true_block = BasicBlock([
Instr("LOAD_NAME", arg="result"),
Instr("LOAD_NAME", arg="letter"),
Instr("INPLACE_ADD"),
Instr("STORE_NAME", arg="result"),
Instr("JUMP_ABSOLUTE", arg=loop_header),
])
loop_if_x_block = BasicBlock([
Instr("LOAD_NAME", arg="letter"),
Instr("LOAD_CONST", arg="x"),
Instr("COMPARE_OP", arg=Compare.EQ),
Instr("POP_JUMP_IF_TRUE", arg=loop_if_true_block),
])
loop_if_a_block = BasicBlock([
Instr("LOAD_NAME", arg="letter"),
Instr("LOAD_CONST", arg="a"),
Instr("COMPARE_OP", arg=Compare.EQ),
Instr("POP_JUMP_IF_FALSE", arg=loop_header)
])
module_block = BasicBlock([
# def abc_generator():
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="abc_generator"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="abc_generator"),
# def abc_xyz_generator():
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="abc_xyz_generator"),
Instr("MAKE_FUNCTION", arg=0),
Instr("STORE_NAME", arg="abc_xyz_generator"),
# generator = abc_xyz_generator()
Instr("LOAD_NAME", arg="abc_xyz_generator"),
Instr("CALL_FUNCTION", arg=0),
Instr("STORE_NAME", arg="generator"),
# result = ""
Instr("LOAD_CONST", arg=""),
Instr("STORE_NAME", arg="result"),
# for letter in generator:
Instr("LOAD_NAME", arg="generator"),
Instr("GET_ITER"),
])
expected_instructions = []
expected_instructions.extend(module_block)
expected_instructions.extend(loop_header)
expected_instructions.extend(loop_block)
expected_instructions.extend(loop_if_x_block)
expected_instructions.extend(loop_if_a_block)
expected_instructions.extend(loop_if_true_block)
expected_instructions.extend(end_block)
expected_instructions.extend(abc_xyz_generator)
expected_instructions.extend(abc_generator)
module_file = "generator.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
self.assertEqual(len(dynamic_slice.sliced_instructions), len(expected_instructions))
self.assertTrue(compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_equal_variable_names(self):
# Data dependencies across modules (explicit, full cover)
main_module_block = BasicBlock([
# class Foo:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Foo"),
Instr("MAKE_FUNCTION", arg=0),
Instr("LOAD_CONST", arg="Foo"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="Foo"),
# duplicate_var = "foo_dup"
Instr("LOAD_CONST", arg="foo_dup"),
Instr("STORE_NAME", arg="duplicate_var"),
# import tests.slicer.integration.example_modules.equal_variable_names_def
# Instr("LOAD_CONST", arg=0),
# Instr("LOAD_CONST", arg=None),
# Instr("IMPORT_NAME", arg="tests.slicer.integration.example_modules.equal_variable_names_def"),
# Instr("STORE_NAME", arg="tests"),
# test = duplicate_var
Instr("LOAD_NAME", arg="duplicate_var"),
Instr("STORE_NAME", arg="test"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
# result = Foo.test
Instr("LOAD_NAME", arg="Foo"),
Instr("LOAD_ATTR", arg="test"),
Instr("STORE_NAME", arg="result"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
dependency_module_block = BasicBlock([
# duplicate_var = "bar_dup"
# Instr("LOAD_CONST", arg="bar_dup"),
# Instr("STORE_NAME", arg="duplicate_var"),
# Instr("LOAD_CONST", arg=None),
# Instr("RETURN_VALUE")
])
expected_instructions = []
expected_instructions.extend(main_module_block)
expected_instructions.extend(dependency_module_block)
module_dependency_file = "equal_variable_names_def.py"
module_dependency_path = example_modules_path + module_dependency_file
instrument_module(module_dependency_path)
module_file = "equal_variable_names_main.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
compile_module(module_dependency_path)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
def test_data_dependency_6(self):
# Data dependencies across modules (explicit, full cover)
main_module_block = BasicBlock([
# from tests.slicer.integration.example_modules.module_dependency_def import module_list, Foo
Instr("LOAD_CONST", arg=0),
Instr("LOAD_CONST", arg=('module_list', 'unused_list', 'Foo')),
Instr("IMPORT_NAME",
arg="tests.slicer.example_modules.module_dependency_def"),
Instr("IMPORT_FROM", arg="module_list"),
Instr("STORE_NAME", arg="module_list"),
# Instr("IMPORT_FROM", arg="unused_list"),
# Instr("STORE_NAME", arg="unused_list"),
Instr("IMPORT_FROM", arg="Foo"),
Instr("STORE_NAME", arg="Foo"),
# result = module_list + Foo.get_class_list()
Instr("LOAD_NAME", arg="module_list"),
Instr("LOAD_NAME", arg="Foo"),
Instr("LOAD_METHOD", arg="get_class_list"),
Instr("CALL_METHOD", arg=0),
Instr("BINARY_ADD"),
Instr("STORE_NAME", arg="result"),
# return
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE")
])
dependency_module_block = BasicBlock([
# module_list = [1, 2, 3]
Instr("LOAD_CONST", arg=1),
Instr("LOAD_CONST", arg=2),
Instr("LOAD_CONST", arg=3),
Instr("BUILD_LIST", arg=3),
Instr("STORE_NAME", arg="module_list"),
# class Foo:
Instr("LOAD_BUILD_CLASS"),
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Foo"),
Instr("MAKE_FUNCTION", arg=0),
Instr("LOAD_CONST", arg="Foo"),
Instr("CALL_FUNCTION", arg=2),
Instr("STORE_NAME", arg="Foo"),
# class_list = [4, 5, 6]
Instr("LOAD_CONST", arg=7),
Instr("LOAD_CONST", arg=8),
Instr("LOAD_CONST", arg=9),
Instr("BUILD_LIST", arg=3),
Instr("STORE_NAME", arg="class_list"),
# @staticmethod
Instr("LOAD_NAME", arg="staticmethod"),
# def get_class_list():
Instr("LOAD_CONST", arg=dummy_code_object),
Instr("LOAD_CONST", arg="Foo.get_class_list"),
Instr("MAKE_FUNCTION", arg=0),
Instr("CALL_FUNCTION", arg=1),
Instr("STORE_NAME", arg="get_class_list"),
# return Foo.class_list
Instr("LOAD_GLOBAL", arg="Foo"),
Instr("LOAD_ATTR", arg="class_list"),
Instr("RETURN_VALUE"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
Instr("LOAD_CONST", arg=None),
Instr("RETURN_VALUE"),
])
expected_instructions = []
expected_instructions.extend(main_module_block)
expected_instructions.extend(dependency_module_block)
module_dependency_file = "module_dependency_def.py"
module_dependency_path = example_modules_path + module_dependency_file
instrument_module(module_dependency_path)
module_file = "module_dependency_main.py"
module_path = example_modules_path + module_file
dynamic_slice = slice_module_at_return(module_path)
compile_module(module_dependency_path)
self.assertEqual(len(dynamic_slice.sliced_instructions),
len(expected_instructions))
self.assertTrue(
compare(dynamic_slice.sliced_instructions, expected_instructions))
