def test_showing_more_jit_codes_when_parsing_something_random(self): """ This test is equivalent to the above test_showing_fewer_jit_codes_with_no_parser except that the Parser(...) IS used, but to parse something completely different. This brings the JIT code count back to what it was in test_for_loop_converted_to_while: [jitcodewriter:info] there are 76 JitCode instances. However, the manual construction keeps the loop size the same as test_showing_fewer_jit_codes_with_no_parser: # Loop 1 (While(condition=LessThanOrEquals(left=LValue(name=i, next=None), right=IntegerValue(9)), body=Sequence (expressions=[Assign(lvalue=LValue(name=a, next=None), expression=Subtract(left=LValue(name=a, next=None), right =LValue(name=i, next=None))), Assign(lvalue=LValue(name=i, next=None), expression=Add(left=LValue(name=i, next=N one), right=IntegerValue(1)))]))) : loop with 162 ops """ def test(): unused_program = Parser('let var a := 0 in a end') # this is never used program = Let(declarations=[VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0))], expressions=[Sequence( expressions=[For(var='i', start=IntegerValue(1), end=IntegerValue(9), body=Assign(lvalue=LValue(name='a', next_lvalue=None), expression=Subtract( left=LValue(name='a', next_lvalue=None), right=LValue(name='i', next_lvalue=None)))), LValue(name='a', next_lvalue=None)])]) program = promote(program) environment = create_environment_with_natives() # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.interpret_in_python(test, []), -45) self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), -45)
def test_merge_point_with_can_enter_jit(self): """ When a can_enter_jit is placed in A and the merge point is moved to the top of the "dispatch loop", we see the same 13-operation trace as in 'test_merge_point_inside_branch_a' but instead of B and C both being executed in the blackhole interpreter, we only see B. The additions for C (e.g. incrementing x from 81 to 90) are executed in the normal (non-trace, non-blackhole) interpreter. """ def get_location(x): return "x=%d" % (x) jitdriver = JitDriver(greens=['x'], reds=['y'], get_printable_location=get_location) def test(): x = 1 y = 0 while True: jitdriver.jit_merge_point(x=x, y=y) if x % 10 == 0: y += 2 jitdriver.can_enter_jit(x=x, y=y) if y >= 100: y = 0 x += 1 elif x >= 100: break else: x += 1 return x self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 101) """
def test_merge_point_inside_branch_a(self): """ Placing the merge point inside branch A (from above) results in the four of the additions to y being compiled in to a 13-operation trace and the additions to x in B and C being executed in the blackhole interpreter """ def get_location(x): return "x=%d" % x jitdriver = JitDriver(greens=['x'], reds=['y'], get_printable_location=get_location) def test(): x = 1 y = 0 while True: if x % 10 == 0: y += 2 jitdriver.jit_merge_point(x=x, y=y) # it does not make sense to: y = promote(y) # this causes the traces to promptly fail when the guard on y is invalidated if y >= 100: y = 0 x += 1 elif x >= 100: break else: x += 1 return x self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 101) """
def test_merge_point_with_both_x_and_y(self): """ Keying the merge point on both x and y does not make sense: no traces are ever produced because the hash of x and y is never the same value twice. """ def get_location(x, y): return "x=%d, y=%d" % (x, y) jitdriver = JitDriver(greens=['x', 'y'], reds='auto', get_printable_location=get_location) def test(): x = 1 y = 0 while True: if x % 10 == 0: y += 2 jitdriver.jit_merge_point(x=x, y=y) if y >= 100: y = 0 x += 1 elif x >= 100: break else: x += 1 return x self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 101)
def test_current_implementation(self): """ # Loop 1 (While(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(100)), body=Assign(lvalue= LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=IntegerValue(1))))) : loop with 88 ops """ def test(): program = Parser( 'let var a := 0 in (while a < 100 do a := a + 1; a) end' ).parse(create_native_functions()) environment = create_environment_with_natives( ) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 100)
def test_meta_interpretation(self): """ Verify that meta-interpretation works as expected """ def print_location(a): return "a=%d" % a jitdriver = JitDriver(greens=['a'], reds='auto', get_printable_location=print_location) def test(a, b): while a > b: jitdriver.jit_merge_point(a=a) b += 1 return 42 self.assertEqual( interpretation_mechanisms.interpret_in_python(test, [10, 0]), 42) self.assertEqual( interpretation_mechanisms.meta_interpret(test, [10, 0]), 42)
def test_sumprimes(self): def test(): program = Parser(""" let var max : int := 50 var s : int := 0 var n : int := 2 in while n <= max do let var p : int := 1 var d : int := 2 in while d <= (n - 1) do let var m : int := d * (n / d) in if n <= m then p := 0; d := d + 1 end; if p <> 0 then s := s + n; n := n + 1 end; s end """).parse() environment = create_environment_with_natives( ) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 328)
def test_merge_point_inside_a_called_function(self): """ This test shows that the merge point must be carefully placed within the same basic block as the loop; the original for-loop implementation (i.e. no while conversion) is used and the merge point is located in a call to loop(). The trace is only 40 operations long (see end of method) but, though it does capture the 'a := a - i' subtraction of the loop body, it does not capture an expected backwards jump at the end of the trace. I believe that this is why it is classified not as a loop but as an entry bridge and results in no execution speed-up (see my results from spring 2018 term report). """ def get_location(code, exp, env): return "%s" % code.to_string() jitdriver = JitDriver(greens=['code', 'expression', 'environment'], reds='auto', get_printable_location=get_location) # adding this in brings the number of operations down from 105 to 31... still not 18 def loop(code, expression, environment): jitdriver.jit_merge_point(code=code, expression=expression, environment=environment) return expression.evaluate(environment) class ExternalMergePointFor(For): """ This is the original implementation of a Tiger for-loop (i.e. no while conversion) """ _immutable_ = True def evaluate(self, env): env.push() start_value = self.start.evaluate(env) assert isinstance(start_value, IntegerValue) end_value = self.end.evaluate(env) assert isinstance(end_value, IntegerValue) iterator = IntegerValue(start_value.integer) for i in range(iterator.integer, end_value.integer + 1): iterator.integer = i env.set_current_level(self.var, iterator) try: result = loop(self, self.body, env) assert result is None except BreakException: break env.pop() def test(): # adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations Parser('let var a := 0 in a end').parse() program = Let(declarations=[VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0))], expressions=[Sequence( expressions=[ExternalMergePointFor(var='i', start=IntegerValue(1), end=IntegerValue(9), body=Assign(lvalue=LValue(name='a', next_lvalue=None), expression=Subtract( left=LValue(name='a', next_lvalue=None), right=LValue(name='i', next_lvalue=None)))), LValue(name='a', next_lvalue=None)])]) environment = create_environment_with_natives() # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), -45) """
def test_not_working_example(self): """ We build a list of AST nodes: some are one-arg native functions and some are zero-arg native functions. RPython seems unable to determine which one it is virtually dispatching to: AnnotatorError: signature mismatch: native_print_newline() takes no arguments (1 given) Occurred processing the following simple_call: function native_print_newline <.../src/experimental/native_functions.py, line 76> returning function native_print <.../src/experimental/native_functions.py, line 72> returning v2 = simple_call(v0, v1) In <FunctionGraph of (native_functions:69)OneArgFunction.call at 0x7f714ff320d0>: Happened at file .../src/experimental/native_functions.py line 70 ==> return self.function(arguments[0]) Known variable annotations: v0 = SomePBC(can_be_None=False, descriptions={...2...}, knowntype=function, subset_of=None) v1 = SomeString(no_nul=True) """ class NativeFunction: def __init__(self, function): self.function = function def call(self, arguments): raise Exception("Use descendants") class ZeroArgFunction(NativeFunction): def call(self, arguments): return self.function() def native_print_newline(): print('no-arg\n') return 1 class OneArgFunction(NativeFunction): def call(self, arguments): return self.function(arguments[0]) def native_print(s): print('one-arg: %s\n' % s) return len(s) def get_location(code): return "%s" % code.__class__.__name__ jitdriver = JitDriver(greens=['code'], reds='auto', get_printable_location=get_location) def test(a, b, c): arguments = ['as', 'b', 'c'] assert isinstance(arguments, list) program = [ ZeroArgFunction(native_print_newline), OneArgFunction(native_print), ZeroArgFunction(native_print_newline) ] result = 0 for node in program: jitdriver.jit_merge_point(code=node) result = node.call(arguments) return result # eventually calls something like: return LLJitMixin().meta_interp(function, arguments, listops=True, inline=True) self.assertEqual( interpretation_mechanisms.meta_interpret(test, [41, 42, 43]), 1)
def test_working_example(self): """ With help from the PyPy crew (Alex Gaynor, Antonio Cuni, see https://botbot.me/freenode/pypy/2018-10-31/?msg=105890623&page=1) I was able to get the above to work """ class NativeFunction: _attrs_ = [] def call(self, arguments): raise Exception("Use descendants") class ZeroArgFunction(NativeFunction): _attrs_ = ['function'] def __init__(self, function): self.function = function def call(self, arguments): return self.function() def native_print_newline(): print('no-arg\n') return 1 class OneArgFunction(NativeFunction): _attrs_ = ['function'] def __init__(self, function): self.function = function def call(self, arguments): return self.function(arguments[0]) def native_print(s): print('one-arg: %s\n' % s) return len(s) def get_location(code): return "%s" % code.__class__.__name__ jitdriver = JitDriver(greens=['code'], reds='auto', get_printable_location=get_location) def test(a, b, c): arguments = ['as', 'b', 'c'] assert isinstance(arguments, list) program = [ ZeroArgFunction(native_print_newline), OneArgFunction(native_print), ZeroArgFunction(native_print_newline) ] result = 0 for node in program: jitdriver.jit_merge_point(code=node) result = node.call(arguments) return result # eventually calls something like: return LLJitMixin().meta_interp(function, arguments, listops=True, inline=True) self.assertEqual( interpretation_mechanisms.meta_interpret(test, [41, 42, 43]), 1)
def test_larger_merge_point_key(self): """ This has the same merge point location as the current implementation but uses a merge point key with more variables: code, expression, environment. I found that this larger merge point key not only failed to recognize loops in sumprimes but the JIT-compiling binary would not terminate. # Loop 1 (ModifiedWhile(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(100)), body=Assign (lvalue=LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=IntegerValue(1))))) : loop with 85 ops """ def get_location(code, exp, env): return "%s" % code.to_string() jitdriver = JitDriver(greens=['code', 'expression', 'environment'], reds='auto', get_printable_location=get_location) class ModifiedWhile(While): _immutable_ = True def evaluate(self, env): condition_value = self.condition.evaluate(env) assert isinstance(condition_value, IntegerValue) result = None while condition_value.integer != 0: jitdriver.jit_merge_point(code=self, expression=self.body, environment=env) try: result = self.body.evaluate(env) except BreakException: break condition_value = self.condition.evaluate(env) return result def test(): # adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations Parser('let var a := 0 in a end').parse() program = Let( declarations=[ VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0)) ], expressions=[ Sequence(expressions=[ ModifiedWhile(condition=LessThan( left=LValue('a'), right=IntegerValue(100)), body=Assign(lvalue=LValue(name='a'), expression=Add( left=LValue(name='a'), right=IntegerValue(1)))), LValue(name='a', next_lvalue=None) ]) ]) environment = create_environment_with_natives( ) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 100)
def test_virtualized_while_loop_changed_merge_point_with_virtualizable( self): """ This is a (currently failed) attempt to virtualize the current level's local variables through the environment display """ # requires enabling _virtualizable_ in EnvironmentLevel EnvironmentLevel._virtualizable_ = ['parent', 'expressions[*]'] # Environment._virtualizable_ = ['local_variables', 'local_types'] def get_location(code): return "%s" % code.to_string() jitdriver = JitDriver(greens=['code'], reds=['env', 'vars', 'result'], virtualizables=['vars'], get_printable_location=get_location) class ModifiedWhile(While): _immutable_ = True def evaluate(self, env): condition_value = self.condition.evaluate(env) assert isinstance(condition_value, IntegerValue) result = None while condition_value.integer != 0: jitdriver.jit_merge_point(code=self, env=env, vars=env.local_variables, result=result) try: result = self.body.evaluate(env) except BreakException: break condition_value = self.condition.evaluate(env) return result def test(): # adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations unused = Parser( 'let var a := 0 in (while a < 100 do a := a + 1) end').parse() program = Let( declarations=[ VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0)) ], expressions=[ Sequence(expressions=[ ModifiedWhile(condition=LessThan( left=LValue('a'), right=IntegerValue(100)), body=Assign(lvalue=LValue(name='a'), expression=Add( left=LValue(name='a'), right=IntegerValue(1)))), LValue(name='a', next_lvalue=None) ]) ]) environment = create_environment_with_natives( ) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 100)
def test_nested_while_loops(self): """ Weirdly, this test takes about 80 seconds on my machine. It has one bridge and two loops: # Loop 1 (ModifiedWhile(condition=LessThan(left=LValue(name=b, next=None), right=IntegerValue(100)), body=Sequen ce(expressions=[Assign(lvalue=LValue(name=b, next=None), expression=Add(left=LValue(name=b, next=None), right=In tegerValue(1)))]))) : loop with 123 ops # Loop 3 (ModifiedWhile(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(50)), body=Sequenc e(expressions=[Assign(lvalue=LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=Int egerValue(1))), ModifiedWhile(condition=LessThan(left=LValue(name=b, next=None), right=IntegerValue(100)), body= Sequence(expressions=[Assign(lvalue=LValue(name=b, next=None), expression=Add(left=LValue(name=b, next=None), ri ght=IntegerValue(1)))])), Assign(lvalue=LValue(name=b, next=None), expression=IntegerValue(0))]))) : loop with 231 ops """ def get_location(code): return "%s" % code.to_string() jitdriver = JitDriver(greens=['code'], reds='auto', get_printable_location=get_location) class ModifiedWhile(While): _immutable_ = True def evaluate(self, env): condition_value = self.condition.evaluate(env) assert isinstance(condition_value, IntegerValue) result = None while condition_value.integer != 0: jitdriver.jit_merge_point(code=self) try: result = self.body.evaluate(env) except BreakException: break condition_value = self.condition.evaluate(env) return result def test(): # this is equivalent to what is constructed below unused = Parser(""" let var a := 0 var b := 0 in (while a < 50 do (a := a + 1; while b < 100 do (b := b + 1); b := 0 ); a) end""").parse() program = Let( declarations=[ VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0)), VariableDeclaration(name='b', type_id=None, expression=IntegerValue(0)) ], expressions=[ Sequence(expressions=[ ModifiedWhile( condition=LessThan(left=LValue(name='a', next_lvalue=None), right=IntegerValue(50)), body=Sequence(expressions=[ Assign(lvalue=LValue(name='a', next_lvalue=None), expression=Add(left=LValue( name='a', next_lvalue=None), right=IntegerValue(1))), ModifiedWhile( condition=LessThan( left=LValue(name='b', next_lvalue=None), right=IntegerValue(100)), body=Sequence(expressions=[ Assign(lvalue=LValue(name='b', next_lvalue=None), expression=Add( left=LValue( name='b', next_lvalue=None), right=IntegerValue(1))) ])), Assign(lvalue=LValue(name='b', next_lvalue=None), expression=IntegerValue(0)) ])), LValue(name='a', next_lvalue=None) ]) ]) environment = create_environment_with_natives( ) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field result = program.evaluate(environment) assert isinstance(result, IntegerValue) return result.integer self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), 50)