def test_showing_more_jit_codes_when_parsing_something_random(self):
"""
This test is equivalent to the above test_showing_fewer_jit_codes_with_no_parser except that the Parser(...) IS
used, but to parse something completely different. This brings the JIT code count back to what it was in
test_for_loop_converted_to_while: [jitcodewriter:info] there are 76 JitCode instances. However, the manual
construction keeps the loop size the same as test_showing_fewer_jit_codes_with_no_parser:
# Loop 1 (While(condition=LessThanOrEquals(left=LValue(name=i, next=None), right=IntegerValue(9)), body=Sequence
(expressions=[Assign(lvalue=LValue(name=a, next=None), expression=Subtract(left=LValue(name=a, next=None), right
=LValue(name=i, next=None))), Assign(lvalue=LValue(name=i, next=None), expression=Add(left=LValue(name=i, next=N
one), right=IntegerValue(1)))]))) : loop with 162 ops
"""
def test():
unused_program = Parser('let var a := 0 in a end') # this is never used
program = Let(declarations=[VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0))],
expressions=[Sequence(
expressions=[For(var='i', start=IntegerValue(1), end=IntegerValue(9),
body=Assign(lvalue=LValue(name='a', next_lvalue=None),
expression=Subtract(
left=LValue(name='a', next_lvalue=None),
right=LValue(name='i',
next_lvalue=None)))),
LValue(name='a', next_lvalue=None)])])
program = promote(program)
environment = create_environment_with_natives() # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.interpret_in_python(test, []), -45)
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), -45)
def test_merge_point_with_can_enter_jit(self):
"""
When a can_enter_jit is placed in A and the merge point is moved to the top of the "dispatch loop", we see the
same 13-operation trace as in 'test_merge_point_inside_branch_a' but instead of B and C both being executed
in the blackhole interpreter, we only see B. The additions for C (e.g. incrementing x from 81 to 90) are
executed in the normal (non-trace, non-blackhole) interpreter.
"""
def get_location(x):
return "x=%d" % (x)
jitdriver = JitDriver(greens=['x'],
reds=['y'],
get_printable_location=get_location)
def test():
x = 1
y = 0
while True:
jitdriver.jit_merge_point(x=x, y=y)
if x % 10 == 0:
y += 2
jitdriver.can_enter_jit(x=x, y=y)
if y >= 100:
y = 0
x += 1
elif x >= 100:
break
else:
x += 1
return x
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
101)
"""
def test_merge_point_inside_branch_a(self):
"""
Placing the merge point inside branch A (from above) results in the four of the additions to y being compiled
in to a 13-operation trace and the additions to x in B and C being executed in the blackhole interpreter
"""
def get_location(x):
return "x=%d" % x
jitdriver = JitDriver(greens=['x'],
reds=['y'],
get_printable_location=get_location)
def test():
x = 1
y = 0
while True:
if x % 10 == 0:
y += 2
jitdriver.jit_merge_point(x=x, y=y)
# it does not make sense to: y = promote(y)
# this causes the traces to promptly fail when the guard on y is invalidated
if y >= 100:
y = 0
x += 1
elif x >= 100:
break
else:
x += 1
return x
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
101)
"""
def test_merge_point_with_both_x_and_y(self):
"""
Keying the merge point on both x and y does not make sense: no traces are ever produced because the hash of
x and y is never the same value twice.
"""
def get_location(x, y):
return "x=%d, y=%d" % (x, y)
jitdriver = JitDriver(greens=['x', 'y'],
reds='auto',
get_printable_location=get_location)
def test():
x = 1
y = 0
while True:
if x % 10 == 0:
y += 2
jitdriver.jit_merge_point(x=x, y=y)
if y >= 100:
y = 0
x += 1
elif x >= 100:
break
else:
x += 1
return x
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
101)
def test_current_implementation(self):
"""
# Loop 1 (While(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(100)), body=Assign(lvalue=
LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=IntegerValue(1))))) :
loop with 88 ops
"""
def test():
program = Parser(
'let var a := 0 in (while a < 100 do a := a + 1; a) end'
).parse(create_native_functions())
environment = create_environment_with_natives(
) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
100)
def test_meta_interpretation(self):
"""
Verify that meta-interpretation works as expected
"""
def print_location(a):
return "a=%d" % a
jitdriver = JitDriver(greens=['a'],
reds='auto',
get_printable_location=print_location)
def test(a, b):
while a > b:
jitdriver.jit_merge_point(a=a)
b += 1
return 42
self.assertEqual(
interpretation_mechanisms.interpret_in_python(test, [10, 0]), 42)
self.assertEqual(
interpretation_mechanisms.meta_interpret(test, [10, 0]), 42)
def test_sumprimes(self):
def test():
program = Parser("""
let
var max : int := 50
var s : int := 0
var n : int := 2
in
while n <= max do
let
var p : int := 1
var d : int := 2
in
while d <= (n - 1) do
let
var m : int := d * (n / d)
in
if n <= m then
p := 0;
d := d + 1
end;
if p <> 0 then
s := s + n;
n := n + 1
end;
s
end
""").parse()
environment = create_environment_with_natives(
) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
328)
def test_merge_point_inside_a_called_function(self):
"""
This test shows that the merge point must be carefully placed within the same basic block as the loop; the
original for-loop implementation (i.e. no while conversion) is used and the merge point is located in a call
to loop(). The trace is only 40 operations long (see end of method) but, though it does capture the 'a := a - i'
subtraction of the loop body, it does not capture an expected backwards jump at the end of the trace. I believe
that this is why it is classified not as a loop but as an entry bridge and results in no execution speed-up
(see my results from spring 2018 term report).
"""
def get_location(code, exp, env):
return "%s" % code.to_string()
jitdriver = JitDriver(greens=['code', 'expression', 'environment'], reds='auto',
get_printable_location=get_location)
# adding this in brings the number of operations down from 105 to 31... still not 18
def loop(code, expression, environment):
jitdriver.jit_merge_point(code=code, expression=expression, environment=environment)
return expression.evaluate(environment)
class ExternalMergePointFor(For):
"""
This is the original implementation of a Tiger for-loop (i.e. no while conversion)
"""
_immutable_ = True
def evaluate(self, env):
env.push()
start_value = self.start.evaluate(env)
assert isinstance(start_value, IntegerValue)
end_value = self.end.evaluate(env)
assert isinstance(end_value, IntegerValue)
iterator = IntegerValue(start_value.integer)
for i in range(iterator.integer, end_value.integer + 1):
iterator.integer = i
env.set_current_level(self.var, iterator)
try:
result = loop(self, self.body, env)
assert result is None
except BreakException:
break
env.pop()
def test():
# adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations
Parser('let var a := 0 in a end').parse()
program = Let(declarations=[VariableDeclaration(name='a', type_id=None, expression=IntegerValue(0))],
expressions=[Sequence(
expressions=[ExternalMergePointFor(var='i', start=IntegerValue(1), end=IntegerValue(9),
body=Assign(lvalue=LValue(name='a', next_lvalue=None),
expression=Subtract(
left=LValue(name='a', next_lvalue=None),
right=LValue(name='i',
next_lvalue=None)))),
LValue(name='a', next_lvalue=None)])])
environment = create_environment_with_natives() # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []), -45)
"""
def test_not_working_example(self):
"""
We build a list of AST nodes: some are one-arg native functions and some are zero-arg native functions. RPython
seems unable to determine which one it is virtually dispatching to:
AnnotatorError:
signature mismatch: native_print_newline() takes no arguments (1 given)
Occurred processing the following simple_call:
function native_print_newline <.../src/experimental/native_functions.py, line 76> returning
function native_print <.../src/experimental/native_functions.py, line 72> returning
v2 = simple_call(v0, v1)
In <FunctionGraph of (native_functions:69)OneArgFunction.call at 0x7f714ff320d0>:
Happened at file .../src/experimental/native_functions.py line 70
==> return self.function(arguments[0])
Known variable annotations:
v0 = SomePBC(can_be_None=False, descriptions={...2...}, knowntype=function, subset_of=None)
v1 = SomeString(no_nul=True)
"""
class NativeFunction:
def __init__(self, function):
self.function = function
def call(self, arguments):
raise Exception("Use descendants")
class ZeroArgFunction(NativeFunction):
def call(self, arguments):
return self.function()
def native_print_newline():
print('no-arg\n')
return 1
class OneArgFunction(NativeFunction):
def call(self, arguments):
return self.function(arguments[0])
def native_print(s):
print('one-arg: %s\n' % s)
return len(s)
def get_location(code):
return "%s" % code.__class__.__name__
jitdriver = JitDriver(greens=['code'],
reds='auto',
get_printable_location=get_location)
def test(a, b, c):
arguments = ['as', 'b', 'c']
assert isinstance(arguments, list)
program = [
ZeroArgFunction(native_print_newline),
OneArgFunction(native_print),
ZeroArgFunction(native_print_newline)
]
result = 0
for node in program:
jitdriver.jit_merge_point(code=node)
result = node.call(arguments)
return result
# eventually calls something like: return LLJitMixin().meta_interp(function, arguments, listops=True, inline=True)
self.assertEqual(
interpretation_mechanisms.meta_interpret(test, [41, 42, 43]), 1)
def test_working_example(self):
"""
With help from the PyPy crew (Alex Gaynor, Antonio Cuni, see
https://botbot.me/freenode/pypy/2018-10-31/?msg=105890623&page=1) I was able to get the above to work
"""
class NativeFunction:
_attrs_ = []
def call(self, arguments):
raise Exception("Use descendants")
class ZeroArgFunction(NativeFunction):
_attrs_ = ['function']
def __init__(self, function):
self.function = function
def call(self, arguments):
return self.function()
def native_print_newline():
print('no-arg\n')
return 1
class OneArgFunction(NativeFunction):
_attrs_ = ['function']
def __init__(self, function):
self.function = function
def call(self, arguments):
return self.function(arguments[0])
def native_print(s):
print('one-arg: %s\n' % s)
return len(s)
def get_location(code):
return "%s" % code.__class__.__name__
jitdriver = JitDriver(greens=['code'],
reds='auto',
get_printable_location=get_location)
def test(a, b, c):
arguments = ['as', 'b', 'c']
assert isinstance(arguments, list)
program = [
ZeroArgFunction(native_print_newline),
OneArgFunction(native_print),
ZeroArgFunction(native_print_newline)
]
result = 0
for node in program:
jitdriver.jit_merge_point(code=node)
result = node.call(arguments)
return result
# eventually calls something like: return LLJitMixin().meta_interp(function, arguments, listops=True, inline=True)
self.assertEqual(
interpretation_mechanisms.meta_interpret(test, [41, 42, 43]), 1)
def test_larger_merge_point_key(self):
"""
This has the same merge point location as the current implementation but uses a merge point key with more
variables: code, expression, environment. I found that this larger merge point key not only failed to recognize
loops in sumprimes but the JIT-compiling binary would not terminate.
# Loop 1 (ModifiedWhile(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(100)), body=Assign
(lvalue=LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=IntegerValue(1))))) :
loop with 85 ops
"""
def get_location(code, exp, env):
return "%s" % code.to_string()
jitdriver = JitDriver(greens=['code', 'expression', 'environment'],
reds='auto',
get_printable_location=get_location)
class ModifiedWhile(While):
_immutable_ = True
def evaluate(self, env):
condition_value = self.condition.evaluate(env)
assert isinstance(condition_value, IntegerValue)
result = None
while condition_value.integer != 0:
jitdriver.jit_merge_point(code=self,
expression=self.body,
environment=env)
try:
result = self.body.evaluate(env)
except BreakException:
break
condition_value = self.condition.evaluate(env)
return result
def test():
# adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations
Parser('let var a := 0 in a end').parse()
program = Let(
declarations=[
VariableDeclaration(name='a',
type_id=None,
expression=IntegerValue(0))
],
expressions=[
Sequence(expressions=[
ModifiedWhile(condition=LessThan(
left=LValue('a'), right=IntegerValue(100)),
body=Assign(lvalue=LValue(name='a'),
expression=Add(
left=LValue(name='a'),
right=IntegerValue(1)))),
LValue(name='a', next_lvalue=None)
])
])
environment = create_environment_with_natives(
) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
100)
def test_virtualized_while_loop_changed_merge_point_with_virtualizable(
self):
"""
This is a (currently failed) attempt to virtualize the current level's local variables through the environment
display
"""
# requires enabling _virtualizable_ in EnvironmentLevel
EnvironmentLevel._virtualizable_ = ['parent', 'expressions[*]']
# Environment._virtualizable_ = ['local_variables', 'local_types']
def get_location(code):
return "%s" % code.to_string()
jitdriver = JitDriver(greens=['code'],
reds=['env', 'vars', 'result'],
virtualizables=['vars'],
get_printable_location=get_location)
class ModifiedWhile(While):
_immutable_ = True
def evaluate(self, env):
condition_value = self.condition.evaluate(env)
assert isinstance(condition_value, IntegerValue)
result = None
while condition_value.integer != 0:
jitdriver.jit_merge_point(code=self,
env=env,
vars=env.local_variables,
result=result)
try:
result = self.body.evaluate(env)
except BreakException:
break
condition_value = self.condition.evaluate(env)
return result
def test():
# adding this line creates more jitcodes in /tmp/usession-exploration-abrown/jitcodes which reduces the number of operations
unused = Parser(
'let var a := 0 in (while a < 100 do a := a + 1) end').parse()
program = Let(
declarations=[
VariableDeclaration(name='a',
type_id=None,
expression=IntegerValue(0))
],
expressions=[
Sequence(expressions=[
ModifiedWhile(condition=LessThan(
left=LValue('a'), right=IntegerValue(100)),
body=Assign(lvalue=LValue(name='a'),
expression=Add(
left=LValue(name='a'),
right=IntegerValue(1)))),
LValue(name='a', next_lvalue=None)
])
])
environment = create_environment_with_natives(
) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
100)
def test_nested_while_loops(self):
"""
Weirdly, this test takes about 80 seconds on my machine. It has one bridge and two loops:
# Loop 1 (ModifiedWhile(condition=LessThan(left=LValue(name=b, next=None), right=IntegerValue(100)), body=Sequen
ce(expressions=[Assign(lvalue=LValue(name=b, next=None), expression=Add(left=LValue(name=b, next=None), right=In
tegerValue(1)))]))) : loop with 123 ops
# Loop 3 (ModifiedWhile(condition=LessThan(left=LValue(name=a, next=None), right=IntegerValue(50)), body=Sequenc
e(expressions=[Assign(lvalue=LValue(name=a, next=None), expression=Add(left=LValue(name=a, next=None), right=Int
egerValue(1))), ModifiedWhile(condition=LessThan(left=LValue(name=b, next=None), right=IntegerValue(100)), body=
Sequence(expressions=[Assign(lvalue=LValue(name=b, next=None), expression=Add(left=LValue(name=b, next=None), ri
ght=IntegerValue(1)))])), Assign(lvalue=LValue(name=b, next=None), expression=IntegerValue(0))]))) :
loop with 231 ops
"""
def get_location(code):
return "%s" % code.to_string()
jitdriver = JitDriver(greens=['code'],
reds='auto',
get_printable_location=get_location)
class ModifiedWhile(While):
_immutable_ = True
def evaluate(self, env):
condition_value = self.condition.evaluate(env)
assert isinstance(condition_value, IntegerValue)
result = None
while condition_value.integer != 0:
jitdriver.jit_merge_point(code=self)
try:
result = self.body.evaluate(env)
except BreakException:
break
condition_value = self.condition.evaluate(env)
return result
def test():
# this is equivalent to what is constructed below
unused = Parser("""
let var a := 0 var b := 0 in
(while a < 50 do
(a := a + 1;
while b < 100 do
(b := b + 1);
b := 0
);
a)
end""").parse()
program = Let(
declarations=[
VariableDeclaration(name='a',
type_id=None,
expression=IntegerValue(0)),
VariableDeclaration(name='b',
type_id=None,
expression=IntegerValue(0))
],
expressions=[
Sequence(expressions=[
ModifiedWhile(
condition=LessThan(left=LValue(name='a',
next_lvalue=None),
right=IntegerValue(50)),
body=Sequence(expressions=[
Assign(lvalue=LValue(name='a',
next_lvalue=None),
expression=Add(left=LValue(
name='a', next_lvalue=None),
right=IntegerValue(1))),
ModifiedWhile(
condition=LessThan(
left=LValue(name='b',
next_lvalue=None),
right=IntegerValue(100)),
body=Sequence(expressions=[
Assign(lvalue=LValue(name='b',
next_lvalue=None),
expression=Add(
left=LValue(
name='b',
next_lvalue=None),
right=IntegerValue(1)))
])),
Assign(lvalue=LValue(name='b',
next_lvalue=None),
expression=IntegerValue(0))
])),
LValue(name='a', next_lvalue=None)
])
])
environment = create_environment_with_natives(
) # apparently RPython barfs if we just use Environment() here because NativeFunctionDeclaration.__init__ is never called so the flowspace does not know about the 'function' field
result = program.evaluate(environment)
assert isinstance(result, IntegerValue)
return result.integer
self.assertEqual(interpretation_mechanisms.meta_interpret(test, []),
50)