def test_evaluate_ir_wide(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: bits[100], y: bits[100]) -> bits[100] { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(10**30)),
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(10**30)),
],
[
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(2**80)),
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(2**81)),
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.ir.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
def test_from_ir_crasher_with_codegen(self):
crasher = textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": false, "input_is_dslx": false, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_system_verilog": false}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}
""")
got = sample.Sample.from_crasher(crasher)
want = sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(input_is_dslx=False,
convert_to_ir=False,
optimize_ir=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=False),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63'))
self.assertMultiLineEqual(want.to_crasher(), got.to_crasher())
self.assertEqual(got, want)
def test_minimize_ir_minimization_possible(self):
# Add an invalid codegen flag to inject an error into the running of the
# sample. The error is unconditional so IR minimization should be able to
# reduce the sample to a minimal function (just returns a parameter).
s = sample.Sample(
'fn main(x: u8) -> u8 { -x }',
sample.SampleOptions(codegen=True,
codegen_args=('--invalid_flag!!!', )),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
with self.assertRaises(sample_runner.SampleError):
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(s, run_dir)
self.assertIsNotNone(minimized_ir_path)
self.assertIn('ir_minimizer_test.sh', os.listdir(run_dir))
# Sanity check the minimized IR.
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_from_crasher_without_codegen(self):
crasher = textwrap.dedent("""\
// options: {"input_is_dslx": false, "convert_to_ir": false, "optimize_ir": true, "codegen": true, "codegen_args": null, "simulate": false, "simulator": null, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}""")
got = sample.Sample.from_crasher(crasher)
want = sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(input_is_dslx=False,
convert_to_ir=False,
codegen=True,
codegen_args=None,
simulate=False,
simulator=None,
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63'))
self.assertEqual(got, want)
def test_evaluate_ir_wide(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: bits[100], y: bits[100]) -> bits[100] { x + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100))
],
[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**80, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**81, bit_count=100))
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.ir.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
def test_to_crasher_with_error_message(self):
s = sample.Sample(
'fn main(x: u8, y: u8) -> u8 {\nx + y\n}',
sample.SampleOptions(input_is_dslx=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:42; bits[8]:11\nbits[8]:44; bits[8]:99'))
crasher = s.to_crasher('oh no\nI crashed\n')
self.assertTrue(
crasher.startswith('// Copyright'),
msg=f'Crasher does not start with copyright:\n{crasher}')
# Split D.N.S. string to avoid triggering presubmit checks.
self.assertIn(
'Issue: DO NOT ' + 'SUBMIT Insert link to GitHub issue here.',
crasher)
self.assertIn(
textwrap.dedent("""\
// Exception:
// oh no
// I crashed"""), crasher)
self.assertIn(
textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": true, "input_is_dslx": true, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_jit": true, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
fn main(x: u8, y: u8) -> u8 {
x + y
}
"""), crasher)
def test_dslx_to_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False)))
self.assertIn('package sample', _read_file(sample_dir, 'sample.ir'))
def test_to_crasher(self):
s = sample.Sample(
'fn main(x: u8, y: u8) -> u8 {\nx + y\n}',
sample.SampleOptions(input_is_dslx=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:42; bits[8]:11\nbits[8]:44; bits[8]:99'))
crasher = s.to_crasher()
self.assertTrue(
crasher.startswith('// Copyright'),
msg=f'Crasher does not start with copyright:\n{crasher}')
self.assertIn(
textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": true, "input_is_dslx": true, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_jit": true, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
fn main(x: u8, y: u8) -> u8 {
x + y
}
"""), crasher)
def test_interpret_dslx_single_value(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_interpret_mixed_signedness_unsigned_inputs(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
sample.parse_args_batch('bits[8]:0xb0; bits[8]:0x0a')))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0xba')
def test_interpret_mixed_signedness(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_sbits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_bad_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """bogus ir string"""
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(ir_text,
sample.SampleOptions(input_is_dslx=False)))
self.assertIn('Expected \'package\' keyword',
_read_file(sample_dir, 'opt_main.stderr'))
self.assertRegex(_read_file(sample_dir, 'exception.txt'),
'.*opt_main.*returned non-zero exit status')
def test_interpret_mixed_signedness(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False), [[
interp_value_from_ir_string('bits[8]:42'),
interp_value_from_ir_string('bits[8]:100').to_signed()
]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_interpret_opt_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package sample', _read_file(sample_dir,
'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def test_interpret_invalid_dslx(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'syntaxerror!!! fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# Verify the exception text is written out to file.
self.assertIn('Expected start of top-level construct',
_read_file(sample_dir, 'exception.txt'))
def test_minimize_ir_no_minimization_possible(self):
# Verify that IR minimization at least generates a minimization test script
# and doesn't blow up if the IR is not minimizable. In this case, "not
# minimizable" means that no error is ever generated when running the
# sample.
s = sample.Sample('fn main(x: u8) -> u8 { -x }',
sample.SampleOptions(),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
self.assertIsNone(run_fuzz.minimize_ir(s, run_dir))
dir_contents = os.listdir(run_dir)
self.assertIn('ir_minimizer_test.sh', dir_contents)
def test_interpret_dslx_multiple_values(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)],
[Value.make_ubits(8, 222),
Value.make_ubits(8, 240)]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'),
['bits[8]:0x8e', 'bits[8]:0xce'])
def test_interpret_dslx_multiple_values(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[
interp_value_from_ir_string('bits[8]:42'),
interp_value_from_ir_string('bits[8]:100')
],
[
interp_value_from_ir_string('bits[8]:222'),
interp_value_from_ir_string('bits[8]:240')
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'),
['bits[8]:0x8e', 'bits[8]:0xce'])
def test_evaluate_ir_miscompare_multiple_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
results = (Value.make_ubits(8, 100), Value.make_ubits(8, 1))
runner._evaluate_ir = lambda *args: results
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 40),
Value.make_ubits(8, 60)],
[Value.make_ubits(8, 2),
Value.make_ubits(8, 1)]]))
self.assertIn('Result miscompare for sample 1', str(e.exception))
self.assertIn('Result miscompare for sample 1',
_read_file(sample_dir, 'exception.txt'))
def test_codegen_combinational_wrong_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
runner._simulate = lambda *args: (Value.make_ubits(8, 1), )
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True,
simulator='iverilog'),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
def test_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """package foo
fn foo(x: bits[8], y: bits[8]) -> bits[8] {
ret add.1: bits[8] = add(x, y)
}
"""
runner.run(
sample.Sample(ir_text, sample.SampleOptions(input_is_dslx=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package foo', _read_file(sample_dir, 'sample.ir'))
self.assertIn('package foo', _read_file(sample_dir, 'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def test_minimize_jit_interpreter_mismatch(self):
s = sample.Sample(
'fn main(x: u8) -> u8 { !x }', sample.SampleOptions(),
sample.parse_args_batch('bits[8]:0xff\nbits[8]:0x42'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(
s, run_dir, inject_jit_result='bits[32]:0x0')
self.assertIsNotNone(minimized_ir_path)
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_codegen_pipeline(self):
sample_dir = self._make_sample_dir()
print('sample_dir = ' + sample_dir)
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# A pipelined block should have a blocking assignment.
self.assertIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'),
['bits[8]:0x8e'])
def test_evaluate_ir_miscompare_single_result(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner._evaluate_ir = lambda *args: (Value.make_ubits(8, 1), )
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn(
'Result miscompare for sample 0:\n'
'args: bits[8]:0x2a; bits[8]:0x64\n'
'evaluated unopt IR (JIT), evaluated unopt IR (interpreter) =\n'
' bits[8]:0x1\n'
'interpreted DSLX =\n'
' bits[8]:0x8e', str(e.exception))
self.assertIn('Result miscompare for sample 0',
_read_file(sample_dir, 'exception.txt'))
def test_codegen_combinational(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('endmodule', _read_file(sample_dir, 'sample.v'))
# A combinational block should not have a blocking assignment.
self.assertNotIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'),
['bits[8]:0x8e'])
def generate_sample(rng: ast_generator.RngState,
ast_options: ast_generator.AstGeneratorOptions,
calls_per_sample: int,
default_options: sample.SampleOptions) -> sample.Sample:
"""Generates and returns a random Sample with the given options."""
assert isinstance(rng, ast_generator.RngState), rng
dslx_text = ast_generator.generate(ast_options, rng)
# Note: we also re-parse here so we can get real positions in error messages.
fn_type = interpreter.get_function_type(dslx_text, 'main')
args_batch = tuple(
generate_arguments(fn_type.params, rng) for _ in range(calls_per_sample))
# The generated sample is DSLX so input_is_dslx must be true.
options = default_options.replace(input_is_dslx=True)
if options.codegen and not options.codegen_args:
# Generate codegen args if codegen is given but no codegen args specified.
options = options.replace(
codegen_args=generate_codegen_args(options.use_system_verilog, rng))
return sample.Sample(dslx_text, options, args_batch)
def test_interpret_opt_ir_miscompare(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
results = [
(Value.make_ubits(8, 100), ), # correct result
(Value.make_ubits(8, 100), ), # correct result
(Value.make_ubits(8, 0), ), # incorrect result
]
def result_gen(*_):
return results.pop(0)
runner._evaluate_ir = result_gen
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(),
[[Value.make_ubits(8, 40),
Value.make_ubits(8, 60)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
self.assertIn('evaluated opt IR (JIT)', str(e.exception))
def test_evaluate_ir_miscompare_multiple_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
results = (interp_value_from_ir_string('bits[8]:100'),
interp_value_from_ir_string('bits[8]:1'))
runner._evaluate_ir = lambda *args: results
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(dslx_text,
sample.SampleOptions(optimize_ir=False),
[[
interp_value_from_ir_string('bits[8]:40'),
interp_value_from_ir_string('bits[8]:60')
],
[
interp_value_from_ir_string('bits[8]:2'),
interp_value_from_ir_string('bits[8]:1')
]]))
self.assertIn('Result miscompare for sample 1', str(e.exception))
self.assertIn('Result miscompare for sample 1',
_read_file(sample_dir, 'exception.txt'))
def test_evaluate_ir_miscompare_number_of_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
# Give back two results instead of one.
def fake_evaluate_ir(*_):
return (Value.make_ubits(8, 100), Value.make_ubits(8, 100))
runner._evaluate_ir = fake_evaluate_ir
args_batch = [(Value.make_ubits(8, 42), Value.make_ubits(8, 64))]
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(dslx_text,
sample.SampleOptions(optimize_ir=False),
args_batch))
self.assertIn(
'Results for evaluated unopt IR (JIT) has 2 values, interpreted DSLX has 1',
str(e.exception))
self.assertIn(
'Results for evaluated unopt IR (JIT) has 2 values, interpreted DSLX has 1',
_read_file(sample_dir, 'exception.txt'))