def test_minimize_ir_minimization_possible(self):
# Add an invalid codegen flag to inject an error into the running of the
# sample. The error is unconditional so IR minimization should be able to
# reduce the sample to a minimal function (just returns a parameter).
s = sample.Sample(
'fn main(x: u8) -> u8 { -x }',
sample.SampleOptions(codegen=True,
codegen_args=('--invalid_flag!!!', )),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
with self.assertRaises(sample_runner.SampleError):
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(s, run_dir)
self.assertIsNotNone(minimized_ir_path)
self.assertIn('ir_minimizer_test.sh', os.listdir(run_dir))
# Sanity check the minimized IR.
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_evaluate_ir_wide(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: bits[100], y: bits[100]) -> bits[100] { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(10**30)),
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(10**30)),
],
[
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(2**80)),
interp_value_from_ir_string(
'bits[100]:{0:#x}'.format(2**81)),
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.ir.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
def test_evaluate_ir_wide(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: bits[100], y: bits[100]) -> bits[100] { x + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100))
],
[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**80, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**81, bit_count=100))
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.ir.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
def test_to_crasher(self):
s = sample.Sample(
'fn main(x: u8, y: u8) -> u8 {\nx + y\n}',
sample.SampleOptions(input_is_dslx=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:42; bits[8]:11\nbits[8]:44; bits[8]:99'))
crasher = s.to_crasher()
self.assertTrue(
crasher.startswith('// Copyright'),
msg=f'Crasher does not start with copyright:\n{crasher}')
self.assertIn(
textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": true, "input_is_dslx": true, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_jit": true, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
fn main(x: u8, y: u8) -> u8 {
x + y
}
"""), crasher)
def test_dslx_to_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False)))
self.assertIn('package sample', _read_file(sample_dir, 'sample.ir'))
def test_to_crasher_with_error_message(self):
s = sample.Sample(
'fn main(x: u8, y: u8) -> u8 {\nx + y\n}',
sample.SampleOptions(input_is_dslx=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:42; bits[8]:11\nbits[8]:44; bits[8]:99'))
crasher = s.to_crasher('oh no\nI crashed\n')
self.assertTrue(
crasher.startswith('// Copyright'),
msg=f'Crasher does not start with copyright:\n{crasher}')
# Split D.N.S. string to avoid triggering presubmit checks.
self.assertIn(
'Issue: DO NOT ' + 'SUBMIT Insert link to GitHub issue here.',
crasher)
self.assertIn(
textwrap.dedent("""\
// Exception:
// oh no
// I crashed"""), crasher)
self.assertIn(
textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": true, "input_is_dslx": true, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_jit": true, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
fn main(x: u8, y: u8) -> u8 {
x + y
}
"""), crasher)
def test_from_ir_crasher_with_codegen(self):
crasher = textwrap.dedent("""\
// options: {"codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "convert_to_ir": false, "input_is_dslx": false, "optimize_ir": true, "simulate": true, "simulator": "goat simulator", "use_system_verilog": false}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}
""")
got = sample.Sample.from_crasher(crasher)
want = sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(input_is_dslx=False,
convert_to_ir=False,
optimize_ir=True,
codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=False),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63'))
self.assertMultiLineEqual(want.to_crasher(), got.to_crasher())
self.assertEqual(got, want)
def test_from_crasher_without_codegen(self):
crasher = textwrap.dedent("""\
// options: {"input_is_dslx": false, "convert_to_ir": false, "optimize_ir": true, "codegen": true, "codegen_args": null, "simulate": false, "simulator": null, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}""")
got = sample.Sample.from_crasher(crasher)
want = sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(input_is_dslx=False,
convert_to_ir=False,
codegen=True,
codegen_args=None,
simulate=False,
simulator=None,
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63'))
self.assertEqual(got, want)
def run_fuzz(
rng: ast_generator.RngState,
ast_generator_options: ast_generator.AstGeneratorOptions,
calls_per_sample: int,
save_temps: bool,
sample_count: int,
codegen: bool,
simulate: bool = False,
return_samples: bool = False) -> Optional[Tuple[sample.Sample, ...]]:
"""Runs a fuzzing loop for "sample_count" samples."""
samples = []
for i in range(sample_count):
smp = sample_generator.generate_sample(
rng, ast_generator_options, calls_per_sample,
sample.SampleOptions(convert_to_ir=True,
optimize_ir=True,
codegen=codegen,
simulate=simulate))
if return_samples:
samples.append(smp)
termcolor.cprint('=== Sample {}'.format(i), color='yellow')
print(smp.input_text)
sample_dir = tempfile.mkdtemp('run_fuzz_')
run_sample(smp, sample_dir)
if not save_temps:
shutil.rmtree(sample_dir)
if return_samples:
return tuple(samples)
def test_interpret_dslx_single_value(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_options_from_json_empty(self):
json_text = '{}'
want = sample.SampleOptions()
got = sample.SampleOptions.from_json(json_text)
self.assertEqual(got, want)
want_text = (
'{"codegen": false, "codegen_args": null, "convert_to_ir": true, '
'"input_is_dslx": true, "optimize_ir": true, "simulate": false, '
'"simulator": null, "use_jit": true, "use_system_verilog": true}')
self.assertEqual(got.to_json(), want_text)
def test_interpret_mixed_signedness_unsigned_inputs(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
sample.parse_args_batch('bits[8]:0xb0; bits[8]:0x0a')))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0xba')
def test_interpret_mixed_signedness(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_sbits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_interpret_mixed_signedness(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False), [[
interp_value_from_ir_string('bits[8]:42'),
interp_value_from_ir_string('bits[8]:100').to_signed()
]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_bad_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """bogus ir string"""
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(ir_text,
sample.SampleOptions(input_is_dslx=False)))
self.assertIn('Expected \'package\' keyword',
_read_file(sample_dir, 'opt_main.stderr'))
self.assertRegex(_read_file(sample_dir, 'exception.txt'),
'.*opt_main.*returned non-zero exit status')
def test_interpret_invalid_dslx(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'syntaxerror!!! fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# Verify the exception text is written out to file.
self.assertIn('Expected start of top-level construct',
_read_file(sample_dir, 'exception.txt'))
def test_options_to_json(self):
json_str = sample.SampleOptions(codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True,
simulator='iverilog').to_json()
sample_json = json.loads(json_str)
self.assertEqual(sample_json['input_is_dslx'], True)
self.assertEqual(sample_json['codegen'], True)
self.assertSequenceEqual(
sample_json['codegen_args'],
('--generator=pipeline', '--pipeline_stages=2'))
self.assertEqual(sample_json['simulator'], 'iverilog')
def test_interpret_opt_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package sample', _read_file(sample_dir,
'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def setup_worker():
"""Creates arguments to repeatedly pass to benchmark_worker."""
rng = ast_generator.RngState(0)
smp = ast_generator.generate_sample(
ast_generator.AstGeneratorOptions(disallow_divide=DISALLOW_DIVIDE),
CALLS_PER_SAMPLE,
sample.SampleOptions(convert_to_ir=True,
optimize_ir=True,
codegen=FLAGS.codegen,
simulate=FLAGS.simulate), rng)
run_dir = tempfile.mkdtemp('run_fuzz_')
return (run_dir, smp)
def test_interpret_dslx_multiple_values(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)],
[Value.make_ubits(8, 222),
Value.make_ubits(8, 240)]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'),
['bits[8]:0x8e', 'bits[8]:0xce'])
def test_minimize_ir_no_minimization_possible(self):
# Verify that IR minimization at least generates a minimization test script
# and doesn't blow up if the IR is not minimizable. In this case, "not
# minimizable" means that no error is ever generated when running the
# sample.
s = sample.Sample('fn main(x: u8) -> u8 { -x }',
sample.SampleOptions(),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
self.assertIsNone(run_fuzz.minimize_ir(s, run_dir))
dir_contents = os.listdir(run_dir)
self.assertIn('ir_minimizer_test.sh', dir_contents)
def test_interpret_dslx_multiple_values(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[
interp_value_from_ir_string('bits[8]:42'),
interp_value_from_ir_string('bits[8]:100')
],
[
interp_value_from_ir_string('bits[8]:222'),
interp_value_from_ir_string('bits[8]:240')
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'),
['bits[8]:0x8e', 'bits[8]:0xce'])
def test_evaluate_ir_miscompare_multiple_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
results = (Value.make_ubits(8, 100), Value.make_ubits(8, 1))
runner._evaluate_ir = lambda *args: results
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 40),
Value.make_ubits(8, 60)],
[Value.make_ubits(8, 2),
Value.make_ubits(8, 1)]]))
self.assertIn('Result miscompare for sample 1', str(e.exception))
self.assertIn('Result miscompare for sample 1',
_read_file(sample_dir, 'exception.txt'))
def test_codegen_combinational_wrong_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
runner._simulate = lambda *args: (Value.make_ubits(8, 1), )
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True,
simulator='iverilog'),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
def test_minimize_jit_interpreter_mismatch(self):
s = sample.Sample(
'fn main(x: u8) -> u8 { !x }', sample.SampleOptions(),
sample.parse_args_batch('bits[8]:0xff\nbits[8]:0x42'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(
s, run_dir, inject_jit_result='bits[32]:0x0')
self.assertIsNotNone(minimized_ir_path)
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """package foo
fn foo(x: bits[8], y: bits[8]) -> bits[8] {
ret add.1: bits[8] = add(x, y)
}
"""
runner.run(
sample.Sample(ir_text, sample.SampleOptions(input_is_dslx=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package foo', _read_file(sample_dir, 'sample.ir'))
self.assertIn('package foo', _read_file(sample_dir, 'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def test_codegen_pipeline(self):
sample_dir = self._make_sample_dir()
print('sample_dir = ' + sample_dir)
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(codegen=True,
codegen_args=('--generator=pipeline',
'--pipeline_stages=2'),
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# A pipelined block should have a blocking assignment.
self.assertIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'),
['bits[8]:0x8e'])
def test_evaluate_ir_miscompare_single_result(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner._evaluate_ir = lambda *args: (Value.make_ubits(8, 1), )
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn(
'Result miscompare for sample 0:\n'
'args: bits[8]:0x2a; bits[8]:0x64\n'
'evaluated unopt IR (JIT), evaluated unopt IR (interpreter) =\n'
' bits[8]:0x1\n'
'interpreted DSLX =\n'
' bits[8]:0x8e', str(e.exception))
self.assertIn('Result miscompare for sample 0',
_read_file(sample_dir, 'exception.txt'))
def test_codegen_combinational(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('endmodule', _read_file(sample_dir, 'sample.v'))
# A combinational block should not have a blocking assignment.
self.assertNotIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'),
['bits[8]:0x8e'])
def test_options_from_json(self):
json_text = (
'{"codegen": true, "codegen_args": ["--generator=pipeline", '
'"--pipeline_stages=2"], "convert_to_ir": true, "input_is_dslx": true,'
' "optimize_ir": true, "simulate": false, "simulator": null, '
'"use_jit": true, "use_system_verilog": true}')
expected_object = sample.SampleOptions(
input_is_dslx=True,
convert_to_ir=True,
optimize_ir=True,
use_jit=True,
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=False,
simulator=None,
use_system_verilog=True)
self.assertEqual(sample.SampleOptions.from_json(json_text),
expected_object)
self.assertEqual(
sample.SampleOptions.from_json(json_text).to_json(), json_text)