def test_evaluate_ir_wide(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: bits[100], y: bits[100]) -> bits[100] { x + y }'
runner.run(
sample.Sample(
dslx_text, sample.SampleOptions(optimize_ir=False),
[[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(10**30, bit_count=100))
],
[
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**80, bit_count=100)),
Value.make_bits(Tag.UBITS,
ir_bits.from_long(2**81, bit_count=100))
]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.ir.results'), [
'bits[100]:0x9_3e59_39a0_8ce9_dbd4_8000_0000',
'bits[100]:0x3_0000_0000_0000_0000_0000'
])
def run_fuzz(
rng: random.Random,
ast_generator_options: ast_generator.AstGeneratorOptions,
calls_per_sample: int,
save_temps: bool,
sample_count: int,
codegen: bool,
simulate: bool = False,
return_samples: bool = False) -> Optional[Tuple[sample.Sample, ...]]:
"""Runs a fuzzing loop for "sample_count" samples."""
samples = []
for i in range(sample_count):
smp = sample_generator.generate_sample(
rng, ast_generator_options, calls_per_sample,
sample.SampleOptions(convert_to_ir=True,
optimize_ir=True,
codegen=codegen,
simulate=simulate))
if return_samples:
samples.append(smp)
termcolor.cprint('=== Sample {}'.format(i), color='yellow')
print(smp.input_text)
sample_dir = tempfile.mkdtemp('run_fuzz_')
run_sample(smp, sample_dir)
if not save_temps:
shutil.rmtree(sample_dir)
if return_samples:
return tuple(samples)
def test_dslx_to_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False)))
self.assertIn('package sample', _read_file(sample_dir, 'sample.ir'))
def test_from_crasher_without_codegen(self):
crasher = textwrap.dedent("""\
// options: {"input_is_dslx": false, "convert_to_ir": false, "optimize_ir": true, "codegen": true, "codegen_args": null, "simulate": false, "simulator": null, "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}""")
self.assertEqual(
sample.Sample.from_crasher(crasher),
sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(
input_is_dslx=False,
convert_to_ir=False,
codegen=True,
codegen_args=None,
simulate=False,
simulator=None,
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63')))
def test_from_ir_crasher_with_codegen(self):
crasher = textwrap.dedent("""\
// options: {"input_is_dslx": false, "convert_to_ir": false, "optimize_ir": true, "codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "simulate": true, "simulator": "goat simulator", "use_system_verilog": false}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}
""")
self.assertEqual(
sample.Sample.from_crasher(crasher),
sample.Sample(
textwrap.dedent("""\
package foo
fn bar(x: bits[16], y: bits[16) -> bits[16] {
ret add.1: bits[16] = add(x, y)
}"""),
sample.SampleOptions(
input_is_dslx=False,
convert_to_ir=False,
optimize_ir=True,
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=False),
sample.parse_args_batch(
'bits[8]:0x2a; bits[8]:0xb\nbits[8]:0x2c; bits[8]:0x63')))
def test_minimize_ir_minimization_possible(self):
# Add an invalid codegen flag to inject an error into the running of the
# sample. The error is unconditional so IR minimization should be able to
# reduce the sample to a minimal function (just returns a parameter).
s = sample.Sample(
'fn main(x: u8) -> u8 { -x }',
sample.SampleOptions(codegen=True,
codegen_args=('--invalid_flag!!!', )),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
with self.assertRaises(sample_runner.SampleError):
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(s, run_dir)
self.assertIsNotNone(minimized_ir_path)
self.assertIn('ir_minimizer_test.sh', os.listdir(run_dir))
# Sanity check the minimized IR.
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_interpret_mixed_signedness_unsigned_inputs(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
sample.parse_args_batch('bits[8]:0xb0; bits[8]:0x0a')))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0xba')
def test_interpret_dslx_single_value(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_interpret_mixed_signedness(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: s8) -> s8 { (x as s8) + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_sbits(8, 100)]]))
self.assertEqual(
_read_file(sample_dir, 'sample.x.results').strip(), 'bits[8]:0x8e')
def test_interpret_opt_ir(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package sample', _read_file(sample_dir, 'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def test_interpret_invalid_dslx(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'syntaxerror!!! fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# Verify the exception text is written out to file.
self.assertIn('Expected start of top-level construct',
_read_file(sample_dir, 'exception.txt'))
def test_options_to_json(self):
sample_json = json.loads(
sample.SampleOptions(
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=True,
simulator='iverilog').to_json())
self.assertEqual(sample_json['input_is_dslx'], True)
self.assertEqual(sample_json['codegen'], True)
self.assertSequenceEqual(sample_json['codegen_args'],
('--generator=pipeline', '--pipeline_stages=2'))
self.assertEqual(sample_json['simulator'], 'iverilog')
def test_bad_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """bogus ir string"""
with self.assertRaises(sample_runner.SampleError):
runner.run(
sample.Sample(ir_text, sample.SampleOptions(input_is_dslx=False)))
self.assertIn('Expected \'package\' keyword',
_read_file(sample_dir, 'opt_main.stderr'))
self.assertRegex(
_read_file(sample_dir, 'exception.txt'),
'.*opt_main.*returned non-zero exit status')
def test_evaluate_ir_miscompare_single_result(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner._evaluate_ir = lambda *args: (Value.make_ubits(8, 1),)
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
self.assertIn('Result miscompare for sample 0',
_read_file(sample_dir, 'exception.txt'))
def test_interpret_dslx_multiple_values(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(convert_to_ir=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)],
[Value.make_ubits(8, 222),
Value.make_ubits(8, 240)]]))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.x.results'),
['bits[8]:0x8e', 'bits[8]:0xce'])
def test_minimize_ir_no_minimization_possible(self):
# Verify that IR minimization at least generates a minimization test script
# and doesn't blow up if the IR is not minimizable. In this case, "not
# minimizable" means that no error is ever generated when running the
# sample.
s = sample.Sample('fn main(x: u8) -> u8 { -x }', sample.SampleOptions(),
sample.parse_args_batch('bits[8]:7\nbits[8]:100'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
self.assertIsNone(run_fuzz.minimize_ir(s, run_dir))
dir_contents = os.listdir(run_dir)
self.assertIn('ir_minimizer_test.sh', dir_contents)
def setup_worker():
"""Creates arguments to repeatedly pass to benchmark_worker."""
rng = random.Random(0)
smp = sample_generator.generate_sample(
rng,
ast_generator.AstGeneratorOptions(disallow_divide=DISALLOW_DIVIDE),
CALLS_PER_SAMPLE,
sample.SampleOptions(convert_to_ir=True,
optimize_ir=True,
codegen=FLAGS.codegen,
simulate=FLAGS.simulate))
run_dir = tempfile.mkdtemp('run_fuzz_')
return (run_dir, smp)
def test_evaluate_ir_miscompare_number_of_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner._evaluate_ir = lambda *args: (Value.make_ubits(8, 100),)
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(optimize_ir=False), []))
self.assertIn(
'Results for interpreted DSLX has 0 values, evaluated unopt IR (interpreter) has 1',
str(e.exception))
self.assertIn(
'Results for interpreted DSLX has 0 values, evaluated unopt IR (interpreter) has 1',
_read_file(sample_dir, 'exception.txt'))
def test_minimize_jit_interpreter_mismatch(self):
s = sample.Sample('fn main(x: u8) -> u8 { !x }', sample.SampleOptions(),
sample.parse_args_batch('bits[8]:0xff\nbits[8]:0x42'))
success = test_base.TempFileCleanup.SUCCESS # type: test_base.TempFileCleanup
run_dir = self.create_tempdir(cleanup=success).full_path
run_fuzz.run_sample(s, run_dir=run_dir)
minimized_ir_path = run_fuzz.minimize_ir(
s, run_dir, inject_jit_result='bits[32]:0x0')
self.assertIsNotNone(minimized_ir_path)
with open(minimized_ir_path, 'r') as f:
contents = f.read()
self.assertIn('package ', contents)
self.assertIn('fn ', contents)
# It should be reduced to simply a literal.
self.assertIn('ret literal', contents)
# And verify the minimized IR parses.
subprocess.check_call([PARSE_IR, minimized_ir_path])
def test_codegen_combinational_wrong_results(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
runner._simulate = lambda *args: (Value.make_ubits(8, 1),)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True,
simulator='iverilog'),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
def test_options_from_json(self):
json_text = ('{"input_is_dslx": true, "convert_to_ir": true, "optimize_ir":'
' true, "use_jit": true, "codegen": true, "codegen_args": '
'["--generator=pipeline", "--pipeline_stages=2"], "simulate": '
'false, "simulator": null, "use_system_verilog": true}')
expected_object = sample.SampleOptions(
input_is_dslx=True,
convert_to_ir=True,
optimize_ir=True,
use_jit=True,
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=False,
simulator=None,
use_system_verilog=True)
self.assertEqual(sample.SampleOptions.from_json(json_text), expected_object)
self.assertEqual(
sample.SampleOptions.from_json(json_text).to_json(), json_text)
def test_ir_input(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
ir_text = """package foo
fn foo(x: bits[8], y: bits[8]) -> bits[8] {
ret add.1: bits[8] = add(x, y)
}
"""
runner.run(
sample.Sample(ir_text, sample.SampleOptions(input_is_dslx=False),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('package foo', _read_file(sample_dir, 'sample.ir'))
self.assertIn('package foo', _read_file(sample_dir, 'sample.opt.ir'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.opt.ir.results'),
['bits[8]:0x8e'])
def test_codegen_pipeline(self):
sample_dir = self._make_sample_dir()
print('sample_dir = ' + sample_dir)
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
# A pipelined block should have a blocking assignment.
self.assertIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'), ['bits[8]:0x8e'])
def test_codegen_combinational(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
runner.run(
sample.Sample(
dslx_text,
sample.SampleOptions(
codegen=True,
codegen_args=['--generator=combinational'],
simulate=True),
[[Value.make_ubits(8, 42),
Value.make_ubits(8, 100)]]))
self.assertIn('endmodule', _read_file(sample_dir, 'sample.v'))
# A combinational block should not have a blocking assignment.
self.assertNotIn('<=', _read_file(sample_dir, 'sample.v'))
self.assertSequenceEqual(
_split_nonempty_lines(sample_dir, 'sample.v.results'), ['bits[8]:0x8e'])
def test_interpret_opt_ir_miscompare(self):
sample_dir = self._make_sample_dir()
runner = sample_runner.SampleRunner(sample_dir)
dslx_text = 'fn main(x: u8, y: u8) -> u8 { x + y }'
results = [
(Value.make_ubits(8, 100),), # correct result
(Value.make_ubits(8, 100),), # correct result
(Value.make_ubits(8, 0),), # incorrect result
]
def result_gen(*_):
return results.pop(0)
runner._evaluate_ir = result_gen
with self.assertRaises(sample_runner.SampleError) as e:
runner.run(
sample.Sample(dslx_text, sample.SampleOptions(),
[[Value.make_ubits(8, 40),
Value.make_ubits(8, 60)]]))
self.assertIn('Result miscompare for sample 0', str(e.exception))
self.assertIn('evaluated opt IR (JIT)', str(e.exception))
def test_to_crasher(self):
s = sample.Sample(
'fn main(x: u8, y: u8) -> u8 {\nx + y\n}',
sample.SampleOptions(
input_is_dslx=True,
codegen=True,
codegen_args=('--generator=pipeline', '--pipeline_stages=2'),
simulate=True,
simulator='goat simulator',
use_system_verilog=True),
sample.parse_args_batch(
'bits[8]:42; bits[8]:11\nbits[8]:44; bits[8]:99'))
self.assertEqual(
s.to_crasher(),
textwrap.dedent("""\
// options: {"input_is_dslx": true, "convert_to_ir": true, "optimize_ir": true, "use_jit": true, "codegen": true, "codegen_args": ["--generator=pipeline", "--pipeline_stages=2"], "simulate": true, "simulator": "goat simulator", "use_system_verilog": true}
// args: bits[8]:0x2a; bits[8]:0xb
// args: bits[8]:0x2c; bits[8]:0x63
fn main(x: u8, y: u8) -> u8 {
x + y
}
"""))
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
if FLAGS.simulate and not FLAGS.codegen:
raise app.UsageError('Must specify --codegen when --simulate is given.')
# Test that we can write to the crash and summary path.
for path in (FLAGS.crash_path, FLAGS.summary_path):
if path:
gfile.make_dirs(path)
with gfile.open(os.path.join(path, 'test'), 'w') as f:
print('test', file=f)
start = datetime.datetime.now()
physical_core_count = psutil.cpu_count(logical=False)
worker_count = FLAGS.worker_count or physical_core_count
worker_count = max(worker_count, 1) # Need at least one worker.
queues = (multiprocess.get_user_data() or
[mp.Queue() for _ in range(worker_count)])
queues = queues[:worker_count]
print('-- Creating pool of {} workers; physical core count {}'.format(
worker_count, physical_core_count))
workers = []
for i in range(worker_count):
queue = None if multiprocess.has_user_data_support() else queues[i]
target = run_fuzz_multiprocess.do_worker_task
args = (i, queue, FLAGS.crash_path, FLAGS.summary_path,
FLAGS.save_temps_path, FLAGS.minimize_ir)
worker = multiprocess.Process(target=target, args=args)
worker.start()
workers.append(worker)
duration_str = FLAGS.duration
duration = None if duration_str is None else cli_helpers.parse_duration(
duration_str)
seed = FLAGS.seed
if not seed:
seed = random.randrange(0, 1 << 31)
print('-- Using randomly generated seed:', seed)
sys.stdout.flush()
generator_options = ast_generator.AstGeneratorOptions(
disallow_divide=FLAGS.disallow_divide,
emit_loops=FLAGS.emit_loops,
short_samples=FLAGS.short_samples,
max_width_bits_types=FLAGS.max_width_bits_types,
max_width_aggregate_types=FLAGS.max_width_aggregate_types)
default_sample_options = sample.SampleOptions(
convert_to_ir=True,
optimize_ir=True,
use_jit=FLAGS.use_llvm_jit,
codegen=FLAGS.codegen,
simulate=FLAGS.simulate,
simulator=FLAGS.simulator,
use_system_verilog=FLAGS.use_system_verilog)
sample_count = run_fuzz_multiprocess.do_generator_task(
queues,
seed,
generator_options,
FLAGS.sample_count,
FLAGS.calls_per_sample,
default_sample_options=default_sample_options,
duration=duration,
print_samples=FLAGS.print_samples)
for i, worker in enumerate(workers):
print('-- Joining on worker {}'.format(i))
worker.join()
delta = datetime.datetime.now() - start
elapsed = delta.total_seconds()
print(
'-- Elapsed end-to-end: {} = {:.2f} seconds; {:,} samples; {:.2f} samples/s'
.format(delta, elapsed, sample_count, sample_count / elapsed))