def setup_generator(): return dict(queues=(_DummyQueue(), ), seed=0, ast_generator_options=ast_generator.AstGeneratorOptions( disallow_divide=DISALLOW_DIVIDE), sample_count=16, duration=None, calls_per_sample=CALLS_PER_SAMPLE, print_samples=False)
def _test_repeatable(self, seed: int) -> None: def generate_one(g): _, m = g.generate_function_in_module('main', 'test') text = str(m) return text count_to_generate = 128 g = ast_generator.AstGenerator(random.Random(seed), ast_generator.AstGeneratorOptions()) fs = [generate_one(g) for _ in range(count_to_generate)] g.reset(seed=seed) gs = [generate_one(g) for _ in range(count_to_generate)] self.assertEqual(fs, gs) # Create a fresh generator as well. g = ast_generator.AstGenerator(random.Random(seed), ast_generator.AstGeneratorOptions()) hs = [generate_one(g) for _ in range(count_to_generate)] self.assertEqual(gs, hs)
def setup_worker(): """Creates arguments to repeatedly pass to benchmark_worker.""" rng = random.Random(0) smp = sample_generator.generate_sample( rng, ast_generator.AstGeneratorOptions(disallow_divide=DISALLOW_DIVIDE), CALLS_PER_SAMPLE, sample.SampleOptions(convert_to_ir=True, optimize_ir=True, codegen=FLAGS.codegen, simulate=FLAGS.simulate)) run_dir = tempfile.mkdtemp('run_fuzz_') return (run_dir, smp)
def test_generate_codegen_sample(self): rng = random.Random(0) sample = sample_generator.generate_sample( rng, ast_generator.AstGeneratorOptions(), calls_per_sample=0, default_options=SampleOptions( convert_to_ir=True, optimize_ir=True, codegen=True, simulate=True)) self.assertTrue(sample.options.input_is_dslx) self.assertTrue(sample.options.convert_to_ir) self.assertTrue(sample.options.optimize_ir) self.assertTrue(sample.options.codegen) self.assertTrue(sample.options.simulate) self.assertNotEmpty(sample.options.codegen_args) self.assertEmpty(sample.args_batch)
def test_generate_basic_sample(self): rng = random.Random(0) sample = sample_generator.generate_sample( rng, ast_generator.AstGeneratorOptions(), calls_per_sample=3, default_options=SampleOptions( convert_to_ir=True, optimize_ir=True, codegen=False, simulate=False)) self.assertTrue(sample.options.input_is_dslx) self.assertTrue(sample.options.convert_to_ir) self.assertTrue(sample.options.optimize_ir) self.assertFalse(sample.options.codegen) self.assertFalse(sample.options.simulate) self.assertLen(sample.args_batch, 3) self.assertIn('fn main', sample.input_text)
def test_generates_valid_functions(self): g = ast_generator.AstGenerator( random.Random(0), ast_generator.AstGeneratorOptions()) for i in range(32): print('Generating sample', i) _, m = g.generate_function_in_module('main', 'test') text = str(m) filename = '/fake/test_sample.x' with fakefs_test_util.scoped_fakefs(filename, text): try: module = parser_helpers.parse_text( text, name='test_sample', print_on_error=True, filename=filename) typecheck.check_module(module, f_import=None) except PositionalError as e: parser_helpers.pprint_positional_error(e) raise
def test_first_n_seeds(self, seed): if FLAGS.update_golden and seed >= self.SEED_TO_CHECK_LIMIT: # Skip running unnecessary tests if updating golden because the test is # slow and runs unsharded. return rng = random.Random(seed) samples = run_fuzz.run_fuzz( rng, ast_generator.AstGeneratorOptions( disallow_divide=True, binop_allowlist=[ast.BinopKind.SHLL]), **self.KWARGS) for i in range(self.KWARGS['sample_count']): if seed < self.SEED_TO_CHECK_LIMIT and i < self.SAMPLE_TO_CHECK_LIMIT: path = self.GOLDEN_REFERENCE_FMT.format(seed=seed, sample=i) if FLAGS.update_golden: with open(path, 'w') as f: f.write(samples[i].input_text) else: # rstrip to avoid miscompares from trailing newline at EOF. expected = runfiles.get_contents_as_text(path).rstrip() self.assertMultiLineEqual(expected, samples[i].input_text)
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') if FLAGS.simulate and not FLAGS.codegen: raise app.UsageError('Must specify --codegen when --simulate is given.') # Test that we can write to the crash and summary path. for path in (FLAGS.crash_path, FLAGS.summary_path): if path: gfile.make_dirs(path) with gfile.open(os.path.join(path, 'test'), 'w') as f: print('test', file=f) start = datetime.datetime.now() physical_core_count = psutil.cpu_count(logical=False) worker_count = FLAGS.worker_count or physical_core_count worker_count = max(worker_count, 1) # Need at least one worker. queues = (multiprocess.get_user_data() or [mp.Queue() for _ in range(worker_count)]) queues = queues[:worker_count] print('-- Creating pool of {} workers; physical core count {}'.format( worker_count, physical_core_count)) workers = [] for i in range(worker_count): queue = None if multiprocess.has_user_data_support() else queues[i] target = run_fuzz_multiprocess.do_worker_task args = (i, queue, FLAGS.crash_path, FLAGS.summary_path, FLAGS.save_temps_path, FLAGS.minimize_ir) worker = multiprocess.Process(target=target, args=args) worker.start() workers.append(worker) duration_str = FLAGS.duration duration = None if duration_str is None else cli_helpers.parse_duration( duration_str) seed = FLAGS.seed if not seed: seed = random.randrange(0, 1 << 31) print('-- Using randomly generated seed:', seed) sys.stdout.flush() generator_options = ast_generator.AstGeneratorOptions( disallow_divide=FLAGS.disallow_divide, emit_loops=FLAGS.emit_loops, short_samples=FLAGS.short_samples, max_width_bits_types=FLAGS.max_width_bits_types, max_width_aggregate_types=FLAGS.max_width_aggregate_types) default_sample_options = sample.SampleOptions( convert_to_ir=True, optimize_ir=True, use_jit=FLAGS.use_llvm_jit, codegen=FLAGS.codegen, simulate=FLAGS.simulate, simulator=FLAGS.simulator, use_system_verilog=FLAGS.use_system_verilog) sample_count = run_fuzz_multiprocess.do_generator_task( queues, seed, generator_options, FLAGS.sample_count, FLAGS.calls_per_sample, default_sample_options=default_sample_options, duration=duration, print_samples=FLAGS.print_samples) for i, worker in enumerate(workers): print('-- Joining on worker {}'.format(i)) worker.join() delta = datetime.datetime.now() - start elapsed = delta.total_seconds() print( '-- Elapsed end-to-end: {} = {:.2f} seconds; {:,} samples; {:.2f} samples/s' .format(delta, elapsed, sample_count, sample_count / elapsed))
def _get_ast_options(self) -> ast_generator.AstGeneratorOptions: return ast_generator.AstGeneratorOptions(disallow_divide=True)
def test_types(self): g = ast_generator.AstGenerator(random.Random(0), ast_generator.AstGeneratorOptions()) _, m = g.generate_function_in_module('main', 'test') u1 = g._make_type_annotation(signed=False, width=1) self.assertEqual(g._get_type_bit_count(u1), 1) self.assertTrue(g._is_bit_vector(u1)) self.assertTrue(g._is_unsigned_bit_vector(u1)) self.assertFalse(g._is_array(u1)) self.assertFalse(g._is_tuple(u1)) s1 = g._make_type_annotation(signed=True, width=1) self.assertTrue(g._is_bit_vector(s1)) self.assertFalse(g._is_unsigned_bit_vector(s1)) self.assertEqual(g._get_type_bit_count(s1), 1) self.assertFalse(g._is_array(s1)) self.assertFalse(g._is_tuple(s1)) u42 = g._make_type_annotation(signed=False, width=42) self.assertEqual(g._get_type_bit_count(u42), 42) self.assertTrue(g._is_bit_vector(u42)) self.assertTrue(g._is_unsigned_bit_vector(u42)) self.assertFalse(g._is_array(u42)) self.assertFalse(g._is_tuple(u42)) empty_tuple = g._make_tuple_type(()) self.assertEqual(g._get_type_bit_count(empty_tuple), 0) self.assertFalse(g._is_bit_vector(empty_tuple)) self.assertFalse(g._is_unsigned_bit_vector(empty_tuple)) self.assertFalse(g._is_array(empty_tuple)) self.assertTrue(g._is_tuple(empty_tuple)) tple = g._make_tuple_type((empty_tuple, s1, u1, u42, u42)) self.assertEqual(g._get_type_bit_count(tple), 86) self.assertFalse(g._is_bit_vector(tple)) self.assertFalse(g._is_unsigned_bit_vector(tple)) self.assertFalse(g._is_array(tple)) self.assertTrue(g._is_tuple(tple)) nested_tuple = g._make_tuple_type((tple, u42, tple)) self.assertEqual(g._get_type_bit_count(nested_tuple), 214) self.assertFalse(g._is_bit_vector(nested_tuple)) self.assertFalse(g._is_unsigned_bit_vector(nested_tuple)) self.assertFalse(g._is_array(nested_tuple)) self.assertTrue(g._is_tuple(nested_tuple)) array_of_u42 = g._make_array_type(u42, 100) self.assertEqual(g._get_type_bit_count(array_of_u42), 4200) self.assertFalse(g._is_bit_vector(array_of_u42)) self.assertFalse(g._is_unsigned_bit_vector(array_of_u42)) self.assertEqual(g._get_array_size(array_of_u42), 100) self.assertTrue(g._is_array(array_of_u42)) self.assertFalse(g._is_tuple(array_of_u42)) array_of_tuple = g._make_array_type(nested_tuple, 10) self.assertEqual(g._get_type_bit_count(array_of_tuple), 2140) self.assertFalse(g._is_bit_vector(array_of_tuple)) self.assertFalse(g._is_unsigned_bit_vector(array_of_tuple)) self.assertEqual(g._get_array_size(array_of_tuple), 10) self.assertTrue(g._is_array(array_of_tuple)) self.assertFalse(g._is_tuple(array_of_tuple)) u11_token = scanner.Token(g.fake_span, scanner.KeywordFromString('u11')) u11 = ast_helpers.make_builtin_type_annotation(m, g.fake_span, u11_token, dims=()) self.assertEqual(g._get_type_bit_count(u11), 11) self.assertTrue(g._is_bit_vector(u11)) self.assertTrue(g._is_unsigned_bit_vector(u11)) self.assertFalse(g._is_array(u11)) self.assertFalse(g._is_tuple(u11)) un_token = scanner.Token(g.fake_span, scanner.KeywordFromString('uN')) un1234 = ast_helpers.make_builtin_type_annotation(m, g.fake_span, un_token, dims=(g._make_number( 1234, None), )) self.assertEqual(g._get_type_bit_count(un1234), 1234) self.assertTrue(g._is_bit_vector(un1234)) self.assertTrue(g._is_unsigned_bit_vector(un1234)) self.assertFalse(g._is_array(un1234)) self.assertFalse(g._is_tuple(un1234)) sn_token = scanner.Token(g.fake_span, scanner.KeywordFromString('sN')) sn1234 = ast_helpers.make_builtin_type_annotation(m, g.fake_span, sn_token, dims=(g._make_number( 1234, None), )) self.assertEqual(g._get_type_bit_count(sn1234), 1234) self.assertTrue(g._is_bit_vector(sn1234)) self.assertFalse(g._is_unsigned_bit_vector(sn1234)) self.assertFalse(g._is_array(sn1234)) self.assertFalse(g._is_tuple(sn1234)) bits_token = scanner.Token(g.fake_span, scanner.KeywordFromString('bits')) bits1234 = ast_helpers.make_builtin_type_annotation( m, g.fake_span, bits_token, dims=(g._make_number(1234, None), )) self.assertEqual(g._get_type_bit_count(bits1234), 1234) self.assertTrue(g._is_bit_vector(bits1234)) self.assertTrue(g._is_unsigned_bit_vector(bits1234)) self.assertFalse(g._is_array(bits1234)) self.assertFalse(g._is_tuple(bits1234)) un1234_10 = ast_helpers.make_builtin_type_annotation( m, g.fake_span, un_token, dims=(g._make_number(1234, None), g._make_number(10, None))) self.assertEqual(g._get_type_bit_count(un1234_10), 12340) self.assertFalse(g._is_bit_vector(un1234_10)) self.assertFalse(g._is_unsigned_bit_vector(un1234_10)) self.assertTrue(g._is_array(un1234_10)) self.assertFalse(g._is_tuple(un1234_10))