def test_generate_summary(mocked_get_coverage_binary, mocked_execute, experiment, fs): """Tests that generate_summary can run the correct command.""" mocked_execute.return_value = new_process.ProcessResult(0, '', False) coverage_binary_path = '/work/coverage-binaries/benchmark-a/fuzz-target' mocked_get_coverage_binary.return_value = coverage_binary_path snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) snapshot_measurer.cov_summary_file = "/reports/cov_summary.txt" snapshot_measurer.profdata_file = "/reports/data.profdata" fs.create_dir('/reports') fs.create_file(snapshot_measurer.profdata_file, contents='fake_contents') snapshot_measurer.generate_summary(CYCLE) expected = [ 'llvm-cov', 'export', '-format=text', '/work/coverage-binaries/benchmark-a/fuzz-target', '-instr-profile=/reports/data.profdata', '-summary-only' ] assert (len(mocked_execute.call_args_list)) == 1 args = mocked_execute.call_args_list[0] assert args[0][0] == expected assert args[1]['output_file'].name == "/reports/cov_summary.txt"
def test_get_current_coverage_no_file(fs, experiment): """Tests that get_current_coverage returns None with no json file.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) json_cov_summary_file = get_test_data_path('cov_summary_not_exist.json') snapshot_measurer.cov_summary_file = json_cov_summary_file covered_regions = snapshot_measurer.get_current_coverage() assert not covered_regions
def test_is_cycle_unchanged_no_file(mocked_cp, fs, experiment): """Test that is_cycle_unchanged returns False when there is no unchanged-cycles file.""" # Make sure we log if there is no unchanged-cycles file. snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) mocked_cp.return_value = new_process.ProcessResult(1, '', False) assert not snapshot_measurer.is_cycle_unchanged(0)
def test_is_cycle_unchanged_doesnt_exist(experiment): """Test that is_cycle_unchanged can properly determine if a cycle is unchanged or not when it needs to copy the file for the first time.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) this_cycle = 1 with test_utils.mock_popen_ctx_mgr(returncode=1): assert not snapshot_measurer.is_cycle_unchanged(this_cycle)
def test_get_current_coverage(fs, experiment): """Tests that get_current_coverage reads the correct data from json file.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) json_cov_summary_file = get_test_data_path('cov_summary.json') fs.add_real_file(json_cov_summary_file, read_only=False) snapshot_measurer.cov_summary_file = json_cov_summary_file covered_regions = snapshot_measurer.get_current_coverage() assert covered_regions == 8
def test_run_cov_new_units(_, mocked_execute, fs, environ): """Tests that run_cov_new_units does a coverage run as we expect.""" os.environ = { 'WORK': '/work', 'EXPERIMENT_FILESTORE': 'gs://bucket', 'EXPERIMENT': 'experiment', } mocked_execute.return_value = new_process.ProcessResult(0, '', False) snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) snapshot_measurer.initialize_measurement_dirs() shared_units = ['shared1', 'shared2'] fs.create_file(snapshot_measurer.measured_files_path, contents='\n'.join(shared_units)) for unit in shared_units: fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit)) new_units = ['new1', 'new2'] for unit in new_units: fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit)) fuzz_target_path = '/work/coverage-binaries/benchmark-a/fuzz-target' fs.create_file(fuzz_target_path) profraw_file_path = os.path.join(snapshot_measurer.coverage_dir, 'data.profraw') fs.create_file(profraw_file_path) snapshot_measurer.run_cov_new_units() assert len(mocked_execute.call_args_list) == 1 # Called once args = mocked_execute.call_args_list[0] command_arg = args[0][0] assert command_arg[0] == fuzz_target_path expected = { 'cwd': '/work/coverage-binaries/benchmark-a', 'env': { 'ASAN_OPTIONS': ('handle_abort=2:handle_segv=2:handle_sigbus=2:handle_sigfpe=2:' 'handle_sigill=2:symbolize=1:symbolize_inline_frames=0'), 'UBSAN_OPTIONS': ('handle_abort=2:handle_segv=2:handle_sigbus=2:handle_sigfpe=2:' 'handle_sigill=2:symbolize=1:symbolize_inline_frames=0'), 'LLVM_PROFILE_FILE': ('/work/measurement-folders/' 'benchmark-a-fuzzer-a/trial-12/coverage/data-%m.profraw'), 'WORK': '/work', 'EXPERIMENT_FILESTORE': 'gs://bucket', 'EXPERIMENT': 'experiment', }, 'expect_zero': False, } args = args[1] for arg, value in expected.items(): assert args[arg] == value
def test_is_cycle_unchanged_skip_cp(mocked_cp, fs, experiment): """Check that is_cycle_unchanged doesn't call filestore_utils.cp unnecessarily.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) this_cycle = 100 initial_unchanged_cycles_file_contents = ( '\n'.join([str(num) for num in range(10)] + [str(this_cycle + 1)])) fs.create_file(snapshot_measurer.unchanged_cycles_path, contents=initial_unchanged_cycles_file_contents) assert not snapshot_measurer.is_cycle_unchanged(this_cycle) mocked_cp.assert_not_called()
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals, too-many-arguments self, mocked_is_cycle_unchanged, mocked_df_container_list, db, experiment, tmp_path): """Integration test for measure_snapshot_coverage.""" # WORK is set by experiment to a directory that only makes sense in a # fakefs. A directory containing necessary llvm tools is also added to # PATH. llvm_tools_path = get_test_data_path('llvm_tools') os.environ["PATH"] += os.pathsep + llvm_tools_path os.environ['WORK'] = str(tmp_path) mocked_is_cycle_unchanged.return_value = False # Set up the coverage binary. benchmark = 'freetype2-2017' coverage_binary_src = get_test_data_path( 'test_measure_snapshot_coverage', benchmark + '-coverage') benchmark_cov_binary_dir = os.path.join( build_utils.get_coverage_binaries_dir(), benchmark) os.makedirs(benchmark_cov_binary_dir) coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir, 'ftfuzzer') shutil.copy(coverage_binary_src, coverage_binary_dst_dir) # Set up entities in database so that the snapshot can be created. experiment = models.Experiment(name=os.environ['EXPERIMENT']) db_utils.add_all([experiment]) trial = models.Trial(fuzzer=FUZZER, benchmark=benchmark, experiment=os.environ['EXPERIMENT']) db_utils.add_all([trial]) snapshot_measurer = measure_manager.SnapshotMeasurer( trial.fuzzer, trial.benchmark, trial.id, SNAPSHOT_LOGGER) # Set up the snapshot archive. cycle = 1 archive = get_test_data_path('test_measure_snapshot_coverage', 'corpus-archive-%04d.tar.gz' % cycle) corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus') os.makedirs(corpus_dir) shutil.copy(archive, corpus_dir) with mock.patch('common.filestore_utils.cp') as mocked_cp: mocked_cp.return_value = new_process.ProcessResult(0, '', False) # TODO(metzman): Create a system for using actual buckets in # integration tests. snapshot = measure_manager.measure_snapshot_coverage( snapshot_measurer.fuzzer, snapshot_measurer.benchmark, snapshot_measurer.trial_num, cycle, mocked_df_container_list) assert snapshot assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds() assert snapshot.edges_covered == 13178
def test_is_cycle_unchanged_first_copy(mocked_read, mocked_cp, experiment): """Test that is_cycle_unchanged can properly determine if a cycle is unchanged or not when it needs to copy the file for the first time.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) this_cycle = 100 unchanged_cycles_file_contents = ( '\n'.join([str(num) for num in range(10)] + [str(this_cycle)])) mocked_read.return_value = unchanged_cycles_file_contents mocked_cp.return_value = new_process.ProcessResult(0, '', False) assert snapshot_measurer.is_cycle_unchanged(this_cycle) assert not snapshot_measurer.is_cycle_unchanged(this_cycle + 1)
def test_generate_profdata_create(mocked_execute, experiment, fs): """Tests that generate_profdata can run the correct command.""" mocked_execute.return_value = new_process.ProcessResult(0, '', False) snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) snapshot_measurer.profdata_file = '/work/reports/data.profdata' snapshot_measurer.profraw_file_pattern = '/work/reports/data-%m.profraw' profraw_file = '/work/reports/data-123.profraw' fs.create_file(profraw_file, contents='fake_contents') snapshot_measurer.generate_profdata(CYCLE) expected = [ 'llvm-profdata', 'merge', '-sparse', '/work/reports/data-123.profraw', '-o', '/work/reports/data.profdata' ] assert (len(mocked_execute.call_args_list)) == 1 args = mocked_execute.call_args_list[0] assert args[0][0] == expected
def test_is_cycle_unchanged_update(fs, experiment): """Test that is_cycle_unchanged can properly determine that a cycle has changed when it has the file but needs to update it.""" snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) this_cycle = 100 initial_unchanged_cycles_file_contents = ( '\n'.join([str(num) for num in range(10)] + [str(this_cycle)])) fs.create_file(snapshot_measurer.unchanged_cycles_path, contents=initial_unchanged_cycles_file_contents) next_cycle = this_cycle + 1 unchanged_cycles_file_contents = (initial_unchanged_cycles_file_contents + '\n' + str(next_cycle)) assert snapshot_measurer.is_cycle_unchanged(this_cycle) with mock.patch('common.filestore_utils.cp') as mocked_cp: with mock.patch('common.filesystem.read') as mocked_read: mocked_cp.return_value = new_process.ProcessResult(0, '', False) mocked_read.return_value = unchanged_cycles_file_contents assert snapshot_measurer.is_cycle_unchanged(next_cycle)
def test_run_cov_new_units(_, mocked_execute, fs, environ): """Tests that run_cov_new_units does a coverage run as we expect.""" os.environ = { 'WORK': '/work', 'EXPERIMENT_FILESTORE': 'gs://bucket', 'EXPERIMENT': 'experiment', } mocked_execute.return_value = new_process.ProcessResult(0, '', False) snapshot_measurer = measure_manager.SnapshotMeasurer( FUZZER, BENCHMARK, TRIAL_NUM, SNAPSHOT_LOGGER) snapshot_measurer.initialize_measurement_dirs() shared_units = ['shared1', 'shared2'] fs.create_file(snapshot_measurer.measured_files_path, contents='\n'.join(shared_units)) for unit in shared_units: fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit)) new_units = ['new1', 'new2'] for unit in new_units: fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit)) fuzz_target_path = '/work/coverage-binaries/benchmark-a/fuzz-target' fs.create_file(fuzz_target_path) profraw_file_path = os.path.join(snapshot_measurer.coverage_dir, 'data.profraw') fs.create_file(profraw_file_path) snapshot_measurer.run_cov_new_units() assert len(mocked_execute.call_args_list) == 1 # Called once args = mocked_execute.call_args_list[0] command_arg = args[0][0] assert command_arg[0] == fuzz_target_path expected = { 'cwd': '/work/coverage-binaries/benchmark-a', 'env': { 'ASAN_OPTIONS': ('alloc_dealloc_mismatch=0:allocator_may_return_null=1:' 'allocator_release_to_os_interval_ms=500:' 'allow_user_segv_handler=0:check_malloc_usable_size=0:' 'detect_leaks=1:detect_odr_violation=0:' 'detect_stack_use_after_return=1:fast_unwind_on_fatal=0:' 'handle_abort=2:handle_segv=2:handle_sigbus=2:handle_sigfpe=2:' 'handle_sigill=2:max_uar_stack_size_log=16:' 'quarantine_size_mb=64:strict_memcmp=1:symbolize=1:' 'symbolize_inline_frames=0'), 'UBSAN_OPTIONS': ('allocator_release_to_os_interval_ms=500:handle_abort=2:' 'handle_segv=2:handle_sigbus=2:handle_sigfpe=2:' 'handle_sigill=2:print_stacktrace=1:' 'symbolize=1:symbolize_inline_frames=0'), 'LLVM_PROFILE_FILE': ('/work/measurement-folders/' 'benchmark-a-fuzzer-a/trial-12/coverage/data-%m.profraw'), 'WORK': '/work', 'EXPERIMENT_FILESTORE': 'gs://bucket', 'EXPERIMENT': 'experiment', }, 'expect_zero': False, } args = args[1] for arg, value in expected.items(): assert args[arg] == value