def test_is_cycle_unchanged_no_file(mocked_warn, fs, experiment):
"""Test that is_cycle_unchanged returns False when there is no
unchanged-cycles file."""
# Make sure we log if there is no unchanged-cycles file.
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
assert not snapshot_measurer.is_cycle_unchanged(0)
def test_run_cov_new_units(mocked_execute, fs, environ):
"""Tests that run_cov_new_units does a coverage run as we expect."""
os.environ = {'WORK': '/work'}
mocked_execute.return_value = new_process.ProcessResult(0, '', False)
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
snapshot_measurer.initialize_measurement_dirs()
shared_units = ['shared1', 'shared2']
for unit in shared_units:
fs.create_file(os.path.join(snapshot_measurer.prev_corpus_dir, unit))
fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit))
new_units = ['new1', 'new2']
for unit in new_units:
fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit))
fuzz_target_path = '/work/coverage-binaries/benchmark-a/fuzz-target'
fs.create_file(fuzz_target_path)
snapshot_measurer.run_cov_new_units()
assert len(mocked_execute.call_args_list) == 1 # Called once
args = mocked_execute.call_args_list[0]
command_arg = args[0][0]
assert command_arg[0] == fuzz_target_path
expected = {
'cwd': '/work/coverage-binaries/benchmark-a',
'env': {
'UBSAN_OPTIONS': ('coverage_dir='
'/work/measurement-folders/benchmark-a/fuzzer-a'
'/trial-12/sancovs'),
'WORK': '/work',
},
'expect_zero': False,
}
args = args[1]
for arg, value in expected.items():
assert args[arg] == value
def test_generate_summary(mocked_get_coverage_binary, mocked_execute,
experiment, fs):
"""Tests that generate_summary can run the correct command."""
mocked_execute.return_value = new_process.ProcessResult(0, '', False)
coverage_binary_path = '/work/coverage-binaries/benchmark-a/fuzz-target'
mocked_get_coverage_binary.return_value = coverage_binary_path
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
snapshot_measurer.cov_summary_file = "/reports/cov_summary.txt"
snapshot_measurer.profdata_file = "/reports/data.profdata"
fs.create_dir('/reports')
fs.create_file(snapshot_measurer.profdata_file, contents='fake_contents')
snapshot_measurer.generate_summary(CYCLE)
expected = [
'llvm-cov', 'export', '-format=text',
'/work/coverage-binaries/benchmark-a/fuzz-target',
'-instr-profile=/reports/data.profdata', '-summary-only'
]
assert (len(mocked_execute.call_args_list)) == 1
args = mocked_execute.call_args_list[0]
assert args[0][0] == expected
assert args[1]['output_file'].name == "/reports/cov_summary.txt"
def test_get_current_coverage_no_file(fs, experiment):
"""Tests that get_current_coverage returns None with no json file."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
json_cov_summary_file = get_test_data_path('cov_summary_not_exist.json')
snapshot_measurer.cov_summary_file = json_cov_summary_file
covered_regions = snapshot_measurer.get_current_coverage()
assert not covered_regions
def test_is_cycle_unchanged_doesnt_exist(experiment):
"""Test that is_cycle_unchanged can properly determine if a cycle is
unchanged or not when it needs to copy the file for the first time."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 1
with test_utils.mock_popen_ctx_mgr(returncode=1):
assert not snapshot_measurer.is_cycle_unchanged(this_cycle)
def test_is_cycle_unchanged_no_file(mocked_cp, fs, experiment):
"""Test that is_cycle_unchanged returns False when there is no
unchanged-cycles file."""
# Make sure we log if there is no unchanged-cycles file.
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
mocked_cp.side_effect = subprocess.CalledProcessError(1, ['fakecommand'])
assert not snapshot_measurer.is_cycle_unchanged(0)
def test_is_cycle_unchanged_no_file(mocked_cp, fs, experiment):
"""Test that is_cycle_unchanged returns False when there is no
unchanged-cycles file."""
# Make sure we log if there is no unchanged-cycles file.
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
mocked_cp.return_value = new_process.ProcessResult(1, '', False)
assert not snapshot_measurer.is_cycle_unchanged(0)
def test_get_current_coverage(fs, experiment):
"""Tests that get_current_coverage reads the correct data from json file."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
json_cov_summary_file = get_test_data_path('cov_summary.json')
fs.add_real_file(json_cov_summary_file, read_only=False)
snapshot_measurer.cov_summary_file = json_cov_summary_file
covered_regions = snapshot_measurer.get_current_coverage()
assert covered_regions == 8
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals
self, mocked_is_cycle_unchanged, db, experiment, tmp_path):
"""Integration test for measure_snapshot_coverage."""
# WORK is set by experiment to a directory that only makes sense in a
# fakefs. A directory containing necessary llvm tools is also added to
# PATH.
llvm_tools_path = get_test_data_path('llvm_tools')
os.environ["PATH"] += os.pathsep + llvm_tools_path
os.environ['WORK'] = str(tmp_path)
mocked_is_cycle_unchanged.return_value = False
# Set up the coverage binary.
benchmark = 'freetype2-2017'
coverage_binary_src = get_test_data_path(
'test_measure_snapshot_coverage', benchmark + '-coverage')
benchmark_cov_binary_dir = os.path.join(
build_utils.get_coverage_binaries_dir(), benchmark)
os.makedirs(benchmark_cov_binary_dir)
coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir,
'ftfuzzer')
shutil.copy(coverage_binary_src, coverage_binary_dst_dir)
# Set up entities in database so that the snapshot can be created.
experiment = models.Experiment(name=os.environ['EXPERIMENT'])
db_utils.add_all([experiment])
trial = models.Trial(fuzzer=FUZZER,
benchmark=benchmark,
experiment=os.environ['EXPERIMENT'])
db_utils.add_all([trial])
snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer,
trial.benchmark,
trial.id,
SNAPSHOT_LOGGER)
# Set up the snapshot archive.
cycle = 1
archive = get_test_data_path('test_measure_snapshot_coverage',
'corpus-archive-%04d.tar.gz' % cycle)
corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus')
os.makedirs(corpus_dir)
shutil.copy(archive, corpus_dir)
with mock.patch('common.filestore_utils.cp') as mocked_cp:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
# TODO(metzman): Create a system for using actual buckets in
# integration tests.
snapshot = measurer.measure_snapshot_coverage(
snapshot_measurer.fuzzer, snapshot_measurer.benchmark,
snapshot_measurer.trial_num, cycle)
assert snapshot
assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds()
assert snapshot.edges_covered == 13178
def test_is_cycle_unchanged_skip_cp(mocked_cp, fs, experiment):
"""Check that is_cycle_unchanged doesn't call gsutil.cp unnecessarily."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 100
initial_unchanged_cycles_file_contents = (
'\n'.join([str(num) for num in range(10)] + [str(this_cycle + 1)]))
fs.create_file(snapshot_measurer.unchanged_cycles_path,
contents=initial_unchanged_cycles_file_contents)
assert not snapshot_measurer.is_cycle_unchanged(this_cycle)
mocked_cp.assert_not_called()
def test_is_cycle_unchanged_first_copy(mocked_read, mocked_cp, experiment):
"""Test that is_cycle_unchanged can properly determine if a cycle is
unchanged or not when it needs to copy the file for the first time."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 100
unchanged_cycles_file_contents = (
'\n'.join([str(num) for num in range(10)] + [str(this_cycle)]))
mocked_read.return_value = unchanged_cycles_file_contents
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
assert snapshot_measurer.is_cycle_unchanged(this_cycle)
assert not snapshot_measurer.is_cycle_unchanged(this_cycle + 1)
def test_run_cov_new_units(_, mocked_execute, fs, environ):
"""Tests that run_cov_new_units does a coverage run as we expect."""
os.environ = {
'WORK': '/work',
'EXPERIMENT_FILESTORE': 'gs://bucket',
'EXPERIMENT': 'experiment',
}
mocked_execute.return_value = new_process.ProcessResult(0, '', False)
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
snapshot_measurer.initialize_measurement_dirs()
shared_units = ['shared1', 'shared2']
fs.create_file(snapshot_measurer.measured_files_path,
contents='\n'.join(shared_units))
for unit in shared_units:
fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit))
new_units = ['new1', 'new2']
for unit in new_units:
fs.create_file(os.path.join(snapshot_measurer.corpus_dir, unit))
fuzz_target_path = '/work/coverage-binaries/benchmark-a/fuzz-target'
fs.create_file(fuzz_target_path)
profraw_file_path = os.path.join(snapshot_measurer.coverage_dir,
'data.profraw')
fs.create_file(profraw_file_path)
snapshot_measurer.run_cov_new_units()
assert len(mocked_execute.call_args_list) == 1 # Called once
args = mocked_execute.call_args_list[0]
command_arg = args[0][0]
assert command_arg[0] == fuzz_target_path
expected = {
'cwd': '/work/coverage-binaries/benchmark-a',
'env': {
'LLVM_PROFILE_FILE':
('/work/measurement-folders/'
'benchmark-a-fuzzer-a/trial-12/coverage/data-%m.profraw'),
'WORK':
'/work',
'EXPERIMENT_FILESTORE':
'gs://bucket',
'EXPERIMENT':
'experiment',
},
'expect_zero': False,
}
args = args[1]
for arg, value in expected.items():
assert args[arg] == value
def test_is_cycle_unchanged(fs, experiment):
"""Test that is_cycle_unchanged can properly determine if a cycle is
unchanged or not."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 100
unchanged_cycles_path = os.path.join(snapshot_measurer.trial_dir, 'results',
'unchanged-cycles')
unchange_cycles_file_contents = ('\n'.join([str(num) for num in range(10)] +
[str(this_cycle)]))
fs.create_file(unchanged_cycles_path,
contents=unchange_cycles_file_contents)
assert snapshot_measurer.is_cycle_unchanged(this_cycle)
assert not snapshot_measurer.is_cycle_unchanged(this_cycle + 1)
os.remove(unchanged_cycles_path)
def test_generate_profdata_create(mocked_execute, experiment, fs):
"""Tests that generate_profdata can run the correct command."""
mocked_execute.return_value = new_process.ProcessResult(0, '', False)
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
snapshot_measurer.profdata_file = '/work/reports/data.profdata'
snapshot_measurer.profraw_file = '/work/reports/data.profraw'
fs.create_file(snapshot_measurer.profraw_file, contents='fake_contents')
snapshot_measurer.generate_profdata(CYCLE)
expected = [
'llvm-profdata', 'merge', '-sparse', '/work/reports/data.profraw',
'-o', '/work/reports/data.profdata'
]
assert (len(mocked_execute.call_args_list)) == 1
args = mocked_execute.call_args_list[0]
assert args[0][0] == expected
def test_is_cycle_unchanged_update(fs, experiment):
"""Test that is_cycle_unchanged can properly determine that a
cycle has changed when it has the file but needs to update it."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 100
initial_unchanged_cycles_file_contents = (
'\n'.join([str(num) for num in range(10)] + [str(this_cycle)]))
fs.create_file(snapshot_measurer.unchanged_cycles_path,
contents=initial_unchanged_cycles_file_contents)
next_cycle = this_cycle + 1
unchanged_cycles_file_contents = (initial_unchanged_cycles_file_contents +
'\n' + str(next_cycle))
assert snapshot_measurer.is_cycle_unchanged(this_cycle)
with mock.patch('common.filestore_utils.cp') as mocked_cp:
with mock.patch('common.filesystem.read') as mocked_read:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
mocked_read.return_value = unchanged_cycles_file_contents
assert snapshot_measurer.is_cycle_unchanged(next_cycle)
def test_merge_new_pcs(new_pcs, fs, experiment):
"""Tests that merge_new_pcs merges new PCs, and updates the covered-pcs
file."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
covered_pcs_filename = get_test_data_path('covered-pcs.txt')
fs.add_real_file(covered_pcs_filename, read_only=False)
snapshot_measurer.sancov_dir = os.path.dirname(covered_pcs_filename)
snapshot_measurer.covered_pcs_filename = covered_pcs_filename
with open(covered_pcs_filename) as file_handle:
initial_contents = file_handle.read()
fs.create_file(os.path.join(snapshot_measurer.sancov_dir, '1.sancov'),
contents='')
with mock.patch('third_party.sancov.GetPCs') as mocked_GetPCs:
mocked_GetPCs.return_value = new_pcs
snapshot_measurer.merge_new_pcs()
with open(covered_pcs_filename) as file_handle:
new_contents = file_handle.read()
assert new_contents == (''.join(pc + '\n' for pc in new_pcs) +
initial_contents)
def _create_measurer(fuzzer, benchmark, trial_num):
return measurer.SnapshotMeasurer(fuzzer, benchmark, trial_num,
SNAPSHOT_LOGGER)
