def test_ssh():
"""Tests that ssh works as expected."""
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
gcloud.ssh(INSTANCE_NAME)
assert mocked_popen.commands == [[
'gcloud', 'beta', 'compute', 'ssh', INSTANCE_NAME
]]
def test_output_report_filestore(experiment_fuzzers, expected_merged_fuzzers,
expected_report_url, fs, experiment):
"""Test that output_report writes the report and rsyncs it to the report
filestore."""
experiment_config = _setup_experiment_files(fs)
experiment_config['fuzzers'] = experiment_fuzzers
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
with mock.patch('analysis.generate_report.generate_report'
) as mocked_generate_report:
reporter.output_report(experiment_config)
reports_dir = os.path.join(os.environ['WORK'], 'reports')
assert mocked_popen.commands == [[
'gsutil', '-h', 'Cache-Control:public,max-age=0,no-transform',
'rsync', '-r', reports_dir, expected_report_url
]]
experiment_name = os.environ['EXPERIMENT']
mocked_generate_report.assert_called_with(
[experiment_name],
reports_dir,
report_name=experiment_name,
fuzzers=expected_merged_fuzzers,
in_progress=False,
merge_with_clobber_nonprivate=False,
coverage_report=False)
def test_output_report_filestore(fs, experiment):
"""Test that output_report writes the report and rsyncs it to the report
filestore."""
# Get the config.
config_filepath = os.path.join(os.path.dirname(__file__), 'test_data',
'experiment-config.yaml')
fs.add_real_file(config_filepath)
with open(config_filepath) as file_handle:
experiment_config = yaml.load(file_handle, yaml.SafeLoader)
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
with mock.patch('analysis.generate_report.generate_report'
) as mocked_generate_report:
reporter.output_report(experiment_config)
reports_dir = os.path.join(os.environ['WORK'], 'reports')
assert mocked_popen.commands == [[
'gsutil', '-h', 'Cache-Control:public,max-age=0,no-transform',
'rsync', '-d', '-r', reports_dir,
'gs://web-reports/test-experiment'
]]
mocked_generate_report.assert_called_with(
[os.environ['EXPERIMENT']],
reports_dir,
in_progress=False,
merge_with_clobber_nonprivate=False)
def test_is_cycle_unchanged_doesnt_exist(experiment):
"""Test that is_cycle_unchanged can properly determine if a cycle is
unchanged or not when it needs to copy the file for the first time."""
snapshot_measurer = measurer.SnapshotMeasurer(FUZZER, BENCHMARK, TRIAL_NUM,
SNAPSHOT_LOGGER)
this_cycle = 1
with test_utils.mock_popen_ctx_mgr(returncode=1):
assert not snapshot_measurer.is_cycle_unchanged(this_cycle)
def test_create_instance_not_preemptible():
"""Tests create_instance doesn't specify preemptible when it isn't supposed
to."""
with test_utils.mock_popen_ctx_mgr(returncode=1) as mocked_popen:
gcloud.create_instance(INSTANCE_NAME, gcloud.InstanceType.RUNNER,
CONFIG)
assert mocked_popen.commands == [
_get_expected_create_runner_command(False)
]
def test_create_instance():
"""Tests create_instance creates an instance."""
with test_utils.mock_popen_ctx_mgr(returncode=1) as mocked_popen:
gcloud.create_instance(INSTANCE_NAME, gcloud.InstanceType.DISPATCHER,
CONFIG)
assert mocked_popen.commands == [[
'gcloud', 'compute', 'instances', 'create', 'instance-a',
'--image-family=cos-stable', '--image-project=cos-cloud',
'--zone=zone-a', '--scopes=cloud-platform',
'--machine-type=n1-standard-96', '--boot-disk-size=4TB'
]]
def test_create_instance_preemptible():
"""Tests create_instance doesn't specify preemptible when it isn't supposed
to."""
config = CONFIG.copy()
config['preemptible_runners'] = True
with test_utils.mock_popen_ctx_mgr(returncode=1) as mocked_popen:
gcloud.create_instance(INSTANCE_NAME, gcloud.InstanceType.RUNNER,
config)
assert mocked_popen.commands == [
_get_expected_create_runner_command(True)
]
def test_output_report_filestore(fs, experiment):
"""Test that output_report writes the report and rsyncs it to the report
filestore."""
config_filepath = os.path.join(os.path.dirname(__file__), '..', 'service',
'core-fuzzers.yaml')
fs.add_real_file(config_filepath)
# Get the config.
config_filepath = os.path.join(os.path.dirname(__file__), 'test_data',
'experiment-config.yaml')
fs.add_real_file(config_filepath)
with open(config_filepath) as file_handle:
experiment_config = yaml.load(file_handle, yaml.SafeLoader)
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
with mock.patch('analysis.generate_report.generate_report'
) as mocked_generate_report:
reporter.output_report(experiment_config)
reports_dir = os.path.join(os.environ['WORK'], 'reports')
assert mocked_popen.commands == [[
'gsutil', '-h', 'Cache-Control:public,max-age=0,no-transform',
'rsync', '-r', reports_dir, 'gs://web-reports/test-experiment'
]]
experiment_name = os.environ['EXPERIMENT']
mocked_generate_report.assert_called_with(
[experiment_name],
reports_dir,
report_name=experiment_name,
fuzzers=[
'afl',
'afl_qemu',
'aflfast',
'aflplusplus',
'aflplusplus_optimal',
'aflplusplus_qemu',
'aflsmart',
'eclipser',
'entropic',
'fairfuzz',
'fuzzer-a',
'fuzzer-b',
'honggfuzz',
'honggfuzz_qemu',
'lafintel',
'libaflfuzzer',
'libfuzzer',
'manul',
'mopt',
'weizz_qemu',
],
in_progress=False,
merge_with_clobber_nonprivate=False,
coverage_report=False)
def test_save_corpus_archive(_, trial_runner, fs):
"""Test that save_corpus_archive calls gsutil rsync on the corpus-archives
directory."""
archive_name = 'x.tar.gz'
fs.create_file(archive_name, contents='')
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
trial_runner.save_corpus_archive(archive_name)
assert mocked_popen.commands == [[
'gsutil', 'cp', archive_name,
posixpath.join(
'gs://bucket/experiment-name/experiment-folders/'
'benchmark-1-fuzzer_a/trial-1/corpus', archive_name)
]]
assert not os.path.exists(archive_name)
def test_output_report_bucket(fs, experiment):
"""Test that output_report writes the report and rsyncs it to the web
bucket."""
web_bucket = 'gs://web-bucket/experiment'
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
with mock.patch('experiment.generate_report.generate_report'
) as mocked_generate_report:
reporter.output_report(web_bucket)
reports_dir = os.path.join(os.environ['WORK'], 'reports')
assert mocked_popen.commands == [[
'gsutil', '-h', 'Cache-Control:public,max-age=0,no-transform',
'rsync', '-d', '-r', reports_dir, 'gs://web-bucket/experiment'
]]
mocked_generate_report.assert_called_with(
[os.environ['EXPERIMENT']], reports_dir, in_progress=False)
def test_do_sync_unchanged(mocked_is_corpus_dir_same, mocked_debug,
trial_runner, fuzzer_module):
"""Test that do_sync records if there was no corpus change since last
cycle."""
trial_runner.cycle = 1337
mocked_is_corpus_dir_same.return_value = True
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
trial_runner.do_sync()
assert mocked_popen.commands == [[
'gsutil', 'rsync', '-d', '-r', 'results-copy',
('gs://bucket/experiment-name/experiment-folders/'
'benchmark-1-fuzzer_a/trial-1/results')
]]
mocked_debug.assert_any_call('Cycle: %d unchanged.', trial_runner.cycle)
unchanged_cycles_path = os.path.join(trial_runner.results_dir,
'unchanged-cycles')
with open(unchanged_cycles_path) as file_handle:
assert str(trial_runner.cycle) == file_handle.read().strip()
assert not os.listdir(trial_runner.corpus_archives_dir)
def test_run_fuzzer_log_file(fs):
"""Test that run_fuzzer invokes the fuzzer defined run_fuzzer function as
expected."""
max_total_time = 1
log_filename = '/log.txt'
os.environ['MAX_TOTAL_TIME'] = str(max_total_time)
os.environ['SEED_CORPUS_DIR'] = '/out/seeds'
os.environ['OUTPUT_CORPUS_DIR'] = '/out/corpus'
os.environ['FUZZ_TARGET'] = '/out/fuzz-target'
os.environ['RUNNER_NICENESS'] = '-5'
fs.create_file('/out/fuzz-target')
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
runner.run_fuzzer(max_total_time, log_filename)
assert mocked_popen.commands == [[
'nice', '-n', '5', 'python3', '-u', '-c', "import fuzzer; "
"fuzzer.fuzz('/out/seeds', '/out/corpus', '/out/fuzz-target')"
]]
assert os.path.exists(log_filename)
def test_run_fuzzer_log_file(mocked_communicate, fs, environ):
"""Test that run_fuzzer invokes the fuzzer defined run_fuzzer function as
expected."""
mocked_communicate.return_value = ('', 0)
max_total_time = 1
log_filename = '/log.txt'
os.environ['MAX_TOTAL_TIME'] = str(max_total_time)
os.environ['SEED_CORPUS_DIR'] = '/out/seeds'
os.environ['OUTPUT_CORPUS_DIR'] = '/out/corpus'
os.environ['FUZZ_TARGET'] = '/out/fuzz-target'
os.environ['RUNNER_NICENESS'] = '-5'
os.environ['FUZZER'] = 'afl'
os.environ['BENCHMARK'] = 'freetype2-2017'
fs.add_real_directory(benchmark_config.BENCHMARKS_DIR)
fs.create_file('/out/fuzz-target')
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
runner.run_fuzzer(max_total_time, log_filename)
assert mocked_popen.commands == [[
'nice', '-n', '5', 'python3', '-u', '-c',
'from fuzzers.afl import fuzzer; '
"fuzzer.fuzz('/out/seeds', '/out/corpus', '/out/fuzz-target')"
]]
assert os.path.exists(log_filename)
def test_gsutil_command():
"""Tests gsutil_command works as expected."""
arguments = ['hello']
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
gsutil.gsutil_command(arguments)
assert mocked_popen.commands == [['gsutil', '-m'] + arguments]
def test_ssh_optional_arg(kwargs_for_ssh, expected_argument):
"""Tests that ssh works as expected when given an optional argument."""
with test_utils.mock_popen_ctx_mgr() as mocked_popen:
gcloud.ssh(INSTANCE_NAME, **kwargs_for_ssh)
assert expected_argument in mocked_popen.commands[0]