def test_run_requested_experiment(mocked_get_requested_experiments,
mocked_stop_experiment,
mocked_start_experiment, db):
"""Tests that run_requested_experiment starts and stops the experiment
properly."""
mocked_get_requested_experiments.return_value = EXPERIMENT_REQUESTS
expected_experiment_name = '2020-06-05'
expected_fuzzers = ['honggfuzz', 'afl']
automatic_run_experiment.run_requested_experiment(dry_run=False)
expected_config_file = os.path.join(utils.ROOT_DIR, 'service',
'experiment-config.yaml')
def sort_key(dictionary):
return dictionary['fuzzer']
expected_fuzzer_configs = list(
sorted([{
'fuzzer': fuzzer
} for fuzzer in expected_fuzzers],
key=sort_key))
expected_benchmarks = [
'bloaty_fuzz_target',
'curl_curl_fuzzer_http',
'jsoncpp_jsoncpp_fuzzer',
'libpcap_fuzz_both',
'mbedtls_fuzz_dtlsclient',
'openssl_x509',
'php_php-fuzz-parser',
'sqlite3_ossfuzz',
'systemd_fuzz-link-parser',
'zlib_zlib_uncompress_fuzzer',
'freetype2-2017',
'harfbuzz-1.3.2',
'lcms-2017-03-21',
'libjpeg-turbo-07-2017',
'libpng-1.2.56',
'libxml2-v2.9.2',
'openthread-2019-12-23',
'proj4-2017-08-14',
're2-2014-12-09',
'vorbis-2017-12-11',
'woff2-2016-05-06',
]
expected_calls = [
mock.call(expected_experiment_name, expected_config_file,
expected_benchmarks, expected_fuzzer_configs)
]
start_experiment_call_args = mocked_start_experiment.call_args_list
assert len(start_experiment_call_args) == 1
# Sort the list of fuzzer configs so that we can assert that the calls were
# what we expected.
start_experiment_call_args[0][0][3].sort(key=sort_key)
assert start_experiment_call_args == expected_calls
mocked_stop_experiment.assert_called_with(expected_experiment_name,
expected_config_file)
def test_run_requested_experiment(mocked_get_requested_experiments,
mocked_start_experiment, db):
"""Tests that run_requested_experiment starts and stops the experiment
properly."""
mocked_get_requested_experiments.return_value = EXPERIMENT_REQUESTS
expected_experiment_name = '2020-06-05'
expected_fuzzers = ['honggfuzz', 'afl']
automatic_run_experiment.run_requested_experiment(dry_run=False)
expected_config_file = os.path.join(utils.ROOT_DIR, 'service',
'experiment-config.yaml')
expected_benchmarks = [
'bloaty_fuzz_target',
'curl_curl_fuzzer_http',
'jsoncpp_jsoncpp_fuzzer',
'libpcap_fuzz_both',
'libxslt_xpath',
'mbedtls_fuzz_dtlsclient',
'openssl_x509',
'php_php-fuzz-parser',
'sqlite3_ossfuzz',
'systemd_fuzz-link-parser',
'zlib_zlib_uncompress_fuzzer',
'freetype2-2017',
'harfbuzz-1.3.2',
'lcms-2017-03-21',
'libjpeg-turbo-07-2017',
'libpng-1.2.56',
'libxml2-v2.9.2',
'openthread-2019-12-23',
'proj4-2017-08-14',
're2-2014-12-09',
'vorbis-2017-12-11',
'woff2-2016-05-06',
]
expected_calls = [
mock.call(expected_experiment_name,
expected_config_file,
expected_benchmarks,
expected_fuzzers,
description='Test experiment',
oss_fuzz_corpus=True)
]
start_experiment_call_args = mocked_start_experiment.call_args_list
assert len(start_experiment_call_args) == 1
assert start_experiment_call_args == expected_calls
def test_run_requested_experiment_pause_service(
mocked_get_requested_experiments, mocked_warning,
mocked_start_experiment, db):
"""Tests that run_requested_experiment doesn't run an experiment when a
pause is requested."""
experiment_requests_with_pause = EXPERIMENT_REQUESTS.copy()
experiment_requests_with_pause.append(
automatic_run_experiment.PAUSE_SERVICE_KEYWORD)
mocked_get_requested_experiments.return_value = (
experiment_requests_with_pause)
assert (automatic_run_experiment.run_requested_experiment(dry_run=False) is
None)
mocked_warning.assert_called_with(
'Pause service requested, not running experiment.')
assert mocked_start_experiment.call_count == 0