def pending_trials(db, experiment_config):
"""Adds trials to the database and returns pending trials."""
other_experiment_name = experiment_config['experiment'] + 'other'
db_utils.add_all([
models.Experiment(name=experiment_config['experiment']),
models.Experiment(name=other_experiment_name)
])
def create_trial(experiment, time_started=None, time_ended=None):
"""Creates a database trial."""
return models.Trial(experiment=experiment,
benchmark=BENCHMARK,
fuzzer=FUZZER,
time_started=time_started,
time_ended=time_ended)
our_pending_trials = [
create_trial(experiment_config['experiment']),
create_trial(experiment_config['experiment'])
]
other_trials = [
create_trial(other_experiment_name),
create_trial(experiment_config['experiment'], datetime.datetime.now()),
create_trial(experiment_config['experiment'], datetime.datetime.now())
]
db_utils.add_all(other_trials + our_pending_trials)
our_trial_ids = [trial.id for trial in our_pending_trials]
return db_utils.query(models.Trial).filter(
models.Trial.id.in_(our_trial_ids))
def test_add_nonprivate_experiments_for_merge_with_clobber(db):
"""Tests that add_nonprivate_experiments_for_merge_with_clobber doesn't
include private experiments and returns the expected results in the correct
order."""
experiment_names = ['1', '2', '3']
arbitrary_datetime = datetime.datetime(2020, 1, 1)
db_utils.add_all([
models.Experiment(name=name,
time_created=arbitrary_datetime,
private=False) for name in experiment_names
])
db_utils.add_all([
models.Experiment(name='private',
time_created=arbitrary_datetime,
private=True),
models.Experiment(name='later-nonprivate',
time_created=arbitrary_datetime +
datetime.timedelta(days=1),
private=False),
models.Experiment(name='nonprivate',
time_created=arbitrary_datetime,
private=False),
])
expected_results = ['nonprivate', 'later-nonprivate', '1', '2', '3']
results = queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names)
assert results == expected_results
def test_add_nonprivate_experiments_for_merge_with_clobber(db):
"""Tests that add_nonprivate_experiments_for_merge_with_clobber doesn't
include private experiments and returns the expected results in the correct
order."""
experiment_names = ['1', '2', '3']
db_utils.add_all([
models.Experiment(name=name,
time_created=ARBITRARY_DATETIME,
time_ended=ARBITRARY_DATETIME +
datetime.timedelta(days=1),
private=False) for name in experiment_names
])
db_utils.add_all([
models.Experiment(name='private',
time_created=ARBITRARY_DATETIME,
private=True),
models.Experiment(name='earlier-nonprivate',
time_created=ARBITRARY_DATETIME -
datetime.timedelta(days=1),
time_ended=ARBITRARY_DATETIME,
private=False),
models.Experiment(name='nonprivate',
time_created=ARBITRARY_DATETIME,
time_ended=ARBITRARY_DATETIME +
datetime.timedelta(days=1),
private=False),
models.Experiment(name='nonprivate-in-progress',
time_created=ARBITRARY_DATETIME,
time_ended=None,
private=False),
])
expected_results = ['earlier-nonprivate', 'nonprivate', '1', '2', '3']
results = queries.add_nonprivate_experiments_for_merge_with_clobber(
experiment_names)
assert results == expected_results
def create_experiments(experiment_config):
"""Create the experiment experiment entity for the experiment in
|experiment_config| and create another one and save the results to the
db."""
other_experiment_name = get_other_experiment_name(experiment_config)
db_utils.add_all([
models.Experiment(name=experiment_config['experiment']),
models.Experiment(name=other_experiment_name)
])
def db_experiment(experiment_config, db):
"""A fixture that populates the database with an experiment entity with the
name specified in the experiment_config fixture."""
experiment = models.Experiment(name=experiment_config['experiment'])
db_utils.add_all([experiment])
# yield so that the experiment exists until the using function exits.
yield
def test_get_fuzzers_changed_since_last_non_master_experiment(
mocked_info, mocked_get_changed_files, db_experiment):
"""Tests that get_fuzzers_changed_since_last returns the
correct result when the first experiment's git hash is not in branch"""
# Set up a newer, out-of-branch experiment.
out_of_branch_experiment = models.Experiment()
out_of_branch_experiment.name = 'out-of-branch-experiment'
out_of_branch_hash = 'out-of-branch-experiment-hash'
out_of_branch_experiment.git_hash = out_of_branch_hash
db_utils.add_all([out_of_branch_experiment])
# Update the time of out_of_branch_experiment to come after db_experiment.
out_of_branch_experiment.time_created = (db_experiment.time_created +
datetime.timedelta(days=1))
db_utils.add_all([out_of_branch_experiment])
def get_changed_files(commit_hash):
if commit_hash == 'out-of-branch-experiment-hash':
raise diff_utils.DiffError(commit_hash)
return AFL_FUZZER_PY
mocked_get_changed_files.side_effect = get_changed_files
assert not experiment_changes.get_fuzzers_changed_since_last()
mocked_info.assert_called_with('Skipping %s. Commit is not in branch.',
out_of_branch_hash)
mocked_get_changed_files.assert_has_calls(
[mock.call(out_of_branch_hash),
mock.call('hash')])
def db_experiment(db):
"""Fixture that creates a database populated the databse with an
experiment."""
experiment = models.Experiment()
experiment.name = 'experiment'
experiment.git_hash = 'hash'
db_utils.add_all([experiment])
return experiment
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals
self, mocked_is_cycle_unchanged, db, experiment, tmp_path):
"""Integration test for measure_snapshot_coverage."""
# WORK is set by experiment to a directory that only makes sense in a
# fakefs. A directory containing necessary llvm tools is also added to
# PATH.
llvm_tools_path = get_test_data_path('llvm_tools')
os.environ["PATH"] += os.pathsep + llvm_tools_path
os.environ['WORK'] = str(tmp_path)
mocked_is_cycle_unchanged.return_value = False
# Set up the coverage binary.
benchmark = 'freetype2-2017'
coverage_binary_src = get_test_data_path(
'test_measure_snapshot_coverage', benchmark + '-coverage')
benchmark_cov_binary_dir = os.path.join(
build_utils.get_coverage_binaries_dir(), benchmark)
os.makedirs(benchmark_cov_binary_dir)
coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir,
'ftfuzzer')
shutil.copy(coverage_binary_src, coverage_binary_dst_dir)
# Set up entities in database so that the snapshot can be created.
experiment = models.Experiment(name=os.environ['EXPERIMENT'])
db_utils.add_all([experiment])
trial = models.Trial(fuzzer=FUZZER,
benchmark=benchmark,
experiment=os.environ['EXPERIMENT'])
db_utils.add_all([trial])
snapshot_measurer = measurer.SnapshotMeasurer(trial.fuzzer,
trial.benchmark,
trial.id,
SNAPSHOT_LOGGER)
# Set up the snapshot archive.
cycle = 1
archive = get_test_data_path('test_measure_snapshot_coverage',
'corpus-archive-%04d.tar.gz' % cycle)
corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus')
os.makedirs(corpus_dir)
shutil.copy(archive, corpus_dir)
with mock.patch('common.filestore_utils.cp') as mocked_cp:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
# TODO(metzman): Create a system for using actual buckets in
# integration tests.
snapshot = measurer.measure_snapshot_coverage(
snapshot_measurer.fuzzer, snapshot_measurer.benchmark,
snapshot_measurer.trial_num, cycle)
assert snapshot
assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds()
assert snapshot.edges_covered == 13178
def test_measure_snapshot_coverage( # pylint: disable=too-many-locals
self, mocked_is_cycle_unchanged, create_measurer, db, experiment):
"""Integration test for measure_snapshot_coverage."""
mocked_is_cycle_unchanged.return_value = False
# Set up the coverage binary.
benchmark = 'freetype2-2017'
coverage_binary_src = get_test_data_path(
'test_measure_snapshot_coverage', benchmark + '-coverage')
benchmark_cov_binary_dir = os.path.join(
build_utils.get_coverage_binaries_dir(), benchmark)
os.makedirs(benchmark_cov_binary_dir)
coverage_binary_dst_dir = os.path.join(benchmark_cov_binary_dir,
'fuzz-target')
shutil.copy(coverage_binary_src, coverage_binary_dst_dir)
# Set up entities in database so that the snapshot can be created.
experiment = models.Experiment(name=os.environ['EXPERIMENT'])
db_utils.add_all([experiment])
trial = models.Trial(fuzzer=FUZZER,
benchmark=benchmark,
experiment=os.environ['EXPERIMENT'])
db_utils.add_all([trial])
snapshot_measurer = create_measurer(trial.fuzzer, trial.benchmark,
trial.id)
# Set up the snapshot archive.
cycle = 1
archive = get_test_data_path('test_measure_snapshot_coverage',
'corpus-archive-%04d.tar.gz' % cycle)
corpus_dir = os.path.join(snapshot_measurer.trial_dir, 'corpus')
os.makedirs(corpus_dir)
shutil.copy(archive, corpus_dir)
with mock.patch('common.gsutil.cp') as mocked_cp:
mocked_cp.return_value = new_process.ProcessResult(0, '', False)
# TODO(metzman): Create a system for using actual buckets in
# integration tests.
snapshot = measurer.measure_snapshot_coverage(
snapshot_measurer.fuzzer, snapshot_measurer.benchmark,
snapshot_measurer.trial_num, cycle)
assert snapshot
assert snapshot.time == cycle * experiment_utils.get_snapshot_seconds()
assert snapshot.edges_covered == 3798
def test_get_last_trial_time_started_called_early(db, experiment_config):
"""Tests that get_last_trial_time_started raises an exception if called
while there are still pending trials."""
experiment = experiment_config['experiment']
db_utils.add_all([
models.Experiment(name=experiment),
])
trial1 = models.Trial(experiment=experiment,
benchmark=BENCHMARK,
fuzzer=FUZZER)
trial2 = models.Trial(experiment=experiment,
benchmark=BENCHMARK,
fuzzer=FUZZER)
first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0)))
trial1.time_started = first_time
trials = [trial1, trial2]
db_utils.add_all(trials)
with pytest.raises(AssertionError):
scheduler.get_last_trial_time_started(experiment)
def test_get_experiment_data_fuzzer_stats(db):
"""Tests that get_experiment_data handles fuzzer_stats correctly."""
experiment_name = 'experiment-1'
db_utils.add_all([
models.Experiment(name=experiment_name,
time_created=ARBITRARY_DATETIME,
private=False)
])
trial = models.Trial(fuzzer='afl',
experiment=experiment_name,
benchmark='libpng')
db_utils.add_all([trial])
fuzzer_stats = {'execs_per_sec': 100.0}
snapshot = models.Snapshot(time=900,
trial_id=trial.id,
edges_covered=100,
fuzzer_stats=fuzzer_stats)
db_utils.add_all([snapshot])
experiment_df = queries.get_experiment_data([experiment_name]) # pylint: disable=unused-variable
def test_get_last_trial_time_started(db, experiment_config):
"""Tests that get_last_trial_time_started returns the time_started of the
last trial to be started."""
experiment = experiment_config['experiment']
db_utils.add_all([
models.Experiment(name=experiment),
])
trial1 = models.Trial(experiment=experiment,
benchmark=BENCHMARK,
fuzzer=FUZZER)
trial2 = models.Trial(experiment=experiment,
benchmark=BENCHMARK,
fuzzer=FUZZER)
first_time = datetime.datetime.fromtimestamp(time.mktime(time.gmtime(0)))
trial1.time_started = first_time
last_time_started = first_time + datetime.timedelta(days=1)
trial2.time_started = last_time_started
trials = [trial1, trial2]
db_utils.add_all(trials)
assert scheduler.get_last_trial_time_started(
experiment) == last_time_started