def _get_cross_pollinate_fuzzers(engine_name, current_fuzzer_name):
"""Return a list of fuzzer objects to use for cross pollination."""
cross_pollinate_fuzzers = []
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name))
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
targets_and_jobs = [(target, target_job)
for target, target_job in zip(targets, target_jobs)
if target_job.fuzz_target_name != current_fuzzer_name]
selected_targets_and_jobs = random.SystemRandom().sample(
targets_and_jobs,
min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT))
default_backup_bucket = utils.default_backup_bucket()
for target, target_job in selected_targets_and_jobs:
job = data_types.Job.query(data_types.Job.name == target_job.job).get()
if not job:
continue
job_environment = job.get_environment()
backup_bucket_name = job_environment.get("BACKUP_BUCKET",
default_backup_bucket)
if not backup_bucket_name:
continue
corpus_engine_name = job_environment.get("CORPUS_FUZZER_NAME_OVERRIDE",
engine_name)
cross_pollinate_fuzzers.append(
CrossPollinateFuzzer(target, backup_bucket_name,
corpus_engine_name))
return cross_pollinate_fuzzers
def get(self):
"""Handle a GET request."""
jobs = ndb_utils.get_all_from_model(data_types.Job)
for job in jobs:
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get('EXPERIMENTAL')):
# Don't use corpus backups from experimental jobs. Skip.
continue
if not utils.string_is_true(job_environment.get('CORPUS_PRUNE')):
# There won't be any corpus backups for these jobs. Skip.
continue
corpus_backup_bucket_name = job_environment.get('BACKUP_BUCKET')
if not corpus_backup_bucket_name:
# No backup bucket found. Skip.
continue
corpus_fuzzer_name_override = job_environment.get(
'CORPUS_FUZZER_NAME_OVERRIDE')
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
_make_corpus_backup_public(target, corpus_fuzzer_name_override,
corpus_backup_bucket_name)
def _get_cross_pollinate_fuzzers(engine, current_fuzzer_name):
"""Return a list of fuzzer objects to use for cross pollination."""
cross_pollinate_fuzzers = {}
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(engine=engine))
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
for target, target_job in zip(targets, target_jobs):
if (target_job.fuzz_target_name == current_fuzzer_name
or target_job.fuzz_target_name in cross_pollinate_fuzzers):
continue
job = data_types.Job.query(data_types.Job.name == target_job.job).get()
if not job:
continue
backup_bucket_name = job.get_environment().get('BACKUP_BUCKET')
if not backup_bucket_name:
continue
corpus_engine_name = job.get_environment().get(
'CORPUS_FUZZER_NAME_OVERRIDE', engine)
cross_pollinate_fuzzers[
target_job.fuzz_target_name] = CrossPollinateFuzzer(
target,
backup_bucket_name,
corpus_engine_name,
)
return random.SystemRandom().sample(
cross_pollinate_fuzzers.values(),
min(len(cross_pollinate_fuzzers), CROSS_POLLINATE_FUZZER_COUNT))
def _select_targets_and_jobs_for_pollination(engine_name, current_fuzzer_name,
method, tag):
"""Select jobs to use for cross pollination."""
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name))
if method == Pollination.TAGGED:
similar_tagged_targets = [
target.fully_qualified_fuzz_target_name
for target in corpus_tagging.get_targets_with_tag(tag)
if target.fully_qualified_fuzz_target_name != current_fuzzer_name
]
# Intersect target_jobs and similar_tagged_targets on fully qualified
# fuzz target name.
target_jobs = [
target for target in target_jobs
if target.fuzz_target_name in similar_tagged_targets
]
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
targets_and_jobs = [(target, target_job)
for target, target_job in zip(targets, target_jobs)
if target_job.fuzz_target_name != current_fuzzer_name]
selected_targets_and_jobs = random.SystemRandom().sample(
targets_and_jobs,
min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT))
return selected_targets_and_jobs
def get(self):
"""Handle a GET request."""
jobs = ndb_utils.get_all_from_model(data_types.Job)
default_backup_bucket = utils.default_backup_bucket()
for job in jobs:
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get("EXPERIMENTAL")):
# Don't use corpus backups from experimental jobs. Skip.
continue
if not utils.string_is_true(job_environment.get("CORPUS_PRUNE")):
# There won't be any corpus backups for these jobs. Skip.
continue
corpus_backup_bucket_name = job_environment.get(
"BACKUP_BUCKET", default_backup_bucket)
if not corpus_backup_bucket_name:
# No backup bucket found. Skip.
continue
corpus_fuzzer_name_override = job_environment.get(
"CORPUS_FUZZER_NAME_OVERRIDE")
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
if not target:
# This is expected if any fuzzer/job combinations become outdated.
continue
_make_corpus_backup_public(target, corpus_fuzzer_name_override,
corpus_backup_bucket_name)
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
models = job.get_environment().get('ML_MODELS_TO_USE')
if not models:
continue
task_list = []
for model_name in models.split(','):
try:
task_list.append(MODEL_NAME_TO_TASK[model_name.strip()])
except KeyError:
logs.log_error(
f'Invalid ML model {model_name} for job {job.name}.')
if not task_list:
continue
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for task_name in task_list:
for target in fuzz_targets:
tasks.add_task(task_name,
target.fully_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
if not utils.string_is_true(
job.get_environment().get('USE_CORPUS_FOR_ML')):
continue
task_list = []
if utils.string_is_true(
job.get_environment().get('USE_GRADIENTFUZZ')):
task_list.append('train_gradientfuzz')
if utils.string_is_true(
job.get_environment().get('USE_RNN_GENERATOR')):
task_list.append('train_rnn_generator')
if len(task_list) == 0:
continue
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for task_name in task_list:
for target in fuzz_targets:
tasks.add_task(task_name,
target.project_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get_tasks_to_schedule():
"""Return (task_target, job_name, queue_name) arguments to schedule a task."""
for job in data_types.Job.query():
if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')):
continue
queue_name = tasks.queue_for_job(job.name)
for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job.name):
task_target = target_job.fuzz_target_name
yield (task_target, job.name, queue_name)
def _get_logs_bucket_from_fuzzer(self, fuzzer_name):
"""Get logs bucket from fuzzer (child fuzzers only)."""
jobs = [
mapping.job for mapping in fuzz_target_utils.get_fuzz_target_jobs(
fuzz_target_name=fuzzer_name)
]
if not jobs:
return None
# Check that the logs bucket is same for all of them.
bucket = self._get_logs_bucket_from_job(jobs[0])
if all(bucket == self._get_logs_bucket_from_job(job) for job in jobs[1:]):
return bucket
return None
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
if not utils.string_is_true(
job.get_environment().get('USE_CORPUS_FOR_ML')):
continue
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
tasks.add_task(
'ml_train',
target.project_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get_fuzz_target_weights():
"""Get a list of fuzz target weights based on the current fuzzer."""
job_type = environment.get_value('JOB_NAME')
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
weights = {}
for fuzz_target, target_job in zip(fuzz_targets, target_jobs):
if not fuzz_target:
logs.log_error('Skipping weight assignment for fuzz target %s.' %
target_job.fuzz_target_name)
continue
weights[fuzz_target.binary] = target_job.weight
return weights
def get_tasks_to_schedule():
"""Return (task_target, job_name, queue_name) arguments to schedule a task."""
for job in data_types.Job.query():
if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')):
continue
if utils.string_is_true(job.get_environment().get('CUSTOM_BINARY')):
# Custom binary jobs do not have revisions.
latest_revision = None
else:
latest_revision = _get_latest_job_revision(job)
if not latest_revision:
continue
queue_name = tasks.queue_for_job(job.name)
for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job.name):
task_target = target_job.fuzz_target_name
if latest_revision:
task_target += '@%s' % latest_revision
yield (task_target, job.name, queue_name)
def _fuzzers_for_job(job_type, include_parents):
"""Return all fuzzers that have the job associated.
Args:
job_type: The job type.
include_parents: Include the parent fuzzer.
Returns:
A list of fuzzer names.
"""
fuzzers = []
engine_fuzzers = data_handler.get_fuzzing_engines()
for fuzzer in data_types.Fuzzer.query(data_types.Fuzzer.jobs == job_type):
# Add this if we're including all parents or this is not an engine fuzzer
# with fuzz targets.
if include_parents or fuzzer.name not in engine_fuzzers:
fuzzers.append(fuzzer.name)
for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job_type):
fuzzers.append(target_job.fuzz_target_name)
return sorted(fuzzers)
def get_fuzz_target_weights():
"""Get a list of fuzz target weights based on the current fuzzer."""
# No work to do if this isn't fuzz task. Weights are only required if a
# fuzzer has not yet been selected.
task_name = environment.get_value('TASK_NAME')
if task_name != 'fuzz':
return None
job_type = environment.get_value('JOB_NAME')
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
weights = {}
for fuzz_target, target_job in zip(fuzz_targets, target_jobs):
if not fuzz_target:
logs.log_error('Skipping weight assignment for fuzz target %s.' %
target_job.fuzz_target_name)
continue
weights[fuzz_target.binary] = target_job.weight
return weights
