def _get_cross_pollinate_fuzzers(engine_name, current_fuzzer_name):
"""Return a list of fuzzer objects to use for cross pollination."""
cross_pollinate_fuzzers = []
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name))
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
targets_and_jobs = [(target, target_job)
for target, target_job in zip(targets, target_jobs)
if target_job.fuzz_target_name != current_fuzzer_name]
selected_targets_and_jobs = random.SystemRandom().sample(
targets_and_jobs,
min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT))
default_backup_bucket = utils.default_backup_bucket()
for target, target_job in selected_targets_and_jobs:
job = data_types.Job.query(data_types.Job.name == target_job.job).get()
if not job:
continue
job_environment = job.get_environment()
backup_bucket_name = job_environment.get("BACKUP_BUCKET",
default_backup_bucket)
if not backup_bucket_name:
continue
corpus_engine_name = job_environment.get("CORPUS_FUZZER_NAME_OVERRIDE",
engine_name)
cross_pollinate_fuzzers.append(
CrossPollinateFuzzer(target, backup_bucket_name,
corpus_engine_name))
return cross_pollinate_fuzzers
def _select_targets_and_jobs_for_pollination(engine_name, current_fuzzer_name,
method, tag):
"""Select jobs to use for cross pollination."""
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name))
if method == Pollination.TAGGED:
similar_tagged_targets = [
target.fully_qualified_fuzz_target_name
for target in corpus_tagging.get_targets_with_tag(tag)
if target.fully_qualified_fuzz_target_name != current_fuzzer_name
]
# Intersect target_jobs and similar_tagged_targets on fully qualified
# fuzz target name.
target_jobs = [
target for target in target_jobs
if target.fuzz_target_name in similar_tagged_targets
]
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
targets_and_jobs = [(target, target_job)
for target, target_job in zip(targets, target_jobs)
if target_job.fuzz_target_name != current_fuzzer_name]
selected_targets_and_jobs = random.SystemRandom().sample(
targets_and_jobs,
min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT))
return selected_targets_and_jobs
def _get_cross_pollinate_fuzzers(engine, current_fuzzer_name):
"""Return a list of fuzzer objects to use for cross pollination."""
cross_pollinate_fuzzers = {}
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(engine=engine))
targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs)
for target, target_job in zip(targets, target_jobs):
if (target_job.fuzz_target_name == current_fuzzer_name
or target_job.fuzz_target_name in cross_pollinate_fuzzers):
continue
job = data_types.Job.query(data_types.Job.name == target_job.job).get()
if not job:
continue
backup_bucket_name = job.get_environment().get('BACKUP_BUCKET')
if not backup_bucket_name:
continue
corpus_engine_name = job.get_environment().get(
'CORPUS_FUZZER_NAME_OVERRIDE', engine)
cross_pollinate_fuzzers[
target_job.fuzz_target_name] = CrossPollinateFuzzer(
target,
backup_bucket_name,
corpus_engine_name,
)
return random.SystemRandom().sample(
cross_pollinate_fuzzers.values(),
min(len(cross_pollinate_fuzzers), CROSS_POLLINATE_FUZZER_COUNT))
def get(self):
"""Handle a GET request."""
jobs = ndb_utils.get_all_from_model(data_types.Job)
default_backup_bucket = utils.default_backup_bucket()
for job in jobs:
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get("EXPERIMENTAL")):
# Don't use corpus backups from experimental jobs. Skip.
continue
if not utils.string_is_true(job_environment.get("CORPUS_PRUNE")):
# There won't be any corpus backups for these jobs. Skip.
continue
corpus_backup_bucket_name = job_environment.get(
"BACKUP_BUCKET", default_backup_bucket)
if not corpus_backup_bucket_name:
# No backup bucket found. Skip.
continue
corpus_fuzzer_name_override = job_environment.get(
"CORPUS_FUZZER_NAME_OVERRIDE")
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
if not target:
# This is expected if any fuzzer/job combinations become outdated.
continue
_make_corpus_backup_public(target, corpus_fuzzer_name_override,
corpus_backup_bucket_name)
def get(self):
"""Handle a GET request."""
jobs = ndb_utils.get_all_from_model(data_types.Job)
for job in jobs:
job_environment = job.get_environment()
if utils.string_is_true(job_environment.get('EXPERIMENTAL')):
# Don't use corpus backups from experimental jobs. Skip.
continue
if not utils.string_is_true(job_environment.get('CORPUS_PRUNE')):
# There won't be any corpus backups for these jobs. Skip.
continue
corpus_backup_bucket_name = job_environment.get('BACKUP_BUCKET')
if not corpus_backup_bucket_name:
# No backup bucket found. Skip.
continue
corpus_fuzzer_name_override = job_environment.get(
'CORPUS_FUZZER_NAME_OVERRIDE')
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
_make_corpus_backup_public(target, corpus_fuzzer_name_override,
corpus_backup_bucket_name)
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
models = job.get_environment().get('ML_MODELS_TO_USE')
if not models:
continue
task_list = []
for model_name in models.split(','):
try:
task_list.append(MODEL_NAME_TO_TASK[model_name.strip()])
except KeyError:
logs.log_error(
f'Invalid ML model {model_name} for job {job.name}.')
if not task_list:
continue
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for task_name in task_list:
for target in fuzz_targets:
tasks.add_task(task_name,
target.fully_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
if not utils.string_is_true(
job.get_environment().get('USE_CORPUS_FOR_ML')):
continue
task_list = []
if utils.string_is_true(
job.get_environment().get('USE_GRADIENTFUZZ')):
task_list.append('train_gradientfuzz')
if utils.string_is_true(
job.get_environment().get('USE_RNN_GENERATOR')):
task_list.append('train_rnn_generator')
if len(task_list) == 0:
continue
target_jobs = list(
fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for task_name in task_list:
for target in fuzz_targets:
tasks.add_task(task_name,
target.project_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get(self):
"""Handle a GET request."""
for job in data_types.Job.query():
if not utils.string_is_true(
job.get_environment().get('USE_CORPUS_FOR_ML')):
continue
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job.name))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
for target in fuzz_targets:
tasks.add_task(
'ml_train',
target.project_qualified_name(),
job.name,
queue=tasks.ML_JOBS_TASKQUEUE)
def get_fuzz_target_weights():
"""Get a list of fuzz target weights based on the current fuzzer."""
job_type = environment.get_value('JOB_NAME')
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
weights = {}
for fuzz_target, target_job in zip(fuzz_targets, target_jobs):
if not fuzz_target:
logs.log_error('Skipping weight assignment for fuzz target %s.' %
target_job.fuzz_target_name)
continue
weights[fuzz_target.binary] = target_job.weight
return weights
def get_fuzz_target_weights():
"""Get a list of fuzz target weights based on the current fuzzer."""
# No work to do if this isn't fuzz task. Weights are only required if a
# fuzzer has not yet been selected.
task_name = environment.get_value('TASK_NAME')
if task_name != 'fuzz':
return None
job_type = environment.get_value('JOB_NAME')
target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type))
fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(
target_jobs)
weights = {}
for fuzz_target, target_job in zip(fuzz_targets, target_jobs):
if not fuzz_target:
logs.log_error('Skipping weight assignment for fuzz target %s.' %
target_job.fuzz_target_name)
continue
weights[fuzz_target.binary] = target_job.weight
return weights
