def _get_cross_pollinate_fuzzers(engine_name, current_fuzzer_name): """Return a list of fuzzer objects to use for cross pollination.""" cross_pollinate_fuzzers = [] target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name)) targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs) targets_and_jobs = [(target, target_job) for target, target_job in zip(targets, target_jobs) if target_job.fuzz_target_name != current_fuzzer_name] selected_targets_and_jobs = random.SystemRandom().sample( targets_and_jobs, min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT)) default_backup_bucket = utils.default_backup_bucket() for target, target_job in selected_targets_and_jobs: job = data_types.Job.query(data_types.Job.name == target_job.job).get() if not job: continue job_environment = job.get_environment() backup_bucket_name = job_environment.get("BACKUP_BUCKET", default_backup_bucket) if not backup_bucket_name: continue corpus_engine_name = job_environment.get("CORPUS_FUZZER_NAME_OVERRIDE", engine_name) cross_pollinate_fuzzers.append( CrossPollinateFuzzer(target, backup_bucket_name, corpus_engine_name)) return cross_pollinate_fuzzers
def get(self): """Handle a GET request.""" jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: job_environment = job.get_environment() if utils.string_is_true(job_environment.get('EXPERIMENTAL')): # Don't use corpus backups from experimental jobs. Skip. continue if not utils.string_is_true(job_environment.get('CORPUS_PRUNE')): # There won't be any corpus backups for these jobs. Skip. continue corpus_backup_bucket_name = job_environment.get('BACKUP_BUCKET') if not corpus_backup_bucket_name: # No backup bucket found. Skip. continue corpus_fuzzer_name_override = job_environment.get( 'CORPUS_FUZZER_NAME_OVERRIDE') target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for target in fuzz_targets: _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_bucket_name)
def _get_cross_pollinate_fuzzers(engine, current_fuzzer_name): """Return a list of fuzzer objects to use for cross pollination.""" cross_pollinate_fuzzers = {} target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(engine=engine)) targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs) for target, target_job in zip(targets, target_jobs): if (target_job.fuzz_target_name == current_fuzzer_name or target_job.fuzz_target_name in cross_pollinate_fuzzers): continue job = data_types.Job.query(data_types.Job.name == target_job.job).get() if not job: continue backup_bucket_name = job.get_environment().get('BACKUP_BUCKET') if not backup_bucket_name: continue corpus_engine_name = job.get_environment().get( 'CORPUS_FUZZER_NAME_OVERRIDE', engine) cross_pollinate_fuzzers[ target_job.fuzz_target_name] = CrossPollinateFuzzer( target, backup_bucket_name, corpus_engine_name, ) return random.SystemRandom().sample( cross_pollinate_fuzzers.values(), min(len(cross_pollinate_fuzzers), CROSS_POLLINATE_FUZZER_COUNT))
def _select_targets_and_jobs_for_pollination(engine_name, current_fuzzer_name, method, tag): """Select jobs to use for cross pollination.""" target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(engine=engine_name)) if method == Pollination.TAGGED: similar_tagged_targets = [ target.fully_qualified_fuzz_target_name for target in corpus_tagging.get_targets_with_tag(tag) if target.fully_qualified_fuzz_target_name != current_fuzzer_name ] # Intersect target_jobs and similar_tagged_targets on fully qualified # fuzz target name. target_jobs = [ target for target in target_jobs if target.fuzz_target_name in similar_tagged_targets ] targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs(target_jobs) targets_and_jobs = [(target, target_job) for target, target_job in zip(targets, target_jobs) if target_job.fuzz_target_name != current_fuzzer_name] selected_targets_and_jobs = random.SystemRandom().sample( targets_and_jobs, min(len(targets_and_jobs), CROSS_POLLINATE_FUZZER_COUNT)) return selected_targets_and_jobs
def get(self): """Handle a GET request.""" jobs = ndb_utils.get_all_from_model(data_types.Job) default_backup_bucket = utils.default_backup_bucket() for job in jobs: job_environment = job.get_environment() if utils.string_is_true(job_environment.get("EXPERIMENTAL")): # Don't use corpus backups from experimental jobs. Skip. continue if not utils.string_is_true(job_environment.get("CORPUS_PRUNE")): # There won't be any corpus backups for these jobs. Skip. continue corpus_backup_bucket_name = job_environment.get( "BACKUP_BUCKET", default_backup_bucket) if not corpus_backup_bucket_name: # No backup bucket found. Skip. continue corpus_fuzzer_name_override = job_environment.get( "CORPUS_FUZZER_NAME_OVERRIDE") target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for target in fuzz_targets: if not target: # This is expected if any fuzzer/job combinations become outdated. continue _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_bucket_name)
def get(self): """Handle a GET request.""" for job in data_types.Job.query(): models = job.get_environment().get('ML_MODELS_TO_USE') if not models: continue task_list = [] for model_name in models.split(','): try: task_list.append(MODEL_NAME_TO_TASK[model_name.strip()]) except KeyError: logs.log_error( f'Invalid ML model {model_name} for job {job.name}.') if not task_list: continue target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for task_name in task_list: for target in fuzz_targets: tasks.add_task(task_name, target.fully_qualified_name(), job.name, queue=tasks.ML_JOBS_TASKQUEUE)
def get(self): """Handle a GET request.""" for job in data_types.Job.query(): if not utils.string_is_true( job.get_environment().get('USE_CORPUS_FOR_ML')): continue task_list = [] if utils.string_is_true( job.get_environment().get('USE_GRADIENTFUZZ')): task_list.append('train_gradientfuzz') if utils.string_is_true( job.get_environment().get('USE_RNN_GENERATOR')): task_list.append('train_rnn_generator') if len(task_list) == 0: continue target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for task_name in task_list: for target in fuzz_targets: tasks.add_task(task_name, target.project_qualified_name(), job.name, queue=tasks.ML_JOBS_TASKQUEUE)
def get_tasks_to_schedule(): """Return (task_target, job_name, queue_name) arguments to schedule a task.""" for job in data_types.Job.query(): if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')): continue queue_name = tasks.queue_for_job(job.name) for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job.name): task_target = target_job.fuzz_target_name yield (task_target, job.name, queue_name)
def _get_logs_bucket_from_fuzzer(self, fuzzer_name): """Get logs bucket from fuzzer (child fuzzers only).""" jobs = [ mapping.job for mapping in fuzz_target_utils.get_fuzz_target_jobs( fuzz_target_name=fuzzer_name) ] if not jobs: return None # Check that the logs bucket is same for all of them. bucket = self._get_logs_bucket_from_job(jobs[0]) if all(bucket == self._get_logs_bucket_from_job(job) for job in jobs[1:]): return bucket return None
def get(self): """Handle a GET request.""" for job in data_types.Job.query(): if not utils.string_is_true( job.get_environment().get('USE_CORPUS_FOR_ML')): continue target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for target in fuzz_targets: tasks.add_task( 'ml_train', target.project_qualified_name(), job.name, queue=tasks.ML_JOBS_TASKQUEUE)
def get_fuzz_target_weights(): """Get a list of fuzz target weights based on the current fuzzer.""" job_type = environment.get_value('JOB_NAME') target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) weights = {} for fuzz_target, target_job in zip(fuzz_targets, target_jobs): if not fuzz_target: logs.log_error('Skipping weight assignment for fuzz target %s.' % target_job.fuzz_target_name) continue weights[fuzz_target.binary] = target_job.weight return weights
def get_tasks_to_schedule(): """Return (task_target, job_name, queue_name) arguments to schedule a task.""" for job in data_types.Job.query(): if not utils.string_is_true(job.get_environment().get('CORPUS_PRUNE')): continue if utils.string_is_true(job.get_environment().get('CUSTOM_BINARY')): # Custom binary jobs do not have revisions. latest_revision = None else: latest_revision = _get_latest_job_revision(job) if not latest_revision: continue queue_name = tasks.queue_for_job(job.name) for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job.name): task_target = target_job.fuzz_target_name if latest_revision: task_target += '@%s' % latest_revision yield (task_target, job.name, queue_name)
def _fuzzers_for_job(job_type, include_parents): """Return all fuzzers that have the job associated. Args: job_type: The job type. include_parents: Include the parent fuzzer. Returns: A list of fuzzer names. """ fuzzers = [] engine_fuzzers = data_handler.get_fuzzing_engines() for fuzzer in data_types.Fuzzer.query(data_types.Fuzzer.jobs == job_type): # Add this if we're including all parents or this is not an engine fuzzer # with fuzz targets. if include_parents or fuzzer.name not in engine_fuzzers: fuzzers.append(fuzzer.name) for target_job in fuzz_target_utils.get_fuzz_target_jobs(job=job_type): fuzzers.append(target_job.fuzz_target_name) return sorted(fuzzers)
def get_fuzz_target_weights(): """Get a list of fuzz target weights based on the current fuzzer.""" # No work to do if this isn't fuzz task. Weights are only required if a # fuzzer has not yet been selected. task_name = environment.get_value('TASK_NAME') if task_name != 'fuzz': return None job_type = environment.get_value('JOB_NAME') target_jobs = list(fuzz_target_utils.get_fuzz_target_jobs(job=job_type)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) weights = {} for fuzz_target, target_job in zip(fuzz_targets, target_jobs): if not fuzz_target: logs.log_error('Skipping weight assignment for fuzz target %s.' % target_job.fuzz_target_name) continue weights[fuzz_target.binary] = target_job.weight return weights