def get(self): """Handle a GET request.""" jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: job_environment = job.get_environment() if utils.string_is_true(job_environment.get('EXPERIMENTAL')): # Don't use corpus backups from experimental jobs. Skip. continue if not utils.string_is_true(job_environment.get('CORPUS_PRUNE')): # There won't be any corpus backups for these jobs. Skip. continue corpus_backup_bucket_name = job_environment.get('BACKUP_BUCKET') if not corpus_backup_bucket_name: # No backup bucket found. Skip. continue corpus_fuzzer_name_override = job_environment.get( 'CORPUS_FUZZER_NAME_OVERRIDE') target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for target in fuzz_targets: _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_bucket_name)
def get(self): """Handle a GET request.""" jobs = ndb_utils.get_all_from_model(data_types.Job) default_backup_bucket = utils.default_backup_bucket() for job in jobs: job_environment = job.get_environment() if utils.string_is_true(job_environment.get("EXPERIMENTAL")): # Don't use corpus backups from experimental jobs. Skip. continue if not utils.string_is_true(job_environment.get("CORPUS_PRUNE")): # There won't be any corpus backups for these jobs. Skip. continue corpus_backup_bucket_name = job_environment.get( "BACKUP_BUCKET", default_backup_bucket) if not corpus_backup_bucket_name: # No backup bucket found. Skip. continue corpus_fuzzer_name_override = job_environment.get( "CORPUS_FUZZER_NAME_OVERRIDE") target_jobs = list( fuzz_target_utils.get_fuzz_target_jobs(job=job.name)) fuzz_targets = fuzz_target_utils.get_fuzz_targets_for_target_jobs( target_jobs) for target in fuzz_targets: if not target: # This is expected if any fuzzer/job combinations become outdated. continue _make_corpus_backup_public(target, corpus_fuzzer_name_override, corpus_backup_bucket_name)
def get_jobs_and_platforms_for_project(): """Return a map of projects to jobs and platforms map to use for picking top crashes.""" all_jobs = ndb_utils.get_all_from_model(data_types.Job) projects_to_jobs_and_platforms = {} for job in all_jobs: job_environment = job.get_environment() # Skip experimental jobs. if utils.string_is_true(job_environment.get('EXPERIMENTAL')): continue # Skip custom binary jobs. if (utils.string_is_true(job_environment.get('CUSTOM_BINARY')) or job_environment.get('SYSTEM_BINARY_DIR')): continue # Skip if explicitly excluded using flag. if utils.string_is_true(job_environment.get('EXCLUDE_FROM_TOP_CRASHES')): continue if job.project not in projects_to_jobs_and_platforms: projects_to_jobs_and_platforms[job.project] = ProjectMap(set(), set()) projects_to_jobs_and_platforms[job.project].jobs.add(job.name) projects_to_jobs_and_platforms[job.project].platforms.add( job_platform_to_real_platform(job.platform)) return projects_to_jobs_and_platforms
def get_jobs_and_platforms_for_top_crashes(): """Return list of jobs and platforms to use for picking top crashes.""" jobs = set() platforms = set() all_jobs = ndb_utils.get_all_from_model(data_types.Job) for job in all_jobs: job_environment = job.get_environment() # Skip experimental jobs. if utils.string_is_true(job_environment.get('EXPERIMENTAL')): continue # Skip custom binary jobs. if (utils.string_is_true(job_environment.get('CUSTOM_BINARY')) or job_environment.get('SYSTEM_BINARY_DIR')): continue # Skip if explicitly excluded using flag. if utils.string_is_true( job_environment.get('EXCLUDE_FROM_TOP_CRASHES')): continue jobs.add(job.name) platforms.add(job_platform_to_real_platform(job.platform)) return jobs, platforms
def _query_and_upload_strategy_probabilities(): """Uploads queried data into datastore. Calls query functions and uploads query results to datastore to use as new probabilities. Probabilities are based on new_edges feature.""" strategy_data = [] data = _query_multi_armed_bandit_probabilities() # TODO(mukundv): Update once we choose a temperature parameter for final # implementation. for row in data: curr_strategy = data_types.FuzzStrategyProbability() curr_strategy.strategy_name = str(row['strategy']) curr_strategy.probability_high_temperature = float( row['bandit_weight_high_temperature']) curr_strategy.probability_low_temperature = float( row['bandit_weight_low_temperature']) curr_strategy.probability_medium_temperature = float( row['bandit_weight_medium_temperature']) strategy_data.append(curr_strategy) ndb.delete_multi([ entity.key for entity in ndb_utils.get_all_from_model( data_types.FuzzStrategyProbability) ]) ndb.put_multi(strategy_data) _store_probabilities_in_bigquery(data)
def create_variant_tasks_if_needed(testcase): """Creates a variant task if needed.""" testcase_id = testcase.key.id() jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: # The variant needs to be tested in a different job type than us. job_type = job.name if testcase.job_type == job_type: continue # Don't try to reproduce engine fuzzer testcase with blackbox fuzzer # testcases and vice versa. if (environment.is_engine_fuzzer_job(testcase.job_type) != environment.is_engine_fuzzer_job(job_type)): continue # Skip experimental jobs. job_environment = job.get_environment() if utils.string_is_true(job_environment.get('EXPERIMENTAL')): continue # Don't look for variants in other projects. project_name = data_handler.get_project_name(job_type) if testcase.project_name != project_name: continue queue = tasks.queue_for_platform(job.platform) tasks.add_task('variant', testcase_id, job_type, queue) variant = data_handler.get_testcase_variant(testcase_id, job_type) variant.status = data_types.TestcaseVariantStatus.PENDING variant.put()
def _query_and_upload_strategy_probabilities(engine): """Uploads queried data into datastore. Calls query functions and uploads query results to datastore to use as new probabilities. Probabilities are based on new_edges feature.""" strategy_data = [] data = _query_multi_armed_bandit_probabilities(engine) logs.log('Queried distribution for {}.'.format(engine.name)) # TODO(mukundv): Update once we choose a temperature parameter for final # implementation. for row in data: curr_strategy = data_types.FuzzStrategyProbability() curr_strategy.strategy_name = str(row['strategy']) curr_strategy.probability = float(row['bandit_weight']) curr_strategy.engine = engine.name strategy_data.append(curr_strategy) ndb.delete_multi([ entity.key for entity in ndb_utils.get_all_from_model( data_types.FuzzStrategyProbability) ]) ndb.put_multi(strategy_data) logs.log('Uploaded queried distribution to ndb for {}'.format(engine.name)) _store_probabilities_in_bigquery(engine, data) logs.log('Uploaded queried distribution to BigQuery for {}'.format( engine.name))
def get(self): """Render the bot list HTML.""" if utils.is_oss_fuzz(): heartbeats = _get_host_workers_heartbeats() else: heartbeats = ndb_utils.get_all_from_model(data_types.Heartbeat) bots = _convert_heartbeats_to_dicts(heartbeats) self.render('bots.html', { 'bots': bots, })
def _get_excluded_jobs(): """Return list of jobs excluded from bug filing.""" excluded_jobs = [] jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: job_environment = job.get_environment() # Exclude experimental jobs. if utils.string_is_true(job_environment.get('EXPERIMENTAL')): excluded_jobs.append(job.name) return excluded_jobs
def store_current_weights_in_bigquery(): """Update a bigquery table containing the daily stats.""" rows = [] target_jobs = ndb_utils.get_all_from_model(data_types.FuzzTargetJob) for target_job in target_jobs: row = { 'fuzzer': target_job.fuzz_target_name, 'job': target_job.job, 'weight': target_job.weight } rows.append(big_query.Insert(row=row, insert_id=None)) client = big_query.Client(dataset_id='main', table_id='fuzzer_weights') client.insert(rows)
def get(self): """Render dead bots as json (used by automated scripts).""" # This a publicly exposed chromium-specific page. if utils.is_chromium(): heartbeats = ndb_utils.get_all_from_model(data_types.Heartbeat) else: raise helpers.EarlyExitException('Dead bots list unavailable.', 400) result = {} alive_cutoff = _get_alive_cutoff() for heartbeat in heartbeats: if heartbeat.last_beat_time <= alive_cutoff: result[heartbeat.bot_name] = 'dead' self.render_json(result)
def cleanup_unused_fuzz_targets_and_jobs(): """Clean up unused FuzzTarget and FuzzTargetJob entities.""" last_run_cutoff = utils.utcnow() - datetime.timedelta( days=FUZZ_TARGET_UNUSED_THRESHOLD) unused_target_jobs = data_types.FuzzTargetJob.query( data_types.FuzzTargetJob.last_run < last_run_cutoff) valid_target_jobs = data_types.FuzzTargetJob.query( data_types.FuzzTargetJob.last_run >= last_run_cutoff) to_delete = [t.key for t in unused_target_jobs] valid_fuzz_targets = set(t.fuzz_target_name for t in valid_target_jobs) for fuzz_target in ndb_utils.get_all_from_model(data_types.FuzzTarget): if fuzz_target.fully_qualified_name() not in valid_fuzz_targets: to_delete.append(fuzz_target.key) ndb.delete_multi(to_delete)
def get_excluded_jobs(): """Return list of jobs excluded from bug filing.""" excluded_jobs = [] jobs = ndb_utils.get_all_from_model(data_types.Job) for job in jobs: job_environment = job.get_environment() # Exclude experimental jobs. if utils.string_is_true(job_environment.get('EXPERIMENTAL')): excluded_jobs.append(job.name) # Exclude custom binary jobs. elif (utils.string_is_true(job_environment.get('CUSTOM_BINARY')) or job_environment.get('SYSTEM_BINARY_DIR')): excluded_jobs.append(job.name) return excluded_jobs
def test_jobs_updated(self): """Ensure that we properly update multiple jobs.""" fuzzer_1_mapping = data_types.FuzzerJob() fuzzer_1_mapping.fuzzer = 'fuzzer_1' fuzzer_1_mapping.job = 'test_job' fuzzer_1_mapping.platform = 'wrong_platform' fuzzer_1_mapping.put() fuzzer_2_mapping = data_types.FuzzerJob() fuzzer_2_mapping.fuzzer = 'fuzzer_2' fuzzer_2_mapping.job = 'test_job' fuzzer_2_mapping.platform = 'wrong_platform' fuzzer_2_mapping.put() fuzzer_selection.update_platform_for_job('test_job', 'right_platform') platforms = [ job.platform for job in ndb_utils.get_all_from_model(data_types.FuzzerJob) ] self.assertListEqual(platforms, ['right_platform', 'right_platform'])
def _query_and_upload_strategy_probabilities(): """Uploads queried data into datastore. Calls query functions and uploads query results to datastore to use as new probabilities. Probabilities are based on new_edges feature.""" strategy_data = [] data = _query_multi_armed_bandit_probabilities() for row in data: curr_strategy = data_types.FuzzStrategyProbability() curr_strategy.strategy_name = str(row['strategy']) curr_strategy.probability = float(row['bandit_weight']) strategy_data.append(curr_strategy) ndb.delete_multi([ entity.key for entity in ndb_utils.get_all_from_model( data_types.FuzzStrategyProbability) ]) ndb.put_multi(strategy_data) _store_probabilities_in_bigquery(data)
def update_admins(new_admins): """Update list of admins.""" existing_admins = ndb_utils.get_all_from_model(data_types.Admin) to_remove = [] existing_admin_emails = set() for admin in existing_admins: if admin.email not in new_admins: logs.log('Removing admin ' + admin.email) to_remove.append(admin.key) existing_admin_emails.add(admin.email) ndb_utils.delete_multi(to_remove) to_add = [] for admin in new_admins: if admin not in existing_admin_emails: to_add.append(data_types.Admin(id=admin, email=admin)) logs.log('Adding admin ' + admin) ndb_utils.put_multi(to_add)
def post(self): """Handle a post request.""" key = helpers.get_integer_key(self.request) job = ndb.Key(data_types.Job, key).get() if not job: raise helpers.EarlyExitException('Job not found.', 400) # Delete from fuzzers' jobs' list. for fuzzer in ndb_utils.get_all_from_model(data_types.Fuzzer): if job.name in fuzzer.jobs: fuzzer.jobs.remove(job.name) fuzzer.put() # Delete associated fuzzer-job mapping(s). query = data_types.FuzzerJob.query() query = query.filter(data_types.FuzzerJob.job == job.name) for mapping in ndb_utils.get_all_from_query(query): mapping.key.delete() # Delete job. job.key.delete() helpers.log('Deleted job %s' % job.name, helpers.MODIFY_OPERATION) self.redirect('/jobs')
def unpack_crash_testcases(crash_testcases_directory): """Unpacks the old crash testcases in the provided directory.""" for testcase in ndb_utils.get_all_from_model(data_types.Testcase): testcase_id = testcase.key.id() # 1. If we have already stored the testcase, then just skip. if testcase_id in STORED_TESTCASES_LIST: continue # 2. Make sure that it is a unique crash testcase. Ignore duplicates, # uploaded repros. if testcase.status != 'Processed': continue # 3. Check if the testcase is fixed. If not, skip. if testcase.open: continue # 4. Check if the testcase has a minimized repro. If not, skip. if not testcase.minimized_keys or testcase.minimized_keys == 'NA': continue # 5. Only use testcases that have bugs associated with them. if not testcase.bug_information: continue # 6. Existing IPC testcases are un-interesting and unused in furthur # mutations. Due to size bloat, ignoring these for now. if testcase.absolute_path.endswith(testcase_manager.IPCDUMP_EXTENSION): continue # 7. Ignore testcases that are archives (e.g. Langfuzz fuzzer tests). if archive.get_archive_type(testcase.absolute_path): continue # 8. Skip in-process fuzzer testcases, since these are only applicable to # fuzz targets and don't run with blackbox binaries. if testcase.fuzzer_name and testcase.fuzzer_name in [ 'afl', 'libFuzzer' ]: continue # Un-pack testcase. try: _, input_directory, _ = setup.unpack_testcase(testcase) except Exception: logs.log_error('Failed to unpack testcase %d.' % testcase.key.id()) continue # Move this to our crash testcases directory. crash_testcase_directory = os.path.join(crash_testcases_directory, str(testcase_id)) shell.move(input_directory, crash_testcase_directory) # Re-create input directory for unpacking testcase in next iteration. shell.create_directory(input_directory) STORED_TESTCASES_LIST.append(testcase_id) # Remove testcase directories that exceed the max size limit. for directory_name in os.listdir(crash_testcases_directory): directory_path = os.path.join(crash_testcases_directory, directory_name) if not os.path.isdir(directory_path): continue if shell.get_directory_size( directory_path) <= MAX_TESTCASE_DIRECTORY_SIZE: continue shell.remove_directory(directory_path) # Rename all fuzzed testcase files as regular files. for root, _, files in os.walk(crash_testcases_directory): for filename in files: if not filename.startswith(testcase_manager.FUZZ_PREFIX): continue file_path = os.path.join(root, filename) stripped_file_name = os.path.basename( file_path)[len(testcase_manager.FUZZ_PREFIX):] stripped_file_path = os.path.join(os.path.dirname(file_path), stripped_file_name) try: os.rename(file_path, stripped_file_path) except: raise Exception('Failed to rename testcase %s.' % file_path) # Remove empty files and dirs to avoid the case where a fuzzer randomly # chooses an empty dir/file and generates zero testcases. shell.remove_empty_files(crash_testcases_directory) shell.remove_empty_directories(crash_testcases_directory)