def test_platform_restriction(self, local_development): """Ensure that we can find a task with a valid platform.""" os.environ['LOCAL_DEVELOPMENT'] = local_development windows_mapping = data_types.FuzzerJob() windows_mapping.fuzzer = 'wrong_fuzzer' windows_mapping.job = 'job_1' windows_mapping.platform = 'windows' windows_mapping.put() data_types.FuzzerJobs(platform='windows', fuzzer_jobs=[windows_mapping]).put() self.assertEqual( (None, None), fuzzer_selection.get_fuzz_task_payload(platform='linux')) linux_mapping = data_types.FuzzerJob() linux_mapping.fuzzer = 'right_fuzzer' linux_mapping.job = 'job_2' linux_mapping.platform = 'linux' linux_mapping.put() data_types.FuzzerJobs(platform='linux', fuzzer_jobs=[linux_mapping]).put() argument, job = fuzzer_selection.get_fuzz_task_payload('linux') self.assertEqual(('right_fuzzer', 'job_2'), (argument, job))
def batch_fuzzer_jobs(): """Batch FuzzerJobs for reduced Datastore read ops by bots.""" platforms = [ item.platform for item in data_types.FuzzerJob.query( projection=[data_types.FuzzerJob.platform], distinct=True) ] for platform in platforms: fuzzer_jobs = list( data_types.FuzzerJob.query( data_types.FuzzerJob.platform == platform)) fuzzer_jobs.sort(key=lambda item: item.job) batches_to_remove = set(b.key for b in data_types.FuzzerJobs.query( data_types.FuzzerJobs.platform == platform)) batch_count = 0 for i in range(0, len(fuzzer_jobs), FUZZER_JOB_BATCH_SIZE): key_id = platform + '-' + str(batch_count) end = min(i + FUZZER_JOB_BATCH_SIZE, len(fuzzer_jobs)) batched = data_types.FuzzerJobs(id=key_id, platform=platform) batched.platform = platform batched.fuzzer_jobs = fuzzer_jobs[i:end] batched.put() batch_count += 1 batches_to_remove.discard(batched.key) # Remove additional batches if number reduced. if batches_to_remove: ndb.delete_multi(batches_to_remove)
def setUp(self): self.total_fuzzer_jobs = 7000 self.platforms = ['LINUX', 'WINDOWS'] fuzzer_jobs = [] for platform in self.platforms: for i in range(self.total_fuzzer_jobs): fuzzer_job = data_types.FuzzerJob( fuzzer='libFuzzer', job='libfuzzer_asan_{}_{:06d}'.format(platform, i), platform=platform) fuzzer_jobs.append(fuzzer_job) ndb.put_multi(fuzzer_jobs) # Should be removed. data_types.FuzzerJobs(id='LINUX-2', platform='LINUX').put() # Should be overwritten and not removed. data_types.FuzzerJobs(id='LINUX-0', platform='LINUX').put()