def test_platform_restriction(self, local_development):
"""Ensure that we can find a task with a valid platform."""
os.environ['LOCAL_DEVELOPMENT'] = local_development
windows_mapping = data_types.FuzzerJob()
windows_mapping.fuzzer = 'wrong_fuzzer'
windows_mapping.job = 'job_1'
windows_mapping.platform = 'windows'
windows_mapping.put()
data_types.FuzzerJobs(platform='windows',
fuzzer_jobs=[windows_mapping]).put()
self.assertEqual(
(None, None),
fuzzer_selection.get_fuzz_task_payload(platform='linux'))
linux_mapping = data_types.FuzzerJob()
linux_mapping.fuzzer = 'right_fuzzer'
linux_mapping.job = 'job_2'
linux_mapping.platform = 'linux'
linux_mapping.put()
data_types.FuzzerJobs(platform='linux',
fuzzer_jobs=[linux_mapping]).put()
argument, job = fuzzer_selection.get_fuzz_task_payload('linux')
self.assertEqual(('right_fuzzer', 'job_2'), (argument, job))
def batch_fuzzer_jobs():
"""Batch FuzzerJobs for reduced Datastore read ops by bots."""
platforms = [
item.platform for item in data_types.FuzzerJob.query(
projection=[data_types.FuzzerJob.platform], distinct=True)
]
for platform in platforms:
fuzzer_jobs = list(
data_types.FuzzerJob.query(
data_types.FuzzerJob.platform == platform))
fuzzer_jobs.sort(key=lambda item: item.job)
batches_to_remove = set(b.key for b in data_types.FuzzerJobs.query(
data_types.FuzzerJobs.platform == platform))
batch_count = 0
for i in range(0, len(fuzzer_jobs), FUZZER_JOB_BATCH_SIZE):
key_id = platform + '-' + str(batch_count)
end = min(i + FUZZER_JOB_BATCH_SIZE, len(fuzzer_jobs))
batched = data_types.FuzzerJobs(id=key_id, platform=platform)
batched.platform = platform
batched.fuzzer_jobs = fuzzer_jobs[i:end]
batched.put()
batch_count += 1
batches_to_remove.discard(batched.key)
# Remove additional batches if number reduced.
if batches_to_remove:
ndb.delete_multi(batches_to_remove)
def setUp(self):
self.total_fuzzer_jobs = 7000
self.platforms = ['LINUX', 'WINDOWS']
fuzzer_jobs = []
for platform in self.platforms:
for i in range(self.total_fuzzer_jobs):
fuzzer_job = data_types.FuzzerJob(
fuzzer='libFuzzer',
job='libfuzzer_asan_{}_{:06d}'.format(platform, i),
platform=platform)
fuzzer_jobs.append(fuzzer_job)
ndb.put_multi(fuzzer_jobs)
# Should be removed.
data_types.FuzzerJobs(id='LINUX-2', platform='LINUX').put()
# Should be overwritten and not removed.
data_types.FuzzerJobs(id='LINUX-0', platform='LINUX').put()