def setUp(self):
"""Set up."""
super(UntrustedRunnerIntegrationTest, self).setUp()
test_helpers.patch(self, [
'datastore.data_handler.get_data_bundle_bucket_name',
])
data_types.Config().put()
environment_string = ('APP_NAME = app\n'
'RELEASE_BUILD_BUCKET_PATH = '
'gs://clusterfuzz-test-data/test_builds/'
'test-build-([0-9]+).zip\n')
data_types.Job(name='job', environment_string=environment_string).put()
environment_string = ('APP_NAME = launcher.py\n'
'RELEASE_BUILD_BUCKET_PATH = '
'gs://clusterfuzz-test-data/test_libfuzzer_builds/'
'test-libfuzzer-build-([0-9]+).zip\n'
'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = True')
data_types.Job(
name='libfuzzer_asan_job', environment_string=environment_string).put()
data_types.Fuzzer(name='fuzzer', data_bundle_name='bundle').put()
data_types.DataBundle(
name='bundle', is_local=True, sync_to_worker=True).put()
def post(self):
"""Handle a post request."""
name = self.request.get('name')
if not name:
raise helpers.EarlyExitException('Please give this corpus a name!',
400)
if not data_types.DataBundle.VALID_NAME_REGEX.match(name):
raise helpers.EarlyExitException(
'Name can only contain letters, numbers, dashes and underscores.',
400)
user_email = helpers.get_user_email()
bucket_name = data_handler.get_data_bundle_bucket_name(name)
bucket_url = data_handler.get_data_bundle_bucket_url(name)
is_local = not self.request.get('nfs', False)
if not data_handler.create_data_bundle_bucket_and_iams(
name, [user_email]):
raise helpers.EarlyExitException(
'Failed to create bucket %s.' % bucket_name, 400)
data_bundle = data_types.DataBundle.query(
data_types.DataBundle.name == name).get()
if not data_bundle:
data_bundle = data_types.DataBundle()
data_bundle.name = name
data_bundle.bucket_name = bucket_name
data_bundle.is_local = is_local
data_bundle.put()
template_values = {
'title':
'Success',
'message':
('Upload data to the corpus using: '
'gsutil -d -m rsync -r <local_corpus_directory> %s' % bucket_url),
}
self.render('message.html', template_values)
def setUp(self):
"""Set up."""
super(CorpusPruningTestUntrusted, self).setUp()
environment.set_value('JOB_NAME', 'libfuzzer_asan_job')
helpers.patch(self, [
'bot.fuzzers.engine.get', 'bot.tasks.setup.get_fuzzer_directory',
'base.tasks.add_task',
'bot.tasks.corpus_pruning_task._record_cross_pollination_stats'
])
self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine()
self.mock.get_fuzzer_directory.return_value = os.path.join(
environment.get_value('ROOT_DIR'), 'src', 'python', 'bot',
'fuzzers', 'libFuzzer')
self.corpus_bucket = os.environ['CORPUS_BUCKET']
self.quarantine_bucket = os.environ['QUARANTINE_BUCKET']
self.backup_bucket = os.environ['BACKUP_BUCKET']
job = data_types.Job(
name='libfuzzer_asan_job',
environment_string=(
'APP_NAME = test_fuzzer\n'
'CORPUS_BUCKET = {corpus_bucket}\n'
'QUARANTINE_BUCKET = {quarantine_bucket}\n'
'BACKUP_BUCKET={backup_bucket}\n'
'RELEASE_BUILD_BUCKET_PATH = '
'gs://clusterfuzz-test-data/test_libfuzzer_builds/'
'test-libfuzzer-build-([0-9]+).zip\n'
'REVISION_VARS_URL = gs://clusterfuzz-test-data/'
'test_libfuzzer_builds/'
'test-libfuzzer-build-%s.srcmap.json\n'.format(
corpus_bucket=self.corpus_bucket,
quarantine_bucket=self.quarantine_bucket,
backup_bucket=self.backup_bucket)))
job.put()
job = data_types.Job(
name='libfuzzer_asan_job2',
environment_string=(
'APP_NAME = test2_fuzzer\n'
'BACKUP_BUCKET = {backup_bucket}\n'
'CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n'.format(
backup_bucket=self.backup_bucket)))
job.put()
os.environ['PROJECT_NAME'] = 'oss-fuzz'
data_types.FuzzTarget(engine='libFuzzer',
project='test',
binary='test_fuzzer').put()
data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test_fuzzer',
engine='libFuzzer',
job='libfuzzer_asan_job',
last_run=datetime.datetime.now()).put()
data_types.FuzzTarget(engine='libFuzzer',
project='test2',
binary='fuzzer').put()
data_types.FuzzTargetJob(fuzz_target_name='libFuzzer_test2_fuzzer',
engine='libFuzzer',
job='libfuzzer_asan_job2',
last_run=datetime.datetime.now()).put()
environment.set_value('USE_MINIJAIL', True)
environment.set_value('SHARED_CORPUS_BUCKET', TEST_SHARED_BUCKET)
# Set up remote corpora.
self.corpus = corpus_manager.FuzzTargetCorpus('libFuzzer',
'test_fuzzer')
self.corpus.rsync_from_disk(os.path.join(TEST_DIR, 'corpus'),
delete=True)
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
'libFuzzer', 'test_fuzzer', quarantine=True)
self.quarantine_corpus.rsync_from_disk(os.path.join(
TEST_DIR, 'quarantine'),
delete=True)
self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET
data_types.DataBundle(name='bundle',
is_local=True,
sync_to_worker=True).put()
data_types.Fuzzer(revision=1,
file_size='builtin',
source='builtin',
name='libFuzzer',
max_testcases=4,
builtin=True,
data_bundle_name='bundle').put()
self.temp_dir = tempfile.mkdtemp()
# Copy corpus backup in the older date format.
corpus_backup_date = (
datetime.datetime.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS))
corpus_backup_dir = ('gs://{bucket}/corpus/libfuzzer/test2_fuzzer/')
gsutil.GSUtilRunner().run_gsutil([
'cp', (corpus_backup_dir +
'backup.zip').format(bucket=TEST2_BACKUP_BUCKET),
(corpus_backup_dir +
'%s.zip' % corpus_backup_date).format(bucket=self.backup_bucket)
])
def setUp(self):
"""Set up."""
super(CorpusPruningTestUntrusted, self).setUp()
environment.set_value("JOB_NAME", "libfuzzer_asan_job")
helpers.patch(
self,
[
"bot.fuzzers.engine.get",
"bot.fuzzers.libFuzzer.fuzzer.LibFuzzer.fuzzer_directory",
"base.tasks.add_task",
"datastore.data_handler.get_data_bundle_bucket_name",
],
)
self.mock.get.return_value = libFuzzer_engine.LibFuzzerEngine()
self.mock.fuzzer_directory.return_value = os.path.join(
environment.get_value("ROOT_DIR"),
"src",
"python",
"bot",
"fuzzers",
"libFuzzer",
)
self.corpus_bucket = os.environ["CORPUS_BUCKET"]
self.quarantine_bucket = os.environ["QUARANTINE_BUCKET"]
self.backup_bucket = os.environ["BACKUP_BUCKET"]
job = data_types.Job(
name="libfuzzer_asan_job",
environment_string=("APP_NAME = test_fuzzer\n"
"CORPUS_BUCKET = {corpus_bucket}\n"
"QUARANTINE_BUCKET = {quarantine_bucket}\n"
"BACKUP_BUCKET={backup_bucket}\n"
"RELEASE_BUILD_BUCKET_PATH = "
"gs://clusterfuzz-test-data/test_libfuzzer_builds/"
"test-libfuzzer-build-([0-9]+).zip\n"
"REVISION_VARS_URL = gs://clusterfuzz-test-data/"
"test_libfuzzer_builds/"
"test-libfuzzer-build-%s.srcmap.json\n".format(
corpus_bucket=self.corpus_bucket,
quarantine_bucket=self.quarantine_bucket,
backup_bucket=self.backup_bucket,
)),
)
job.put()
job = data_types.Job(
name="libfuzzer_asan_job2",
environment_string=("APP_NAME = test2_fuzzer\n"
"BACKUP_BUCKET = {backup_bucket}\n"
"CORPUS_FUZZER_NAME_OVERRIDE = libfuzzer\n".format(
backup_bucket=self.backup_bucket)),
)
job.put()
os.environ["PROJECT_NAME"] = "oss-fuzz"
data_types.FuzzTarget(
engine="libFuzzer", project="test", binary="test_fuzzer").put()
data_types.FuzzTargetJob(
fuzz_target_name="libFuzzer_test_fuzzer",
engine="libFuzzer",
job="libfuzzer_asan_job",
last_run=datetime.datetime.now(),
).put()
data_types.FuzzTarget(
engine="libFuzzer", project="test2", binary="fuzzer").put()
data_types.FuzzTargetJob(
fuzz_target_name="libFuzzer_test2_fuzzer",
engine="libFuzzer",
job="libfuzzer_asan_job2",
last_run=datetime.datetime.now(),
).put()
environment.set_value("USE_MINIJAIL", True)
environment.set_value("SHARED_CORPUS_BUCKET", TEST_SHARED_BUCKET)
# Set up remote corpora.
self.corpus = corpus_manager.FuzzTargetCorpus("libFuzzer", "test_fuzzer")
self.corpus.rsync_from_disk(os.path.join(TEST_DIR, "corpus"), delete=True)
self.quarantine_corpus = corpus_manager.FuzzTargetCorpus(
"libFuzzer", "test_fuzzer", quarantine=True)
self.quarantine_corpus.rsync_from_disk(
os.path.join(TEST_DIR, "quarantine"), delete=True)
self.mock.get_data_bundle_bucket_name.return_value = TEST_GLOBAL_BUCKET
data_types.DataBundle(
name="bundle", is_local=True, sync_to_worker=True).put()
data_types.Fuzzer(
revision=1,
file_size="builtin",
source="builtin",
name="libFuzzer",
max_testcases=4,
builtin=True,
data_bundle_name="bundle",
).put()
self.temp_dir = tempfile.mkdtemp()
# Copy corpus backup in the older date format.
corpus_backup_date = datetime.datetime.utcnow().date() - datetime.timedelta(
days=data_types.CORPUS_BACKUP_PUBLIC_LOOKBACK_DAYS)
corpus_backup_dir = "gs://{bucket}/corpus/libfuzzer/test2_fuzzer/"
gsutil.GSUtilRunner().run_gsutil([
"cp",
(corpus_backup_dir + "backup.zip").format(bucket=TEST2_BACKUP_BUCKET),
(corpus_backup_dir +
"%s.zip" % corpus_backup_date).format(bucket=self.backup_bucket),
])
def setUp(self):
helpers.patch_environ(self)
project_config_get = local_config.ProjectConfig.get
helpers.patch(self, [
'base.utils.default_project_name',
'config.db_config.get',
('project_config_get', 'config.local_config.ProjectConfig.get'),
])
self.job = data_types.Job(
name='linux_asan_chrome',
environment_string=('SUMMARY_PREFIX = project\n'
'PROJECT_NAME = project\n'
'HELP_URL = help_url\n'))
self.job2 = data_types.Job(
name='windows_asan_chrome',
environment_string=('SUMMARY_PREFIX = project\n'
'PROJECT_NAME = project\n'
'HELP_URL = help_url\n'))
self.testcase = data_types.Testcase(
job_type='linux_asan_chrome',
fuzzer_name='libfuzzer_binary_name',
crash_type='Crash-type',
crash_address='0x1337',
crash_state='A\nB\nC\n')
self.testcase.set_metadata('fuzzer_binary_name',
'binary_name',
update_testcase=False)
self.testcase_assert = data_types.Testcase(
job_type='linux_asan_chrome',
fuzzer_name='libfuzzer_binary_name',
crash_type='ASSERT',
crash_address='0x1337',
crash_state='foo != bar\nB\nC\n')
self.testcase_assert.set_metadata('fuzzer_binary_name',
'binary_name',
update_testcase=False)
self.testcase_null = data_types.Testcase(
job_type='linux_asan_chrome',
fuzzer_name='libfuzzer_binary_name',
crash_type='UNKNOWN',
crash_address='0x1337',
crash_state='NULL')
self.testcase_bad_cast = data_types.Testcase(
job_type='linux_asan_chrome',
fuzzer_name='libfuzzer_binary_name',
crash_type='Bad-cast',
crash_address='0x1337',
crash_state=
('Bad-cast to blink::LayoutBlock from blink::LayoutTableSection\n'
'blink::LayoutObject::ContainerForFixedPosition\n'
'blink::LayoutObject::Container\n'))
self.testcase_bad_cast_without_crash_function = data_types.Testcase(
job_type='linux_asan_chrome',
fuzzer_name='libfuzzer_binary_name',
crash_type='Bad-cast',
crash_address='0x1337',
crash_state=
('Bad-cast to blink::LayoutBlock from blink::LayoutTableSection\n'
))
self.local_data_bundle = data_types.DataBundle(
name='local_data_bundle')
self.cloud_data_bundle = data_types.DataBundle(
name='cloud_data_bundle')
self.fuzzer1 = data_types.Fuzzer(name='fuzzer1',
data_bundle_name=None,
jobs=['linux_asan_chrome'])
self.fuzzer2 = data_types.Fuzzer(name='fuzzer2',
data_bundle_name='local_data_bundle',
jobs=['linux_asan_chrome'])
self.fuzzer3 = data_types.Fuzzer(name='fuzzer3',
data_bundle_name='cloud_data_bundle',
jobs=['linux_asan_chrome'])
entities_to_put = [
self.testcase, self.testcase_assert, self.testcase_null,
self.testcase_bad_cast,
self.testcase_bad_cast_without_crash_function, self.job, self.job2,
self.local_data_bundle, self.cloud_data_bundle, self.fuzzer1,
self.fuzzer2, self.fuzzer3
]
for entity in entities_to_put:
entity.put()
environment.set_value('FUZZ_DATA',
'/tmp/inputs/fuzzer-common-data-bundles')
environment.set_value('FUZZERS_DIR', '/tmp/inputs/fuzzers')
self.mock.default_project_name.return_value = 'project'
self.mock.project_config_get.side_effect = project_config_get
