def setUp(self):
helpers.patch_environ(self)
helpers.patch(self, [
'clusterfuzz._internal.base.persistent_cache.get_value',
'clusterfuzz._internal.base.persistent_cache.set_value',
'clusterfuzz._internal.base.utils.utcnow',
'time.sleep',
])
self.mock.get_value.return_value = None
self.mock.sleep.return_value = None
data_types.Job(name='job').put()
client = pubsub.PubSubClient()
topic = pubsub.topic_name('test-clusterfuzz', 'jobs-linux')
client.create_topic(topic)
client.create_subscription(
pubsub.subscription_name('test-clusterfuzz', 'jobs-linux'), topic)
topic = pubsub.topic_name('test-clusterfuzz', 'high-end-jobs-linux')
client.create_topic(topic)
client.create_subscription(
pubsub.subscription_name('test-clusterfuzz',
'high-end-jobs-linux'), topic)
self.mock.utcnow.return_value = test_utils.CURRENT_TIME.replace(
microsecond=0)
def cleanup_pubsub_topics(project_names):
"""Delete old pubsub topics and subscriptions."""
client = pubsub.PubSubClient()
application_id = utils.get_application_id()
expected_topics = set()
for platform in PUBSUB_PLATFORMS:
expected_topics.update([
untrusted.queue_name(project, platform)
for project in project_names
])
pubsub_config = local_config.Config('pubsub.queues')
unmanaged_queues = [
queue['name'] for queue in pubsub_config.get('resources')
]
for topic in client.list_topics(pubsub.project_name(application_id)):
_, name = pubsub.parse_name(topic)
if (not name.startswith(tasks.JOBS_PREFIX)
and not name.startswith(tasks.HIGH_END_JOBS_PREFIX)):
# Some topic created by another service, ignore.
continue
if name in unmanaged_queues:
continue
if name in expected_topics:
continue
for subscription in client.list_topic_subscriptions(topic):
client.delete_subscription(subscription)
client.delete_topic(topic)
def get_regular_task(queue=None):
"""Get a regular task."""
if not queue:
queue = regular_queue()
pubsub_client = pubsub.PubSubClient()
application_id = utils.get_application_id()
while True:
messages = pubsub_client.pull_from_subscription(
pubsub.subscription_name(application_id, queue), max_messages=1)
if not messages:
return None
try:
task = PubSubTask(messages[0])
except KeyError:
logs.log_error('Received an invalid task, discarding...')
messages[0].ack()
continue
# Check that this task should be run now (past the ETA). Otherwise we defer
# its execution.
if not task.defer():
return task
def notify_issue_update(testcase, status):
"""Notify that an issue update occurred (i.e. issue was filed or closed)."""
topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic')
if not topic:
return
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(
topic, [
pubsub.Message(
attributes={
'crash_address': testcase.crash_address,
'crash_state': testcase.crash_state,
'crash_type': testcase.crash_type,
'issue_id': testcase.bug_information or '',
'security': str(testcase.security_flag).lower(),
'status': status,
'testcase_id': str(testcase.key.id()),
})
])
if status in ('verified', 'wontfix'):
logs.log(f'Closing issue {testcase.github_issue_num} '
f'in GitHub repo {testcase.github_repo_id}: '
f'Testcase {testcase.key.id()} is marked as {status}.')
oss_fuzz_github.close_issue(testcase)
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
os.environ['BOT_TMPDIR'] = os.path.join(self.tmp_dir, 'bot_tmpdir')
test_helpers.patch(self, [
'clusterfuzz._internal.datastore.data_handler.'
'get_data_bundle_bucket_name',
'clusterfuzz._internal.system.environment.'
'set_environment_parameters_from_file',
])
test_helpers.patch_environ(self)
# Our tests write data/logs into subdirs of ROOT_DIR. Pivot the ROOT_DIR to
# a temporary one.
new_root = _create_test_root()
os.environ['ROOT_DIR'] = new_root
self.saved_cwd = os.getcwd()
os.chdir(new_root)
environment.set_bot_environment()
fuzz_inputs = os.environ['FUZZ_INPUTS']
shell.remove_directory(fuzz_inputs, recreate=True)
worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs)
shell.remove_directory(worker_fuzz_inputs, recreate=True)
environment.set_value('GSUTIL_PATH', os.path.dirname(_which('gsutil')))
test_utils.setup_pubsub('test-clusterfuzz')
test_utils.create_pubsub_topic(pubsub.PubSubClient(),
'test-clusterfuzz',
'jobs-project-linux')
def add_task(command, argument, job_type, queue=None, wait_time=None):
"""Add a new task to the job queue."""
# Old testcases may pass in queue=None explicitly,
# so we must check this here.
if not queue:
queue = default_queue()
if wait_time is None:
wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL)
if job_type != 'none':
job = data_types.Job.query(data_types.Job.name == job_type).get()
if not job:
raise Error(f'Job {job_type} not found.')
if job.is_external():
external_tasks.add_external_task(command, argument, job)
return
# Add the task.
eta = utils.utcnow() + datetime.timedelta(seconds=wait_time)
task = Task(command, argument, job_type, eta=eta)
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue),
[task.to_pubsub_message()])
def _make_bisection_request(pubsub_topic, testcase, target, bisect_type):
"""Make a bisection request to the external bisection service. Returns whether
or not a request was actually made."""
if bisect_type == 'fixed':
old_commit, new_commit = _get_commits(testcase.fixed,
testcase.job_type)
elif bisect_type == 'regressed':
old_commit, new_commit = _get_commits(testcase.regression,
testcase.job_type)
else:
raise ValueError('Invalid bisection type: ' + bisect_type)
if not new_commit:
# old_commit can be empty (i.e. '0' case), but new_commit should never be.
return False
old_commit, new_commit = _check_commits(testcase, bisect_type, old_commit,
new_commit)
repo_url = data_handler.get_main_repo(testcase.job_type) or ''
reproducer = blobs.read_key(testcase.minimized_keys
or testcase.fuzzed_keys)
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(pubsub_topic, [
pubsub.Message(
reproducer, {
'type':
bisect_type,
'project_name':
target.project,
'sanitizer':
environment.SANITIZER_NAME_MAP[
environment.get_memory_tool_name(testcase.job_type)],
'fuzz_target':
target.binary,
'old_commit':
old_commit,
'new_commit':
new_commit,
'testcase_id':
str(testcase.key.id()),
'issue_id':
testcase.bug_information,
'crash_type':
testcase.crash_type,
'crash_state':
testcase.crash_state,
'security':
str(testcase.security_flag),
'severity':
severity_analyzer.severity_to_string(
testcase.security_severity),
'timestamp':
testcase.timestamp.isoformat(),
'repo_url':
repo_url,
})
])
return True
def setup_pubsub(project):
"""Set up pubsub topics and subscriptions."""
config = local_config.Config('pubsub.queues')
client = pubsub.PubSubClient()
queues = config.get('resources')
for queue in queues:
create_pubsub_topic(client, project, queue['name'])
create_pubsub_subscription(client, project, queue['name'],
queue['name'])
def setUp(self):
helpers.patch_environ(self)
self.topic = pubsub.topic_name(PROJECT_NAME, 'test-topic')
self.subscription = pubsub.subscription_name(PROJECT_NAME,
'subscription')
self.client = pubsub.PubSubClient()
self.client.create_topic(self.topic)
self.client.create_subscription(self.subscription,
self.topic,
ack_deadline=ACK_DEADLINE)
def setUp(self):
helpers.patch_environ(self)
helpers.patch(self, [
'clusterfuzz._internal.google_cloud_utils.blobs.read_key',
])
self.mock.read_key.return_value = b'reproducer'
self.client = pubsub.PubSubClient()
self.topic = pubsub.topic_name('proj', 'repro')
self.client.create_topic(self.topic)
self.subscription = pubsub.subscription_name('proj', 'repro')
self.client.create_subscription(self.subscription, self.topic)
data_types.Job(
name='libfuzzer_asan_blah_external',
platform='LINUX',
environment_string=
('RELEASE_BUILD_BUCKET_PATH = gs://bucket/a/b/release-([0-9]+).zip\n'
'PROJECT_NAME = proj\n'),
external_reproduction_topic=self.topic,
external_updates_subscription='projects/proj/subscriptions/updates'
).put()
data_types.Job(
name='libfuzzer_msan_blah_external',
platform='LINUX',
environment_string=(
'FUZZ_TARGET_BUILD_BUCKET_PATH = '
'gs://bucket/a/b/%TARGET%/release-([0-9]+).zip\n'
'PROJECT_NAME = proj\n'),
external_reproduction_topic=self.topic,
external_updates_subscription='projects/proj/subscriptions/updates'
).put()
data_types.FuzzTarget(id='libFuzzer_abc',
engine='libFuzzer',
binary='abc').put()
self.testcase_0 = data_types.Testcase(
fuzzer_name='libFuzzer',
overridden_fuzzer_name='libFuzzer_abc',
crash_revision=1336,
minimized_keys='key')
self.testcase_0.set_metadata('last_tested_revision', 1337)
self.testcase_0.put()
self.testcase_1 = data_types.Testcase(
fuzzer_name='libFuzzer',
overridden_fuzzer_name='libFuzzer_abc',
crash_revision=1336,
fuzzed_keys='key')
self.testcase_1.put()
def create_pubsub_topics(project):
"""Create pubsub topics for tasks."""
for platform in PUBSUB_PLATFORMS:
name = untrusted.queue_name(project, platform)
client = pubsub.PubSubClient()
application_id = utils.get_application_id()
topic_name = pubsub.topic_name(application_id, name)
if client.get_topic(topic_name) is None:
client.create_topic(topic_name)
subscription_name = pubsub.subscription_name(application_id, name)
if client.get_subscription(subscription_name) is None:
client.create_subscription(subscription_name, topic_name)
def notify_bisection_invalid(testcase):
"""Notify the bisection infrastructure of a testcase getting into invalid
state."""
pubsub_topic = _get_topic()
if not pubsub_topic:
return
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(pubsub_topic, [
pubsub.Message(b'', {
'type': 'invalid',
'testcase_id': str(testcase.key.id()),
})
])
def add_external_task(command, testcase_id, job):
"""Add external task."""
if command != 'progression':
# Only progression is supported.
return
pubsub_client = pubsub.PubSubClient()
topic_name = job.external_reproduction_topic
assert topic_name is not None
testcase = data_handler.get_testcase_by_id(testcase_id)
fuzz_target = testcase.get_fuzz_target()
memory_tool_name = environment.get_memory_tool_name(job.name)
sanitizer = environment.SANITIZER_NAME_MAP.get(memory_tool_name)
job_environment = job.get_environment()
if job_environment.get('CUSTOM_BINARY'):
raise RuntimeError('External jobs should never have custom binaries.')
build_path = (
job_environment.get('RELEASE_BUILD_BUCKET_PATH') or
job_environment.get('FUZZ_TARGET_BUILD_BUCKET_PATH'))
if build_path is None:
raise RuntimeError(f'{job.name} has no build path defined.')
min_revision = (
testcase.get_metadata('last_tested_revision') or testcase.crash_revision)
logs.log(f'Publishing external reproduction task for {testcase_id}.')
attributes = {
'project': job.project,
'target': fuzz_target.binary,
'fuzzer': testcase.fuzzer_name,
'sanitizer': sanitizer,
'job': job.name,
'testcaseId': str(testcase_id),
'buildPath': build_path,
'minRevisionAbove': str(min_revision),
'numTrials': str(_NUM_TRIALS),
}
reproducer = blobs.read_key(testcase.fuzzed_keys)
message = pubsub.Message(data=reproducer, attributes=attributes)
pubsub_client.publish(topic_name, [message])
def notify_issue_update(testcase, status):
"""Notify that an issue update occurred (i.e. issue was filed or closed)."""
topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic')
if not topic:
return
pubsub_client = pubsub.PubSubClient()
pubsub_client.publish(topic, [
pubsub.Message(
attributes={
'crash_address': testcase.crash_address,
'crash_state': testcase.crash_state,
'crash_type': testcase.crash_type,
'issue_id': testcase.bug_information or '',
'security': str(testcase.security_flag).lower(),
'status': status,
'testcase_id': str(testcase.key.id()),
})
])
def execute_task(testcase_id, _):
"""Attempt to find the CL introducing the bug associated with testcase_id."""
# Locate the testcase associated with the id.
testcase = data_handler.get_testcase_by_id(testcase_id)
if not testcase:
return
# Make sure that predator topic is configured. If not, nothing to do here.
topic = db_config.get_value('predator_crash_topic')
if not topic:
logs.log('Predator is not configured, skipping blame task.')
return
data_handler.update_testcase_comment(testcase,
data_types.TaskState.STARTED)
# Prepare pubsub message to send to predator.
message = _prepare_predator_message(testcase)
if not message:
testcase = data_handler.get_testcase_by_id(testcase_id)
data_handler.update_testcase_comment(
testcase, data_types.TaskState.ERROR,
'Failed to generate request for Predator')
return
# Clear existing results and mark blame result as pending.
testcase = data_handler.get_testcase_by_id(testcase_id)
_clear_blame_result_and_set_pending_flag(testcase)
# Post request to pub sub.
client = pubsub.PubSubClient()
message_ids = client.publish(topic, [message])
logs.log(
'Successfully published testcase %s to Predator. Message IDs: %s.' %
(testcase_id, message_ids))
data_handler.update_testcase_comment(testcase,
data_types.TaskState.FINISHED)
def get(self):
"""Process a GET request."""
subscription = db_config.get_value('predator_result_topic')
if not subscription:
logs.log('No Predator subscription configured. Aborting.')
return
client = pubsub.PubSubClient()
messages = client.pull_from_subscription(subscription, acknowledge=True)
for message in messages:
message = json.loads(message.data)
testcase_id = message['crash_identifiers']
try:
testcase = data_handler.get_testcase_by_id(testcase_id)
except errors.InvalidTestcaseError:
logs.log('Testcase %s no longer exists.' % str(testcase_id))
continue
testcase.set_metadata('predator_result', message, update_testcase=False)
testcase.delete_metadata('blame_pending', update_testcase=False)
testcase.put()
logs.log('Set predator result for testcase %d.' % testcase.key.id())
logs.log('Finished processing predator results. %d total.' % len(messages))
def setUp(self):
helpers.patch(self, [
'clusterfuzz._internal.config.local_config.ProjectConfig',
])
self.topic = 'projects/project/topics/issue-updates'
self.subscription = 'projects/project/subscriptions/issue-updates'
self.mock.ProjectConfig.return_value = mock_config.MockConfig({
'issue_updates': {
'pubsub_topic': self.topic,
},
})
self.pubsub_client = pubsub.PubSubClient()
self.pubsub_client.create_topic(self.topic)
self.pubsub_client.create_subscription(self.subscription, self.topic)
self.testcase = data_types.Testcase(crash_address='0xffff',
security_flag=True,
crash_type='CRASH TYPE',
crash_state='CRASH STATE',
bug_information='123')
self.testcase.put()
