def setUp(self): helpers.patch_environ(self) helpers.patch(self, [ 'clusterfuzz._internal.base.persistent_cache.get_value', 'clusterfuzz._internal.base.persistent_cache.set_value', 'clusterfuzz._internal.base.utils.utcnow', 'time.sleep', ]) self.mock.get_value.return_value = None self.mock.sleep.return_value = None data_types.Job(name='job').put() client = pubsub.PubSubClient() topic = pubsub.topic_name('test-clusterfuzz', 'jobs-linux') client.create_topic(topic) client.create_subscription( pubsub.subscription_name('test-clusterfuzz', 'jobs-linux'), topic) topic = pubsub.topic_name('test-clusterfuzz', 'high-end-jobs-linux') client.create_topic(topic) client.create_subscription( pubsub.subscription_name('test-clusterfuzz', 'high-end-jobs-linux'), topic) self.mock.utcnow.return_value = test_utils.CURRENT_TIME.replace( microsecond=0)
def cleanup_pubsub_topics(project_names): """Delete old pubsub topics and subscriptions.""" client = pubsub.PubSubClient() application_id = utils.get_application_id() expected_topics = set() for platform in PUBSUB_PLATFORMS: expected_topics.update([ untrusted.queue_name(project, platform) for project in project_names ]) pubsub_config = local_config.Config('pubsub.queues') unmanaged_queues = [ queue['name'] for queue in pubsub_config.get('resources') ] for topic in client.list_topics(pubsub.project_name(application_id)): _, name = pubsub.parse_name(topic) if (not name.startswith(tasks.JOBS_PREFIX) and not name.startswith(tasks.HIGH_END_JOBS_PREFIX)): # Some topic created by another service, ignore. continue if name in unmanaged_queues: continue if name in expected_topics: continue for subscription in client.list_topic_subscriptions(topic): client.delete_subscription(subscription) client.delete_topic(topic)
def get_regular_task(queue=None): """Get a regular task.""" if not queue: queue = regular_queue() pubsub_client = pubsub.PubSubClient() application_id = utils.get_application_id() while True: messages = pubsub_client.pull_from_subscription( pubsub.subscription_name(application_id, queue), max_messages=1) if not messages: return None try: task = PubSubTask(messages[0]) except KeyError: logs.log_error('Received an invalid task, discarding...') messages[0].ack() continue # Check that this task should be run now (past the ETA). Otherwise we defer # its execution. if not task.defer(): return task
def notify_issue_update(testcase, status): """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish( topic, [ pubsub.Message( attributes={ 'crash_address': testcase.crash_address, 'crash_state': testcase.crash_state, 'crash_type': testcase.crash_type, 'issue_id': testcase.bug_information or '', 'security': str(testcase.security_flag).lower(), 'status': status, 'testcase_id': str(testcase.key.id()), }) ]) if status in ('verified', 'wontfix'): logs.log(f'Closing issue {testcase.github_issue_num} ' f'in GitHub repo {testcase.github_repo_id}: ' f'Testcase {testcase.key.id()} is marked as {status}.') oss_fuzz_github.close_issue(testcase)
def setUp(self): self.tmp_dir = tempfile.mkdtemp() os.environ['BOT_TMPDIR'] = os.path.join(self.tmp_dir, 'bot_tmpdir') test_helpers.patch(self, [ 'clusterfuzz._internal.datastore.data_handler.' 'get_data_bundle_bucket_name', 'clusterfuzz._internal.system.environment.' 'set_environment_parameters_from_file', ]) test_helpers.patch_environ(self) # Our tests write data/logs into subdirs of ROOT_DIR. Pivot the ROOT_DIR to # a temporary one. new_root = _create_test_root() os.environ['ROOT_DIR'] = new_root self.saved_cwd = os.getcwd() os.chdir(new_root) environment.set_bot_environment() fuzz_inputs = os.environ['FUZZ_INPUTS'] shell.remove_directory(fuzz_inputs, recreate=True) worker_fuzz_inputs = file_host.rebase_to_worker_root(fuzz_inputs) shell.remove_directory(worker_fuzz_inputs, recreate=True) environment.set_value('GSUTIL_PATH', os.path.dirname(_which('gsutil'))) test_utils.setup_pubsub('test-clusterfuzz') test_utils.create_pubsub_topic(pubsub.PubSubClient(), 'test-clusterfuzz', 'jobs-project-linux')
def add_task(command, argument, job_type, queue=None, wait_time=None): """Add a new task to the job queue.""" # Old testcases may pass in queue=None explicitly, # so we must check this here. if not queue: queue = default_queue() if wait_time is None: wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL) if job_type != 'none': job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: raise Error(f'Job {job_type} not found.') if job.is_external(): external_tasks.add_external_task(command, argument, job) return # Add the task. eta = utils.utcnow() + datetime.timedelta(seconds=wait_time) task = Task(command, argument, job_type, eta=eta) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue), [task.to_pubsub_message()])
def _make_bisection_request(pubsub_topic, testcase, target, bisect_type): """Make a bisection request to the external bisection service. Returns whether or not a request was actually made.""" if bisect_type == 'fixed': old_commit, new_commit = _get_commits(testcase.fixed, testcase.job_type) elif bisect_type == 'regressed': old_commit, new_commit = _get_commits(testcase.regression, testcase.job_type) else: raise ValueError('Invalid bisection type: ' + bisect_type) if not new_commit: # old_commit can be empty (i.e. '0' case), but new_commit should never be. return False old_commit, new_commit = _check_commits(testcase, bisect_type, old_commit, new_commit) repo_url = data_handler.get_main_repo(testcase.job_type) or '' reproducer = blobs.read_key(testcase.minimized_keys or testcase.fuzzed_keys) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub_topic, [ pubsub.Message( reproducer, { 'type': bisect_type, 'project_name': target.project, 'sanitizer': environment.SANITIZER_NAME_MAP[ environment.get_memory_tool_name(testcase.job_type)], 'fuzz_target': target.binary, 'old_commit': old_commit, 'new_commit': new_commit, 'testcase_id': str(testcase.key.id()), 'issue_id': testcase.bug_information, 'crash_type': testcase.crash_type, 'crash_state': testcase.crash_state, 'security': str(testcase.security_flag), 'severity': severity_analyzer.severity_to_string( testcase.security_severity), 'timestamp': testcase.timestamp.isoformat(), 'repo_url': repo_url, }) ]) return True
def setup_pubsub(project): """Set up pubsub topics and subscriptions.""" config = local_config.Config('pubsub.queues') client = pubsub.PubSubClient() queues = config.get('resources') for queue in queues: create_pubsub_topic(client, project, queue['name']) create_pubsub_subscription(client, project, queue['name'], queue['name'])
def setUp(self): helpers.patch_environ(self) self.topic = pubsub.topic_name(PROJECT_NAME, 'test-topic') self.subscription = pubsub.subscription_name(PROJECT_NAME, 'subscription') self.client = pubsub.PubSubClient() self.client.create_topic(self.topic) self.client.create_subscription(self.subscription, self.topic, ack_deadline=ACK_DEADLINE)
def setUp(self): helpers.patch_environ(self) helpers.patch(self, [ 'clusterfuzz._internal.google_cloud_utils.blobs.read_key', ]) self.mock.read_key.return_value = b'reproducer' self.client = pubsub.PubSubClient() self.topic = pubsub.topic_name('proj', 'repro') self.client.create_topic(self.topic) self.subscription = pubsub.subscription_name('proj', 'repro') self.client.create_subscription(self.subscription, self.topic) data_types.Job( name='libfuzzer_asan_blah_external', platform='LINUX', environment_string= ('RELEASE_BUILD_BUCKET_PATH = gs://bucket/a/b/release-([0-9]+).zip\n' 'PROJECT_NAME = proj\n'), external_reproduction_topic=self.topic, external_updates_subscription='projects/proj/subscriptions/updates' ).put() data_types.Job( name='libfuzzer_msan_blah_external', platform='LINUX', environment_string=( 'FUZZ_TARGET_BUILD_BUCKET_PATH = ' 'gs://bucket/a/b/%TARGET%/release-([0-9]+).zip\n' 'PROJECT_NAME = proj\n'), external_reproduction_topic=self.topic, external_updates_subscription='projects/proj/subscriptions/updates' ).put() data_types.FuzzTarget(id='libFuzzer_abc', engine='libFuzzer', binary='abc').put() self.testcase_0 = data_types.Testcase( fuzzer_name='libFuzzer', overridden_fuzzer_name='libFuzzer_abc', crash_revision=1336, minimized_keys='key') self.testcase_0.set_metadata('last_tested_revision', 1337) self.testcase_0.put() self.testcase_1 = data_types.Testcase( fuzzer_name='libFuzzer', overridden_fuzzer_name='libFuzzer_abc', crash_revision=1336, fuzzed_keys='key') self.testcase_1.put()
def create_pubsub_topics(project): """Create pubsub topics for tasks.""" for platform in PUBSUB_PLATFORMS: name = untrusted.queue_name(project, platform) client = pubsub.PubSubClient() application_id = utils.get_application_id() topic_name = pubsub.topic_name(application_id, name) if client.get_topic(topic_name) is None: client.create_topic(topic_name) subscription_name = pubsub.subscription_name(application_id, name) if client.get_subscription(subscription_name) is None: client.create_subscription(subscription_name, topic_name)
def notify_bisection_invalid(testcase): """Notify the bisection infrastructure of a testcase getting into invalid state.""" pubsub_topic = _get_topic() if not pubsub_topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub_topic, [ pubsub.Message(b'', { 'type': 'invalid', 'testcase_id': str(testcase.key.id()), }) ])
def add_external_task(command, testcase_id, job): """Add external task.""" if command != 'progression': # Only progression is supported. return pubsub_client = pubsub.PubSubClient() topic_name = job.external_reproduction_topic assert topic_name is not None testcase = data_handler.get_testcase_by_id(testcase_id) fuzz_target = testcase.get_fuzz_target() memory_tool_name = environment.get_memory_tool_name(job.name) sanitizer = environment.SANITIZER_NAME_MAP.get(memory_tool_name) job_environment = job.get_environment() if job_environment.get('CUSTOM_BINARY'): raise RuntimeError('External jobs should never have custom binaries.') build_path = ( job_environment.get('RELEASE_BUILD_BUCKET_PATH') or job_environment.get('FUZZ_TARGET_BUILD_BUCKET_PATH')) if build_path is None: raise RuntimeError(f'{job.name} has no build path defined.') min_revision = ( testcase.get_metadata('last_tested_revision') or testcase.crash_revision) logs.log(f'Publishing external reproduction task for {testcase_id}.') attributes = { 'project': job.project, 'target': fuzz_target.binary, 'fuzzer': testcase.fuzzer_name, 'sanitizer': sanitizer, 'job': job.name, 'testcaseId': str(testcase_id), 'buildPath': build_path, 'minRevisionAbove': str(min_revision), 'numTrials': str(_NUM_TRIALS), } reproducer = blobs.read_key(testcase.fuzzed_keys) message = pubsub.Message(data=reproducer, attributes=attributes) pubsub_client.publish(topic_name, [message])
def notify_issue_update(testcase, status): """Notify that an issue update occurred (i.e. issue was filed or closed).""" topic = local_config.ProjectConfig().get('issue_updates.pubsub_topic') if not topic: return pubsub_client = pubsub.PubSubClient() pubsub_client.publish(topic, [ pubsub.Message( attributes={ 'crash_address': testcase.crash_address, 'crash_state': testcase.crash_state, 'crash_type': testcase.crash_type, 'issue_id': testcase.bug_information or '', 'security': str(testcase.security_flag).lower(), 'status': status, 'testcase_id': str(testcase.key.id()), }) ])
def execute_task(testcase_id, _): """Attempt to find the CL introducing the bug associated with testcase_id.""" # Locate the testcase associated with the id. testcase = data_handler.get_testcase_by_id(testcase_id) if not testcase: return # Make sure that predator topic is configured. If not, nothing to do here. topic = db_config.get_value('predator_crash_topic') if not topic: logs.log('Predator is not configured, skipping blame task.') return data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) # Prepare pubsub message to send to predator. message = _prepare_predator_message(testcase) if not message: testcase = data_handler.get_testcase_by_id(testcase_id) data_handler.update_testcase_comment( testcase, data_types.TaskState.ERROR, 'Failed to generate request for Predator') return # Clear existing results and mark blame result as pending. testcase = data_handler.get_testcase_by_id(testcase_id) _clear_blame_result_and_set_pending_flag(testcase) # Post request to pub sub. client = pubsub.PubSubClient() message_ids = client.publish(topic, [message]) logs.log( 'Successfully published testcase %s to Predator. Message IDs: %s.' % (testcase_id, message_ids)) data_handler.update_testcase_comment(testcase, data_types.TaskState.FINISHED)
def get(self): """Process a GET request.""" subscription = db_config.get_value('predator_result_topic') if not subscription: logs.log('No Predator subscription configured. Aborting.') return client = pubsub.PubSubClient() messages = client.pull_from_subscription(subscription, acknowledge=True) for message in messages: message = json.loads(message.data) testcase_id = message['crash_identifiers'] try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: logs.log('Testcase %s no longer exists.' % str(testcase_id)) continue testcase.set_metadata('predator_result', message, update_testcase=False) testcase.delete_metadata('blame_pending', update_testcase=False) testcase.put() logs.log('Set predator result for testcase %d.' % testcase.key.id()) logs.log('Finished processing predator results. %d total.' % len(messages))
def setUp(self): helpers.patch(self, [ 'clusterfuzz._internal.config.local_config.ProjectConfig', ]) self.topic = 'projects/project/topics/issue-updates' self.subscription = 'projects/project/subscriptions/issue-updates' self.mock.ProjectConfig.return_value = mock_config.MockConfig({ 'issue_updates': { 'pubsub_topic': self.topic, }, }) self.pubsub_client = pubsub.PubSubClient() self.pubsub_client.create_topic(self.topic) self.pubsub_client.create_subscription(self.subscription, self.topic) self.testcase = data_types.Testcase(crash_address='0xffff', security_flag=True, crash_type='CRASH TYPE', crash_state='CRASH STATE', bug_information='123') self.testcase.put()